1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
38 #include "internal-fn.h"
39 #include "gimple-fold.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
50 #include "tree-into-ssa.h"
52 #include "insn-config.h"
63 #include "tree-pass.h"
65 #include "splay-tree.h"
66 #include "insn-codes.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "pretty-print.h"
75 #include "alloc-pool.h"
76 #include "symbol-summary.h"
78 #include "tree-nested.h"
82 #include "lto-section-names.h"
83 #include "gomp-constants.h"
86 /* Lowering of OMP parallel and workshare constructs proceeds in two
87 phases. The first phase scans the function looking for OMP statements
88 and then for variables that must be replaced to satisfy data sharing
89 clauses. The second phase expands code for the constructs, as well as
90 re-gimplifying things when variables have been replaced with complex
93 Final code generation is done by pass_expand_omp. The flowgraph is
94 scanned for regions which are then moved to a new
95 function, to be invoked by the thread library, or offloaded. */
97 /* OMP region information. Every parallel and workshare
98 directive is enclosed between two markers, the OMP_* directive
99 and a corresponding OMP_RETURN statement. */
103 /* The enclosing region. */
104 struct omp_region
*outer
;
106 /* First child region. */
107 struct omp_region
*inner
;
109 /* Next peer region. */
110 struct omp_region
*next
;
112 /* Block containing the omp directive as its last stmt. */
115 /* Block containing the OMP_RETURN as its last stmt. */
118 /* Block containing the OMP_CONTINUE as its last stmt. */
121 /* If this is a combined parallel+workshare region, this is a list
122 of additional arguments needed by the combined parallel+workshare
124 vec
<tree
, va_gc
> *ws_args
;
126 /* The code for the omp directive of this region. */
127 enum gimple_code type
;
129 /* Schedule kind, only used for OMP_FOR type regions. */
130 enum omp_clause_schedule_kind sched_kind
;
132 /* True if this is a combined parallel+workshare region. */
133 bool is_combined_parallel
;
136 /* Levels of parallelism as defined by OpenACC. Increasing numbers
137 correspond to deeper loop nesting levels. */
139 #define MASK_WORKER 2
140 #define MASK_VECTOR 4
142 /* Context structure. Used to store information about each parallel
143 directive in the code. */
145 typedef struct omp_context
147 /* This field must be at the beginning, as we do "inheritance": Some
148 callback functions for tree-inline.c (e.g., omp_copy_decl)
149 receive a copy_body_data pointer that is up-casted to an
150 omp_context pointer. */
153 /* The tree of contexts corresponding to the encountered constructs. */
154 struct omp_context
*outer
;
157 /* Map variables to fields in a structure that allows communication
158 between sending and receiving threads. */
159 splay_tree field_map
;
164 /* These are used just by task contexts, if task firstprivate fn is
165 needed. srecord_type is used to communicate from the thread
166 that encountered the task construct to task firstprivate fn,
167 record_type is allocated by GOMP_task, initialized by task firstprivate
168 fn and passed to the task body fn. */
169 splay_tree sfield_map
;
172 /* A chain of variables to add to the top-level block surrounding the
173 construct. In the case of a parallel, this is in the child function. */
176 /* A map of reduction pointer variables. For accelerators, each
177 reduction variable is replaced with an array. Each thread, in turn,
178 is assigned to a slot on that array. */
179 splay_tree reduction_map
;
181 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
182 barriers should jump to during omplower pass. */
185 /* What to do with variables with implicitly determined sharing
187 enum omp_clause_default_kind default_kind
;
189 /* Nesting depth of this context. Used to beautify error messages re
190 invalid gotos. The outermost ctx is depth 1, with depth 0 being
191 reserved for the main body of the function. */
194 /* True if this parallel directive is nested within another. */
197 /* True if this construct can be cancelled. */
200 /* For OpenACC loops, a mask of gang, worker and vector used at
201 levels below this one. */
203 /* For OpenACC loops, a mask of gang, worker and vector used at
204 this level and above. For parallel and kernels clauses, a mask
205 indicating which of num_gangs/num_workers/num_vectors was used. */
209 /* A structure holding the elements of:
210 for (V = N1; V cond N2; V += STEP) [...] */
212 struct omp_for_data_loop
214 tree v
, n1
, n2
, step
;
215 enum tree_code cond_code
;
218 /* A structure describing the main elements of a parallel loop. */
222 struct omp_for_data_loop loop
;
227 bool have_nowait
, have_ordered
;
228 enum omp_clause_schedule_kind sched_kind
;
229 struct omp_for_data_loop
*loops
;
233 static splay_tree all_contexts
;
234 static int taskreg_nesting_level
;
235 static int target_nesting_level
;
236 static struct omp_region
*root_omp_region
;
237 static bitmap task_shared_vars
;
238 static vec
<omp_context
*> taskreg_contexts
;
240 static void scan_omp (gimple_seq
*, omp_context
*);
241 static tree
scan_omp_1_op (tree
*, int *, void *);
243 #define WALK_SUBSTMTS \
247 case GIMPLE_EH_FILTER: \
248 case GIMPLE_TRANSACTION: \
249 /* The sub-statements for these should be walked. */ \
250 *handled_ops_p = false; \
253 /* Helper function to get the name of the array containing the partial
254 reductions for OpenACC reductions. */
256 oacc_get_reduction_array_id (tree node
)
258 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
259 int len
= strlen ("OACC") + strlen (id
);
260 char *temp_name
= XALLOCAVEC (char, len
+ 1);
261 snprintf (temp_name
, len
+ 1, "OACC%s", id
);
262 return IDENTIFIER_POINTER (get_identifier (temp_name
));
265 /* Determine the number of threads OpenACC threads used to determine the
266 size of the array of partial reductions. Currently, this is num_gangs
267 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
268 because it is independed of the device used. */
271 oacc_max_threads (omp_context
*ctx
)
273 tree nthreads
, vector_length
, gangs
, clauses
;
275 gangs
= fold_convert (sizetype
, integer_one_node
);
276 vector_length
= gangs
;
278 /* The reduction clause may be nested inside a loop directive.
279 Scan for the innermost vector_length clause. */
280 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
282 if (gimple_code (oc
->stmt
) != GIMPLE_OMP_TARGET
283 || (gimple_omp_target_kind (oc
->stmt
)
284 != GF_OMP_TARGET_KIND_OACC_PARALLEL
))
287 clauses
= gimple_omp_target_clauses (oc
->stmt
);
289 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
291 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
293 OMP_CLAUSE_VECTOR_LENGTH_EXPR
296 vector_length
= fold_convert (sizetype
, integer_one_node
);
298 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
300 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
301 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
303 gangs
= fold_convert (sizetype
, integer_one_node
);
308 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
313 /* Holds offload tables with decls. */
314 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
316 /* Convenience function for calling scan_omp_1_op on tree operands. */
319 scan_omp_op (tree
*tp
, omp_context
*ctx
)
321 struct walk_stmt_info wi
;
323 memset (&wi
, 0, sizeof (wi
));
325 wi
.want_locations
= true;
327 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
330 static void lower_omp (gimple_seq
*, omp_context
*);
331 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
332 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
334 /* Find an OMP clause of type KIND within CLAUSES. */
337 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
339 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
340 if (OMP_CLAUSE_CODE (clauses
) == kind
)
346 /* Return true if CTX is for an omp parallel. */
349 is_parallel_ctx (omp_context
*ctx
)
351 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
355 /* Return true if CTX is for an omp task. */
358 is_task_ctx (omp_context
*ctx
)
360 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
364 /* Return true if CTX is for an omp parallel or omp task. */
367 is_taskreg_ctx (omp_context
*ctx
)
369 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
370 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
374 /* Return true if REGION is a combined parallel+workshare region. */
377 is_combined_parallel (struct omp_region
*region
)
379 return region
->is_combined_parallel
;
383 /* Extract the header elements of parallel loop FOR_STMT and store
387 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
388 struct omp_for_data_loop
*loops
)
390 tree t
, var
, *collapse_iter
, *collapse_count
;
391 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
392 struct omp_for_data_loop
*loop
;
394 struct omp_for_data_loop dummy_loop
;
395 location_t loc
= gimple_location (for_stmt
);
396 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
397 bool distribute
= gimple_omp_for_kind (for_stmt
)
398 == GF_OMP_FOR_KIND_DISTRIBUTE
;
400 fd
->for_stmt
= for_stmt
;
402 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
403 if (fd
->collapse
> 1)
406 fd
->loops
= &fd
->loop
;
408 fd
->have_nowait
= distribute
|| simd
;
409 fd
->have_ordered
= false;
410 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
411 fd
->chunk_size
= NULL_TREE
;
412 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
413 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
414 collapse_iter
= NULL
;
415 collapse_count
= NULL
;
417 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
418 switch (OMP_CLAUSE_CODE (t
))
420 case OMP_CLAUSE_NOWAIT
:
421 fd
->have_nowait
= true;
423 case OMP_CLAUSE_ORDERED
:
424 fd
->have_ordered
= true;
426 case OMP_CLAUSE_SCHEDULE
:
427 gcc_assert (!distribute
);
428 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
429 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
431 case OMP_CLAUSE_DIST_SCHEDULE
:
432 gcc_assert (distribute
);
433 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
435 case OMP_CLAUSE_COLLAPSE
:
436 if (fd
->collapse
> 1)
438 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
439 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
446 /* FIXME: for now map schedule(auto) to schedule(static).
447 There should be analysis to determine whether all iterations
448 are approximately the same amount of work (then schedule(static)
449 is best) or if it varies (then schedule(dynamic,N) is better). */
450 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
452 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
453 gcc_assert (fd
->chunk_size
== NULL
);
455 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
456 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
457 gcc_assert (fd
->chunk_size
== NULL
);
458 else if (fd
->chunk_size
== NULL
)
460 /* We only need to compute a default chunk size for ordered
461 static loops and dynamic loops. */
462 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
464 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
465 ? integer_zero_node
: integer_one_node
;
468 for (i
= 0; i
< fd
->collapse
; i
++)
470 if (fd
->collapse
== 1)
472 else if (loops
!= NULL
)
477 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
478 gcc_assert (SSA_VAR_P (loop
->v
));
479 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
480 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
481 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
482 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
484 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
485 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
486 switch (loop
->cond_code
)
492 gcc_assert (gimple_omp_for_kind (for_stmt
)
493 == GF_OMP_FOR_KIND_CILKSIMD
494 || (gimple_omp_for_kind (for_stmt
)
495 == GF_OMP_FOR_KIND_CILKFOR
));
498 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
499 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
501 loop
->n2
= fold_build2_loc (loc
,
502 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
503 build_int_cst (TREE_TYPE (loop
->n2
), 1));
504 loop
->cond_code
= LT_EXPR
;
507 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
508 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
510 loop
->n2
= fold_build2_loc (loc
,
511 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
512 build_int_cst (TREE_TYPE (loop
->n2
), 1));
513 loop
->cond_code
= GT_EXPR
;
519 t
= gimple_omp_for_incr (for_stmt
, i
);
520 gcc_assert (TREE_OPERAND (t
, 0) == var
);
521 switch (TREE_CODE (t
))
524 loop
->step
= TREE_OPERAND (t
, 1);
526 case POINTER_PLUS_EXPR
:
527 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
530 loop
->step
= TREE_OPERAND (t
, 1);
531 loop
->step
= fold_build1_loc (loc
,
532 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
540 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
541 && !fd
->have_ordered
))
543 if (fd
->collapse
== 1)
544 iter_type
= TREE_TYPE (loop
->v
);
546 || TYPE_PRECISION (iter_type
)
547 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
549 = build_nonstandard_integer_type
550 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
552 else if (iter_type
!= long_long_unsigned_type_node
)
554 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
555 iter_type
= long_long_unsigned_type_node
;
556 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
557 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
558 >= TYPE_PRECISION (iter_type
))
562 if (loop
->cond_code
== LT_EXPR
)
563 n
= fold_build2_loc (loc
,
564 PLUS_EXPR
, TREE_TYPE (loop
->v
),
565 loop
->n2
, loop
->step
);
568 if (TREE_CODE (n
) != INTEGER_CST
569 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
570 iter_type
= long_long_unsigned_type_node
;
572 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
573 > TYPE_PRECISION (iter_type
))
577 if (loop
->cond_code
== LT_EXPR
)
580 n2
= fold_build2_loc (loc
,
581 PLUS_EXPR
, TREE_TYPE (loop
->v
),
582 loop
->n2
, loop
->step
);
586 n1
= fold_build2_loc (loc
,
587 MINUS_EXPR
, TREE_TYPE (loop
->v
),
588 loop
->n2
, loop
->step
);
591 if (TREE_CODE (n1
) != INTEGER_CST
592 || TREE_CODE (n2
) != INTEGER_CST
593 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
594 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
595 iter_type
= long_long_unsigned_type_node
;
599 if (collapse_count
&& *collapse_count
== NULL
)
601 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
602 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
603 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
604 if (t
&& integer_zerop (t
))
605 count
= build_zero_cst (long_long_unsigned_type_node
);
606 else if ((i
== 0 || count
!= NULL_TREE
)
607 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
608 && TREE_CONSTANT (loop
->n1
)
609 && TREE_CONSTANT (loop
->n2
)
610 && TREE_CODE (loop
->step
) == INTEGER_CST
)
612 tree itype
= TREE_TYPE (loop
->v
);
614 if (POINTER_TYPE_P (itype
))
615 itype
= signed_type_for (itype
);
616 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
617 t
= fold_build2_loc (loc
,
619 fold_convert_loc (loc
, itype
, loop
->step
), t
);
620 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
621 fold_convert_loc (loc
, itype
, loop
->n2
));
622 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
623 fold_convert_loc (loc
, itype
, loop
->n1
));
624 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
625 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
626 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
627 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
628 fold_convert_loc (loc
, itype
,
631 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
632 fold_convert_loc (loc
, itype
, loop
->step
));
633 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
634 if (count
!= NULL_TREE
)
635 count
= fold_build2_loc (loc
,
636 MULT_EXPR
, long_long_unsigned_type_node
,
640 if (TREE_CODE (count
) != INTEGER_CST
)
643 else if (count
&& !integer_zerop (count
))
650 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
651 || fd
->have_ordered
))
653 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
654 iter_type
= long_long_unsigned_type_node
;
656 iter_type
= long_integer_type_node
;
658 else if (collapse_iter
&& *collapse_iter
!= NULL
)
659 iter_type
= TREE_TYPE (*collapse_iter
);
660 fd
->iter_type
= iter_type
;
661 if (collapse_iter
&& *collapse_iter
== NULL
)
662 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
663 if (collapse_count
&& *collapse_count
== NULL
)
666 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
668 *collapse_count
= create_tmp_var (iter_type
, ".count");
671 if (fd
->collapse
> 1)
673 fd
->loop
.v
= *collapse_iter
;
674 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
675 fd
->loop
.n2
= *collapse_count
;
676 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
677 fd
->loop
.cond_code
= LT_EXPR
;
680 /* For OpenACC loops, force a chunk size of one, as this avoids the default
681 scheduling where several subsequent iterations are being executed by the
683 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
685 gcc_assert (fd
->chunk_size
== NULL_TREE
);
686 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
691 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
692 is the immediate dominator of PAR_ENTRY_BB, return true if there
693 are no data dependencies that would prevent expanding the parallel
694 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
696 When expanding a combined parallel+workshare region, the call to
697 the child function may need additional arguments in the case of
698 GIMPLE_OMP_FOR regions. In some cases, these arguments are
699 computed out of variables passed in from the parent to the child
700 via 'struct .omp_data_s'. For instance:
702 #pragma omp parallel for schedule (guided, i * 4)
707 # BLOCK 2 (PAR_ENTRY_BB)
709 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
711 # BLOCK 3 (WS_ENTRY_BB)
712 .omp_data_i = &.omp_data_o;
713 D.1667 = .omp_data_i->i;
715 #pragma omp for schedule (guided, D.1598)
717 When we outline the parallel region, the call to the child function
718 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
719 that value is computed *after* the call site. So, in principle we
720 cannot do the transformation.
722 To see whether the code in WS_ENTRY_BB blocks the combined
723 parallel+workshare call, we collect all the variables used in the
724 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
725 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
728 FIXME. If we had the SSA form built at this point, we could merely
729 hoist the code in block 3 into block 2 and be done with it. But at
730 this point we don't have dataflow information and though we could
731 hack something up here, it is really not worth the aggravation. */
734 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
736 struct omp_for_data fd
;
737 gimple ws_stmt
= last_stmt (ws_entry_bb
);
739 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
742 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
744 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
746 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
748 if (fd
.iter_type
!= long_integer_type_node
)
751 /* FIXME. We give up too easily here. If any of these arguments
752 are not constants, they will likely involve variables that have
753 been mapped into fields of .omp_data_s for sharing with the child
754 function. With appropriate data flow, it would be possible to
756 if (!is_gimple_min_invariant (fd
.loop
.n1
)
757 || !is_gimple_min_invariant (fd
.loop
.n2
)
758 || !is_gimple_min_invariant (fd
.loop
.step
)
759 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
766 /* Collect additional arguments needed to emit a combined
767 parallel+workshare call. WS_STMT is the workshare directive being
770 static vec
<tree
, va_gc
> *
771 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
774 location_t loc
= gimple_location (ws_stmt
);
775 vec
<tree
, va_gc
> *ws_args
;
777 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
779 struct omp_for_data fd
;
782 extract_omp_for_data (for_stmt
, &fd
, NULL
);
786 if (gimple_omp_for_combined_into_p (for_stmt
))
789 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
790 OMP_CLAUSE__LOOPTEMP_
);
792 n1
= OMP_CLAUSE_DECL (innerc
);
793 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
794 OMP_CLAUSE__LOOPTEMP_
);
796 n2
= OMP_CLAUSE_DECL (innerc
);
799 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
801 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
802 ws_args
->quick_push (t
);
804 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
805 ws_args
->quick_push (t
);
807 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
808 ws_args
->quick_push (t
);
812 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
813 ws_args
->quick_push (t
);
818 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
820 /* Number of sections is equal to the number of edges from the
821 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
822 the exit of the sections region. */
823 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
824 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
825 vec_alloc (ws_args
, 1);
826 ws_args
->quick_push (t
);
834 /* Discover whether REGION is a combined parallel+workshare region. */
837 determine_parallel_type (struct omp_region
*region
)
839 basic_block par_entry_bb
, par_exit_bb
;
840 basic_block ws_entry_bb
, ws_exit_bb
;
842 if (region
== NULL
|| region
->inner
== NULL
843 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
844 || region
->inner
->cont
== NULL
)
847 /* We only support parallel+for and parallel+sections. */
848 if (region
->type
!= GIMPLE_OMP_PARALLEL
849 || (region
->inner
->type
!= GIMPLE_OMP_FOR
850 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
853 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
854 WS_EXIT_BB -> PAR_EXIT_BB. */
855 par_entry_bb
= region
->entry
;
856 par_exit_bb
= region
->exit
;
857 ws_entry_bb
= region
->inner
->entry
;
858 ws_exit_bb
= region
->inner
->exit
;
860 if (single_succ (par_entry_bb
) == ws_entry_bb
861 && single_succ (ws_exit_bb
) == par_exit_bb
862 && workshare_safe_to_combine_p (ws_entry_bb
)
863 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
864 || (last_and_only_stmt (ws_entry_bb
)
865 && last_and_only_stmt (par_exit_bb
))))
867 gimple par_stmt
= last_stmt (par_entry_bb
);
868 gimple ws_stmt
= last_stmt (ws_entry_bb
);
870 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
872 /* If this is a combined parallel loop, we need to determine
873 whether or not to use the combined library calls. There
874 are two cases where we do not apply the transformation:
875 static loops and any kind of ordered loop. In the first
876 case, we already open code the loop so there is no need
877 to do anything else. In the latter case, the combined
878 parallel loop call would still need extra synchronization
879 to implement ordered semantics, so there would not be any
880 gain in using the combined call. */
881 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
882 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
884 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
885 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
887 region
->is_combined_parallel
= false;
888 region
->inner
->is_combined_parallel
= false;
893 region
->is_combined_parallel
= true;
894 region
->inner
->is_combined_parallel
= true;
895 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
900 /* Return true if EXPR is variable sized. */
903 is_variable_sized (const_tree expr
)
905 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
908 /* Return true if DECL is a reference type. */
911 is_reference (tree decl
)
913 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
916 /* Return the type of a decl. If the decl is reference type,
917 return its base type. */
919 get_base_type (tree decl
)
921 tree type
= TREE_TYPE (decl
);
922 if (is_reference (decl
))
923 type
= TREE_TYPE (type
);
927 /* Lookup variables. The "maybe" form
928 allows for the variable form to not have been entered, otherwise we
929 assert that the variable must have been entered. */
932 lookup_decl (tree var
, omp_context
*ctx
)
934 tree
*n
= ctx
->cb
.decl_map
->get (var
);
939 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
941 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
942 return n
? *n
: NULL_TREE
;
946 lookup_field (tree var
, omp_context
*ctx
)
949 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
950 return (tree
) n
->value
;
954 lookup_sfield (tree var
, omp_context
*ctx
)
957 n
= splay_tree_lookup (ctx
->sfield_map
958 ? ctx
->sfield_map
: ctx
->field_map
,
959 (splay_tree_key
) var
);
960 return (tree
) n
->value
;
964 maybe_lookup_field (tree var
, omp_context
*ctx
)
967 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
968 return n
? (tree
) n
->value
: NULL_TREE
;
972 lookup_oacc_reduction (const char *id
, omp_context
*ctx
)
975 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
976 return (tree
) n
->value
;
980 maybe_lookup_oacc_reduction (tree var
, omp_context
*ctx
)
982 splay_tree_node n
= NULL
;
983 if (ctx
->reduction_map
)
984 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
985 return n
? (tree
) n
->value
: NULL_TREE
;
988 /* Return true if DECL should be copied by pointer. SHARED_CTX is
989 the parallel context if DECL is to be shared. */
992 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
994 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
997 /* We can only use copy-in/copy-out semantics for shared variables
998 when we know the value is not accessible from an outer scope. */
1001 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1003 /* ??? Trivially accessible from anywhere. But why would we even
1004 be passing an address in this case? Should we simply assert
1005 this to be false, or should we have a cleanup pass that removes
1006 these from the list of mappings? */
1007 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1010 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1011 without analyzing the expression whether or not its location
1012 is accessible to anyone else. In the case of nested parallel
1013 regions it certainly may be. */
1014 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1017 /* Do not use copy-in/copy-out for variables that have their
1019 if (TREE_ADDRESSABLE (decl
))
1022 /* lower_send_shared_vars only uses copy-in, but not copy-out
1024 if (TREE_READONLY (decl
)
1025 || ((TREE_CODE (decl
) == RESULT_DECL
1026 || TREE_CODE (decl
) == PARM_DECL
)
1027 && DECL_BY_REFERENCE (decl
)))
1030 /* Disallow copy-in/out in nested parallel if
1031 decl is shared in outer parallel, otherwise
1032 each thread could store the shared variable
1033 in its own copy-in location, making the
1034 variable no longer really shared. */
1035 if (shared_ctx
->is_nested
)
1039 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1040 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1047 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1048 c
; c
= OMP_CLAUSE_CHAIN (c
))
1049 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1050 && OMP_CLAUSE_DECL (c
) == decl
)
1054 goto maybe_mark_addressable_and_ret
;
1058 /* For tasks avoid using copy-in/out. As tasks can be
1059 deferred or executed in different thread, when GOMP_task
1060 returns, the task hasn't necessarily terminated. */
1061 if (is_task_ctx (shared_ctx
))
1064 maybe_mark_addressable_and_ret
:
1065 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1066 if (is_gimple_reg (outer
))
1068 /* Taking address of OUTER in lower_send_shared_vars
1069 might need regimplification of everything that uses the
1071 if (!task_shared_vars
)
1072 task_shared_vars
= BITMAP_ALLOC (NULL
);
1073 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1074 TREE_ADDRESSABLE (outer
) = 1;
1083 /* Construct a new automatic decl similar to VAR. */
1086 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1088 tree copy
= copy_var_decl (var
, name
, type
);
1090 DECL_CONTEXT (copy
) = current_function_decl
;
1091 DECL_CHAIN (copy
) = ctx
->block_vars
;
1092 ctx
->block_vars
= copy
;
1098 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1100 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1103 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1106 omp_build_component_ref (tree obj
, tree field
)
1108 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1109 if (TREE_THIS_VOLATILE (field
))
1110 TREE_THIS_VOLATILE (ret
) |= 1;
1111 if (TREE_READONLY (field
))
1112 TREE_READONLY (ret
) |= 1;
1116 /* Build tree nodes to access the field for VAR on the receiver side. */
1119 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1121 tree x
, field
= lookup_field (var
, ctx
);
1123 /* If the receiver record type was remapped in the child function,
1124 remap the field into the new record type. */
1125 x
= maybe_lookup_field (field
, ctx
);
1129 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1130 TREE_THIS_NOTRAP (x
) = 1;
1131 x
= omp_build_component_ref (x
, field
);
1133 x
= build_simple_mem_ref (x
);
1138 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1139 of a parallel, this is a component reference; for workshare constructs
1140 this is some variable. */
1143 build_outer_var_ref (tree var
, omp_context
*ctx
)
1147 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1149 else if (is_variable_sized (var
))
1151 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1152 x
= build_outer_var_ref (x
, ctx
);
1153 x
= build_simple_mem_ref (x
);
1155 else if (is_taskreg_ctx (ctx
))
1157 bool by_ref
= use_pointer_for_field (var
, NULL
);
1158 x
= build_receiver_ref (var
, by_ref
, ctx
);
1160 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1161 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1163 /* #pragma omp simd isn't a worksharing construct, and can reference even
1164 private vars in its linear etc. clauses. */
1166 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1167 x
= lookup_decl (var
, ctx
->outer
);
1168 else if (ctx
->outer
)
1169 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1173 else if (ctx
->outer
)
1174 x
= lookup_decl (var
, ctx
->outer
);
1175 else if (is_reference (var
))
1176 /* This can happen with orphaned constructs. If var is reference, it is
1177 possible it is shared and as such valid. */
1182 if (is_reference (var
))
1183 x
= build_simple_mem_ref (x
);
1188 /* Build tree nodes to access the field for VAR on the sender side. */
1191 build_sender_ref (tree var
, omp_context
*ctx
)
1193 tree field
= lookup_sfield (var
, ctx
);
1194 return omp_build_component_ref (ctx
->sender_decl
, field
);
1197 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1200 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1202 tree field
, type
, sfield
= NULL_TREE
;
1204 gcc_assert ((mask
& 1) == 0
1205 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1206 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1207 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1208 gcc_assert ((mask
& 3) == 3
1209 || !is_gimple_omp_oacc (ctx
->stmt
));
1211 type
= TREE_TYPE (var
);
1214 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1215 type
= build_pointer_type (build_pointer_type (type
));
1218 type
= build_pointer_type (type
);
1219 else if ((mask
& 3) == 1 && is_reference (var
))
1220 type
= TREE_TYPE (type
);
1222 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1223 FIELD_DECL
, DECL_NAME (var
), type
);
1225 /* Remember what variable this field was created for. This does have a
1226 side effect of making dwarf2out ignore this member, so for helpful
1227 debugging we clear it later in delete_omp_context. */
1228 DECL_ABSTRACT_ORIGIN (field
) = var
;
1229 if (type
== TREE_TYPE (var
))
1231 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1232 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1233 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1236 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1238 if ((mask
& 3) == 3)
1240 insert_field_into_struct (ctx
->record_type
, field
);
1241 if (ctx
->srecord_type
)
1243 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1244 FIELD_DECL
, DECL_NAME (var
), type
);
1245 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1246 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1247 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1248 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1249 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1254 if (ctx
->srecord_type
== NULL_TREE
)
1258 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1259 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1260 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1262 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1263 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1264 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1265 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1266 splay_tree_insert (ctx
->sfield_map
,
1267 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1268 (splay_tree_value
) sfield
);
1272 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1273 : ctx
->srecord_type
, field
);
1277 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1278 (splay_tree_value
) field
);
1279 if ((mask
& 2) && ctx
->sfield_map
)
1280 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1281 (splay_tree_value
) sfield
);
1285 install_var_local (tree var
, omp_context
*ctx
)
1287 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1288 insert_decl_map (&ctx
->cb
, var
, new_var
);
1292 /* Adjust the replacement for DECL in CTX for the new context. This means
1293 copying the DECL_VALUE_EXPR, and fixing up the type. */
1296 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1298 tree new_decl
, size
;
1300 new_decl
= lookup_decl (decl
, ctx
);
1302 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1304 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1305 && DECL_HAS_VALUE_EXPR_P (decl
))
1307 tree ve
= DECL_VALUE_EXPR (decl
);
1308 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1309 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1310 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1313 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1315 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1316 if (size
== error_mark_node
)
1317 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1318 DECL_SIZE (new_decl
) = size
;
1320 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1321 if (size
== error_mark_node
)
1322 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1323 DECL_SIZE_UNIT (new_decl
) = size
;
1327 /* The callback for remap_decl. Search all containing contexts for a
1328 mapping of the variable; this avoids having to duplicate the splay
1329 tree ahead of time. We know a mapping doesn't already exist in the
1330 given context. Create new mappings to implement default semantics. */
1333 omp_copy_decl (tree var
, copy_body_data
*cb
)
1335 omp_context
*ctx
= (omp_context
*) cb
;
1338 if (TREE_CODE (var
) == LABEL_DECL
)
1340 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1341 DECL_CONTEXT (new_var
) = current_function_decl
;
1342 insert_decl_map (&ctx
->cb
, var
, new_var
);
1346 while (!is_taskreg_ctx (ctx
))
1351 new_var
= maybe_lookup_decl (var
, ctx
);
1356 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1359 return error_mark_node
;
1363 /* Debugging dumps for parallel regions. */
1364 void dump_omp_region (FILE *, struct omp_region
*, int);
1365 void debug_omp_region (struct omp_region
*);
1366 void debug_all_omp_regions (void);
1368 /* Dump the parallel region tree rooted at REGION. */
1371 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1373 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1374 gimple_code_name
[region
->type
]);
1377 dump_omp_region (file
, region
->inner
, indent
+ 4);
1381 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1382 region
->cont
->index
);
1386 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1387 region
->exit
->index
);
1389 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1392 dump_omp_region (file
, region
->next
, indent
);
1396 debug_omp_region (struct omp_region
*region
)
1398 dump_omp_region (stderr
, region
, 0);
1402 debug_all_omp_regions (void)
1404 dump_omp_region (stderr
, root_omp_region
, 0);
1408 /* Create a new parallel region starting at STMT inside region PARENT. */
1410 static struct omp_region
*
1411 new_omp_region (basic_block bb
, enum gimple_code type
,
1412 struct omp_region
*parent
)
1414 struct omp_region
*region
= XCNEW (struct omp_region
);
1416 region
->outer
= parent
;
1418 region
->type
= type
;
1422 /* This is a nested region. Add it to the list of inner
1423 regions in PARENT. */
1424 region
->next
= parent
->inner
;
1425 parent
->inner
= region
;
1429 /* This is a toplevel region. Add it to the list of toplevel
1430 regions in ROOT_OMP_REGION. */
1431 region
->next
= root_omp_region
;
1432 root_omp_region
= region
;
1438 /* Release the memory associated with the region tree rooted at REGION. */
1441 free_omp_region_1 (struct omp_region
*region
)
1443 struct omp_region
*i
, *n
;
1445 for (i
= region
->inner
; i
; i
= n
)
1448 free_omp_region_1 (i
);
1454 /* Release the memory for the entire omp region tree. */
1457 free_omp_regions (void)
1459 struct omp_region
*r
, *n
;
1460 for (r
= root_omp_region
; r
; r
= n
)
1463 free_omp_region_1 (r
);
1465 root_omp_region
= NULL
;
1469 /* Create a new context, with OUTER_CTX being the surrounding context. */
1471 static omp_context
*
1472 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1474 omp_context
*ctx
= XCNEW (omp_context
);
1476 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1477 (splay_tree_value
) ctx
);
1482 ctx
->outer
= outer_ctx
;
1483 ctx
->cb
= outer_ctx
->cb
;
1484 ctx
->cb
.block
= NULL
;
1485 ctx
->depth
= outer_ctx
->depth
+ 1;
1486 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1490 ctx
->cb
.src_fn
= current_function_decl
;
1491 ctx
->cb
.dst_fn
= current_function_decl
;
1492 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1493 gcc_checking_assert (ctx
->cb
.src_node
);
1494 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1495 ctx
->cb
.src_cfun
= cfun
;
1496 ctx
->cb
.copy_decl
= omp_copy_decl
;
1497 ctx
->cb
.eh_lp_nr
= 0;
1498 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1502 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1507 static gimple_seq
maybe_catch_exception (gimple_seq
);
1509 /* Finalize task copyfn. */
1512 finalize_task_copyfn (gomp_task
*task_stmt
)
1514 struct function
*child_cfun
;
1516 gimple_seq seq
= NULL
, new_seq
;
1519 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1520 if (child_fn
== NULL_TREE
)
1523 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1524 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1526 push_cfun (child_cfun
);
1527 bind
= gimplify_body (child_fn
, false);
1528 gimple_seq_add_stmt (&seq
, bind
);
1529 new_seq
= maybe_catch_exception (seq
);
1532 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1534 gimple_seq_add_stmt (&seq
, bind
);
1536 gimple_set_body (child_fn
, seq
);
1539 /* Inform the callgraph about the new function. */
1540 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1541 node
->parallelized_function
= 1;
1542 cgraph_node::add_new_function (child_fn
, false);
1545 /* Destroy a omp_context data structures. Called through the splay tree
1546 value delete callback. */
1549 delete_omp_context (splay_tree_value value
)
1551 omp_context
*ctx
= (omp_context
*) value
;
1553 delete ctx
->cb
.decl_map
;
1556 splay_tree_delete (ctx
->field_map
);
1557 if (ctx
->sfield_map
)
1558 splay_tree_delete (ctx
->sfield_map
);
1559 /* Reduction map is copied to nested contexts, so only delete it in the
1561 if (ctx
->reduction_map
1562 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
1563 && is_gimple_omp_offloaded (ctx
->stmt
)
1564 && is_gimple_omp_oacc (ctx
->stmt
))
1565 splay_tree_delete (ctx
->reduction_map
);
1567 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1568 it produces corrupt debug information. */
1569 if (ctx
->record_type
)
1572 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1573 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1575 if (ctx
->srecord_type
)
1578 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1579 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1582 if (is_task_ctx (ctx
))
1583 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1588 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1592 fixup_child_record_type (omp_context
*ctx
)
1594 tree f
, type
= ctx
->record_type
;
1596 /* ??? It isn't sufficient to just call remap_type here, because
1597 variably_modified_type_p doesn't work the way we expect for
1598 record types. Testing each field for whether it needs remapping
1599 and creating a new record by hand works, however. */
1600 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1601 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1605 tree name
, new_fields
= NULL
;
1607 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1608 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1609 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1610 TYPE_DECL
, name
, type
);
1611 TYPE_NAME (type
) = name
;
1613 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1615 tree new_f
= copy_node (f
);
1616 DECL_CONTEXT (new_f
) = type
;
1617 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1618 DECL_CHAIN (new_f
) = new_fields
;
1619 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1620 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1622 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1626 /* Arrange to be able to look up the receiver field
1627 given the sender field. */
1628 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1629 (splay_tree_value
) new_f
);
1631 TYPE_FIELDS (type
) = nreverse (new_fields
);
1635 TREE_TYPE (ctx
->receiver_decl
)
1636 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1639 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1640 specified by CLAUSES. */
1643 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1646 bool scan_array_reductions
= false;
1648 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1652 switch (OMP_CLAUSE_CODE (c
))
1654 case OMP_CLAUSE_PRIVATE
:
1655 decl
= OMP_CLAUSE_DECL (c
);
1656 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1658 else if (!is_variable_sized (decl
))
1659 install_var_local (decl
, ctx
);
1662 case OMP_CLAUSE_SHARED
:
1663 decl
= OMP_CLAUSE_DECL (c
);
1664 /* Ignore shared directives in teams construct. */
1665 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1667 /* Global variables don't need to be copied,
1668 the receiver side will use them directly. */
1669 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1670 if (is_global_var (odecl
))
1672 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1675 gcc_assert (is_taskreg_ctx (ctx
));
1676 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1677 || !is_variable_sized (decl
));
1678 /* Global variables don't need to be copied,
1679 the receiver side will use them directly. */
1680 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1682 by_ref
= use_pointer_for_field (decl
, ctx
);
1683 if (! TREE_READONLY (decl
)
1684 || TREE_ADDRESSABLE (decl
)
1686 || is_reference (decl
))
1688 install_var_field (decl
, by_ref
, 3, ctx
);
1689 install_var_local (decl
, ctx
);
1692 /* We don't need to copy const scalar vars back. */
1693 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1696 case OMP_CLAUSE_LASTPRIVATE
:
1697 /* Let the corresponding firstprivate clause create
1699 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1703 case OMP_CLAUSE_FIRSTPRIVATE
:
1704 if (is_gimple_omp_oacc (ctx
->stmt
))
1706 sorry ("clause not supported yet");
1710 case OMP_CLAUSE_REDUCTION
:
1711 case OMP_CLAUSE_LINEAR
:
1712 decl
= OMP_CLAUSE_DECL (c
);
1714 if (is_variable_sized (decl
))
1716 if (is_task_ctx (ctx
))
1717 install_var_field (decl
, false, 1, ctx
);
1720 else if (is_taskreg_ctx (ctx
))
1723 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1724 by_ref
= use_pointer_for_field (decl
, NULL
);
1726 if (is_task_ctx (ctx
)
1727 && (global
|| by_ref
|| is_reference (decl
)))
1729 install_var_field (decl
, false, 1, ctx
);
1731 install_var_field (decl
, by_ref
, 2, ctx
);
1734 install_var_field (decl
, by_ref
, 3, ctx
);
1736 install_var_local (decl
, ctx
);
1737 if (is_gimple_omp_oacc (ctx
->stmt
)
1738 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
1740 /* Create a decl for the reduction array. */
1741 tree var
= OMP_CLAUSE_DECL (c
);
1742 tree type
= get_base_type (var
);
1743 tree ptype
= build_pointer_type (type
);
1744 tree array
= create_tmp_var (ptype
,
1745 oacc_get_reduction_array_id (var
));
1746 omp_context
*c
= (ctx
->field_map
? ctx
: ctx
->outer
);
1747 install_var_field (array
, true, 3, c
);
1748 install_var_local (array
, c
);
1750 /* Insert it into the current context. */
1751 splay_tree_insert (ctx
->reduction_map
, (splay_tree_key
)
1752 oacc_get_reduction_array_id (var
),
1753 (splay_tree_value
) array
);
1754 splay_tree_insert (ctx
->reduction_map
,
1755 (splay_tree_key
) array
,
1756 (splay_tree_value
) array
);
1760 case OMP_CLAUSE__LOOPTEMP_
:
1761 gcc_assert (is_parallel_ctx (ctx
));
1762 decl
= OMP_CLAUSE_DECL (c
);
1763 install_var_field (decl
, false, 3, ctx
);
1764 install_var_local (decl
, ctx
);
1767 case OMP_CLAUSE_COPYPRIVATE
:
1768 case OMP_CLAUSE_COPYIN
:
1769 decl
= OMP_CLAUSE_DECL (c
);
1770 by_ref
= use_pointer_for_field (decl
, NULL
);
1771 install_var_field (decl
, by_ref
, 3, ctx
);
1774 case OMP_CLAUSE_DEFAULT
:
1775 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1778 case OMP_CLAUSE_FINAL
:
1780 case OMP_CLAUSE_NUM_THREADS
:
1781 case OMP_CLAUSE_NUM_TEAMS
:
1782 case OMP_CLAUSE_THREAD_LIMIT
:
1783 case OMP_CLAUSE_DEVICE
:
1784 case OMP_CLAUSE_SCHEDULE
:
1785 case OMP_CLAUSE_DIST_SCHEDULE
:
1786 case OMP_CLAUSE_DEPEND
:
1787 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1788 case OMP_CLAUSE_NUM_GANGS
:
1789 case OMP_CLAUSE_NUM_WORKERS
:
1790 case OMP_CLAUSE_VECTOR_LENGTH
:
1792 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1796 case OMP_CLAUSE_FROM
:
1797 case OMP_CLAUSE_MAP
:
1799 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1800 decl
= OMP_CLAUSE_DECL (c
);
1801 /* Global variables with "omp declare target" attribute
1802 don't need to be copied, the receiver side will use them
1804 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1806 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1807 && varpool_node::get_create (decl
)->offloadable
)
1809 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1810 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
1812 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1813 not offloaded; there is nothing to map for those. */
1814 if (!is_gimple_omp_offloaded (ctx
->stmt
)
1815 && !POINTER_TYPE_P (TREE_TYPE (decl
))
1816 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
1821 if (DECL_SIZE (decl
)
1822 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1824 tree decl2
= DECL_VALUE_EXPR (decl
);
1825 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1826 decl2
= TREE_OPERAND (decl2
, 0);
1827 gcc_assert (DECL_P (decl2
));
1828 install_var_field (decl2
, true, 3, ctx
);
1829 install_var_local (decl2
, ctx
);
1830 install_var_local (decl
, ctx
);
1834 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1835 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1836 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1837 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1838 install_var_field (decl
, true, 7, ctx
);
1840 install_var_field (decl
, true, 3, ctx
);
1841 if (is_gimple_omp_offloaded (ctx
->stmt
))
1842 install_var_local (decl
, ctx
);
1847 tree base
= get_base_address (decl
);
1848 tree nc
= OMP_CLAUSE_CHAIN (c
);
1851 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1852 && OMP_CLAUSE_DECL (nc
) == base
1853 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
1854 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1856 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1857 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1863 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1864 decl
= OMP_CLAUSE_DECL (c
);
1866 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1867 (splay_tree_key
) decl
));
1869 = build_decl (OMP_CLAUSE_LOCATION (c
),
1870 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1871 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1872 insert_field_into_struct (ctx
->record_type
, field
);
1873 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1874 (splay_tree_value
) field
);
1879 case OMP_CLAUSE_NOWAIT
:
1880 case OMP_CLAUSE_ORDERED
:
1881 case OMP_CLAUSE_COLLAPSE
:
1882 case OMP_CLAUSE_UNTIED
:
1883 case OMP_CLAUSE_MERGEABLE
:
1884 case OMP_CLAUSE_PROC_BIND
:
1885 case OMP_CLAUSE_SAFELEN
:
1886 case OMP_CLAUSE_ASYNC
:
1887 case OMP_CLAUSE_WAIT
:
1888 case OMP_CLAUSE_GANG
:
1889 case OMP_CLAUSE_WORKER
:
1890 case OMP_CLAUSE_VECTOR
:
1893 case OMP_CLAUSE_ALIGNED
:
1894 decl
= OMP_CLAUSE_DECL (c
);
1895 if (is_global_var (decl
)
1896 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1897 install_var_local (decl
, ctx
);
1900 case OMP_CLAUSE_DEVICE_RESIDENT
:
1901 case OMP_CLAUSE_USE_DEVICE
:
1902 case OMP_CLAUSE__CACHE_
:
1903 case OMP_CLAUSE_INDEPENDENT
:
1904 case OMP_CLAUSE_AUTO
:
1905 case OMP_CLAUSE_SEQ
:
1906 sorry ("Clause not supported yet");
1914 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1916 switch (OMP_CLAUSE_CODE (c
))
1918 case OMP_CLAUSE_LASTPRIVATE
:
1919 /* Let the corresponding firstprivate clause create
1921 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1922 scan_array_reductions
= true;
1923 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1927 case OMP_CLAUSE_FIRSTPRIVATE
:
1928 if (is_gimple_omp_oacc (ctx
->stmt
))
1930 sorry ("clause not supported yet");
1934 case OMP_CLAUSE_PRIVATE
:
1935 case OMP_CLAUSE_REDUCTION
:
1936 case OMP_CLAUSE_LINEAR
:
1937 decl
= OMP_CLAUSE_DECL (c
);
1938 if (is_variable_sized (decl
))
1939 install_var_local (decl
, ctx
);
1940 fixup_remapped_decl (decl
, ctx
,
1941 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1942 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1943 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1944 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1945 scan_array_reductions
= true;
1946 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1947 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1948 scan_array_reductions
= true;
1951 case OMP_CLAUSE_SHARED
:
1952 /* Ignore shared directives in teams construct. */
1953 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1955 decl
= OMP_CLAUSE_DECL (c
);
1956 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1957 fixup_remapped_decl (decl
, ctx
, false);
1960 case OMP_CLAUSE_MAP
:
1961 if (!is_gimple_omp_offloaded (ctx
->stmt
))
1963 decl
= OMP_CLAUSE_DECL (c
);
1965 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1966 && varpool_node::get_create (decl
)->offloadable
)
1970 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1971 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1972 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1974 tree new_decl
= lookup_decl (decl
, ctx
);
1975 TREE_TYPE (new_decl
)
1976 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1978 else if (DECL_SIZE (decl
)
1979 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1981 tree decl2
= DECL_VALUE_EXPR (decl
);
1982 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1983 decl2
= TREE_OPERAND (decl2
, 0);
1984 gcc_assert (DECL_P (decl2
));
1985 fixup_remapped_decl (decl2
, ctx
, false);
1986 fixup_remapped_decl (decl
, ctx
, true);
1989 fixup_remapped_decl (decl
, ctx
, false);
1993 case OMP_CLAUSE_COPYPRIVATE
:
1994 case OMP_CLAUSE_COPYIN
:
1995 case OMP_CLAUSE_DEFAULT
:
1997 case OMP_CLAUSE_NUM_THREADS
:
1998 case OMP_CLAUSE_NUM_TEAMS
:
1999 case OMP_CLAUSE_THREAD_LIMIT
:
2000 case OMP_CLAUSE_DEVICE
:
2001 case OMP_CLAUSE_SCHEDULE
:
2002 case OMP_CLAUSE_DIST_SCHEDULE
:
2003 case OMP_CLAUSE_NOWAIT
:
2004 case OMP_CLAUSE_ORDERED
:
2005 case OMP_CLAUSE_COLLAPSE
:
2006 case OMP_CLAUSE_UNTIED
:
2007 case OMP_CLAUSE_FINAL
:
2008 case OMP_CLAUSE_MERGEABLE
:
2009 case OMP_CLAUSE_PROC_BIND
:
2010 case OMP_CLAUSE_SAFELEN
:
2011 case OMP_CLAUSE_ALIGNED
:
2012 case OMP_CLAUSE_DEPEND
:
2013 case OMP_CLAUSE__LOOPTEMP_
:
2015 case OMP_CLAUSE_FROM
:
2016 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2017 case OMP_CLAUSE_ASYNC
:
2018 case OMP_CLAUSE_WAIT
:
2019 case OMP_CLAUSE_NUM_GANGS
:
2020 case OMP_CLAUSE_NUM_WORKERS
:
2021 case OMP_CLAUSE_VECTOR_LENGTH
:
2022 case OMP_CLAUSE_GANG
:
2023 case OMP_CLAUSE_WORKER
:
2024 case OMP_CLAUSE_VECTOR
:
2027 case OMP_CLAUSE_DEVICE_RESIDENT
:
2028 case OMP_CLAUSE_USE_DEVICE
:
2029 case OMP_CLAUSE__CACHE_
:
2030 case OMP_CLAUSE_INDEPENDENT
:
2031 case OMP_CLAUSE_AUTO
:
2032 case OMP_CLAUSE_SEQ
:
2033 sorry ("Clause not supported yet");
2041 gcc_checking_assert (!scan_array_reductions
2042 || !is_gimple_omp_oacc (ctx
->stmt
));
2043 if (scan_array_reductions
)
2044 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2045 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2046 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2048 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2049 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2051 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2052 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2053 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2054 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2055 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2056 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2059 /* Create a new name for omp child function. Returns an identifier. If
2060 IS_CILK_FOR is true then the suffix for the child function is
2064 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2067 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2068 return clone_function_name (current_function_decl
,
2069 task_copy
? "_omp_cpyfn" : "_omp_fn");
2072 /* Returns the type of the induction variable for the child function for
2073 _Cilk_for and the types for _high and _low variables based on TYPE. */
2076 cilk_for_check_loop_diff_type (tree type
)
2078 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2080 if (TYPE_UNSIGNED (type
))
2081 return uint32_type_node
;
2083 return integer_type_node
;
2087 if (TYPE_UNSIGNED (type
))
2088 return uint64_type_node
;
2090 return long_long_integer_type_node
;
2094 /* Build a decl for the omp child function. It'll not contain a body
2095 yet, just the bare decl. */
2098 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2100 tree decl
, type
, name
, t
;
2103 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2104 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2105 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2106 tree cilk_var_type
= NULL_TREE
;
2108 name
= create_omp_child_function_name (task_copy
,
2109 cilk_for_count
!= NULL_TREE
);
2111 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2112 ptr_type_node
, NULL_TREE
);
2113 else if (cilk_for_count
)
2115 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2116 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2117 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2118 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2121 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2123 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2125 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2128 ctx
->cb
.dst_fn
= decl
;
2130 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2132 TREE_STATIC (decl
) = 1;
2133 TREE_USED (decl
) = 1;
2134 DECL_ARTIFICIAL (decl
) = 1;
2135 DECL_IGNORED_P (decl
) = 0;
2136 TREE_PUBLIC (decl
) = 0;
2137 DECL_UNINLINABLE (decl
) = 1;
2138 DECL_EXTERNAL (decl
) = 0;
2139 DECL_CONTEXT (decl
) = NULL_TREE
;
2140 DECL_INITIAL (decl
) = make_node (BLOCK
);
2141 if (cgraph_node::get (current_function_decl
)->offloadable
)
2142 cgraph_node::get_create (decl
)->offloadable
= 1;
2146 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2147 if (is_gimple_omp_offloaded (octx
->stmt
))
2149 cgraph_node::get_create (decl
)->offloadable
= 1;
2150 #ifdef ENABLE_OFFLOADING
2151 g
->have_offload
= true;
2157 if (cgraph_node::get_create (decl
)->offloadable
2158 && !lookup_attribute ("omp declare target",
2159 DECL_ATTRIBUTES (current_function_decl
)))
2160 DECL_ATTRIBUTES (decl
)
2161 = tree_cons (get_identifier ("omp target entrypoint"),
2162 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2164 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2165 RESULT_DECL
, NULL_TREE
, void_type_node
);
2166 DECL_ARTIFICIAL (t
) = 1;
2167 DECL_IGNORED_P (t
) = 1;
2168 DECL_CONTEXT (t
) = decl
;
2169 DECL_RESULT (decl
) = t
;
2171 /* _Cilk_for's child function requires two extra parameters called
2172 __low and __high that are set the by Cilk runtime when it calls this
2176 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2177 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2178 DECL_ARTIFICIAL (t
) = 1;
2179 DECL_NAMELESS (t
) = 1;
2180 DECL_ARG_TYPE (t
) = ptr_type_node
;
2181 DECL_CONTEXT (t
) = current_function_decl
;
2183 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2184 DECL_ARGUMENTS (decl
) = t
;
2186 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2187 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2188 DECL_ARTIFICIAL (t
) = 1;
2189 DECL_NAMELESS (t
) = 1;
2190 DECL_ARG_TYPE (t
) = ptr_type_node
;
2191 DECL_CONTEXT (t
) = current_function_decl
;
2193 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2194 DECL_ARGUMENTS (decl
) = t
;
2197 tree data_name
= get_identifier (".omp_data_i");
2198 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2200 DECL_ARTIFICIAL (t
) = 1;
2201 DECL_NAMELESS (t
) = 1;
2202 DECL_ARG_TYPE (t
) = ptr_type_node
;
2203 DECL_CONTEXT (t
) = current_function_decl
;
2206 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2207 DECL_ARGUMENTS (decl
) = t
;
2209 ctx
->receiver_decl
= t
;
2212 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2213 PARM_DECL
, get_identifier (".omp_data_o"),
2215 DECL_ARTIFICIAL (t
) = 1;
2216 DECL_NAMELESS (t
) = 1;
2217 DECL_ARG_TYPE (t
) = ptr_type_node
;
2218 DECL_CONTEXT (t
) = current_function_decl
;
2220 TREE_ADDRESSABLE (t
) = 1;
2221 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2222 DECL_ARGUMENTS (decl
) = t
;
2225 /* Allocate memory for the function structure. The call to
2226 allocate_struct_function clobbers CFUN, so we need to restore
2228 push_struct_function (decl
);
2229 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2233 /* Callback for walk_gimple_seq. Check if combined parallel
2234 contains gimple_omp_for_combined_into_p OMP_FOR. */
2237 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2238 bool *handled_ops_p
,
2239 struct walk_stmt_info
*wi
)
2241 gimple stmt
= gsi_stmt (*gsi_p
);
2243 *handled_ops_p
= true;
2244 switch (gimple_code (stmt
))
2248 case GIMPLE_OMP_FOR
:
2249 if (gimple_omp_for_combined_into_p (stmt
)
2250 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2253 return integer_zero_node
;
2262 /* Scan an OpenMP parallel directive. */
2265 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2269 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2271 /* Ignore parallel directives with empty bodies, unless there
2272 are copyin clauses. */
2274 && empty_body_p (gimple_omp_body (stmt
))
2275 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2276 OMP_CLAUSE_COPYIN
) == NULL
)
2278 gsi_replace (gsi
, gimple_build_nop (), false);
2282 if (gimple_omp_parallel_combined_p (stmt
))
2284 struct walk_stmt_info wi
;
2286 memset (&wi
, 0, sizeof (wi
));
2288 walk_gimple_seq (gimple_omp_body (stmt
),
2289 find_combined_for
, NULL
, &wi
);
2292 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
) wi
.info
);
2293 struct omp_for_data fd
;
2294 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2295 /* We need two temporaries with fd.loop.v type (istart/iend)
2296 and then (fd.collapse - 1) temporaries with the same
2297 type for count2 ... countN-1 vars if not constant. */
2298 size_t count
= 2, i
;
2299 tree type
= fd
.iter_type
;
2301 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2302 count
+= fd
.collapse
- 1;
2303 for (i
= 0; i
< count
; i
++)
2305 tree temp
= create_tmp_var (type
);
2306 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2307 OMP_CLAUSE__LOOPTEMP_
);
2308 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2309 OMP_CLAUSE_DECL (c
) = temp
;
2310 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2311 gimple_omp_parallel_set_clauses (stmt
, c
);
2316 ctx
= new_omp_context (stmt
, outer_ctx
);
2317 taskreg_contexts
.safe_push (ctx
);
2318 if (taskreg_nesting_level
> 1)
2319 ctx
->is_nested
= true;
2320 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2321 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2322 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2323 name
= create_tmp_var_name (".omp_data_s");
2324 name
= build_decl (gimple_location (stmt
),
2325 TYPE_DECL
, name
, ctx
->record_type
);
2326 DECL_ARTIFICIAL (name
) = 1;
2327 DECL_NAMELESS (name
) = 1;
2328 TYPE_NAME (ctx
->record_type
) = name
;
2329 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2330 create_omp_child_function (ctx
, false);
2331 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2333 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2334 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2336 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2337 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2340 /* Scan an OpenMP task directive. */
2343 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2347 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2349 /* Ignore task directives with empty bodies. */
2351 && empty_body_p (gimple_omp_body (stmt
)))
2353 gsi_replace (gsi
, gimple_build_nop (), false);
2357 ctx
= new_omp_context (stmt
, outer_ctx
);
2358 taskreg_contexts
.safe_push (ctx
);
2359 if (taskreg_nesting_level
> 1)
2360 ctx
->is_nested
= true;
2361 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2362 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2363 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2364 name
= create_tmp_var_name (".omp_data_s");
2365 name
= build_decl (gimple_location (stmt
),
2366 TYPE_DECL
, name
, ctx
->record_type
);
2367 DECL_ARTIFICIAL (name
) = 1;
2368 DECL_NAMELESS (name
) = 1;
2369 TYPE_NAME (ctx
->record_type
) = name
;
2370 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2371 create_omp_child_function (ctx
, false);
2372 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2374 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2376 if (ctx
->srecord_type
)
2378 name
= create_tmp_var_name (".omp_data_a");
2379 name
= build_decl (gimple_location (stmt
),
2380 TYPE_DECL
, name
, ctx
->srecord_type
);
2381 DECL_ARTIFICIAL (name
) = 1;
2382 DECL_NAMELESS (name
) = 1;
2383 TYPE_NAME (ctx
->srecord_type
) = name
;
2384 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2385 create_omp_child_function (ctx
, true);
2388 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2390 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2392 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2393 t
= build_int_cst (long_integer_type_node
, 0);
2394 gimple_omp_task_set_arg_size (stmt
, t
);
2395 t
= build_int_cst (long_integer_type_node
, 1);
2396 gimple_omp_task_set_arg_align (stmt
, t
);
2401 /* If any decls have been made addressable during scan_omp,
2402 adjust their fields if needed, and layout record types
2403 of parallel/task constructs. */
2406 finish_taskreg_scan (omp_context
*ctx
)
2408 if (ctx
->record_type
== NULL_TREE
)
2411 /* If any task_shared_vars were needed, verify all
2412 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2413 statements if use_pointer_for_field hasn't changed
2414 because of that. If it did, update field types now. */
2415 if (task_shared_vars
)
2419 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2420 c
; c
= OMP_CLAUSE_CHAIN (c
))
2421 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2423 tree decl
= OMP_CLAUSE_DECL (c
);
2425 /* Global variables don't need to be copied,
2426 the receiver side will use them directly. */
2427 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2429 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2430 || !use_pointer_for_field (decl
, ctx
))
2432 tree field
= lookup_field (decl
, ctx
);
2433 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2434 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2436 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2437 TREE_THIS_VOLATILE (field
) = 0;
2438 DECL_USER_ALIGN (field
) = 0;
2439 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2440 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2441 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2442 if (ctx
->srecord_type
)
2444 tree sfield
= lookup_sfield (decl
, ctx
);
2445 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2446 TREE_THIS_VOLATILE (sfield
) = 0;
2447 DECL_USER_ALIGN (sfield
) = 0;
2448 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2449 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2450 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2455 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2457 layout_type (ctx
->record_type
);
2458 fixup_child_record_type (ctx
);
2462 location_t loc
= gimple_location (ctx
->stmt
);
2463 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2464 /* Move VLA fields to the end. */
2465 p
= &TYPE_FIELDS (ctx
->record_type
);
2467 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2468 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2471 *p
= TREE_CHAIN (*p
);
2472 TREE_CHAIN (*q
) = NULL_TREE
;
2473 q
= &TREE_CHAIN (*q
);
2476 p
= &DECL_CHAIN (*p
);
2478 layout_type (ctx
->record_type
);
2479 fixup_child_record_type (ctx
);
2480 if (ctx
->srecord_type
)
2481 layout_type (ctx
->srecord_type
);
2482 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2483 TYPE_SIZE_UNIT (ctx
->record_type
));
2484 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2485 t
= build_int_cst (long_integer_type_node
,
2486 TYPE_ALIGN_UNIT (ctx
->record_type
));
2487 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2492 static omp_context
*
2493 enclosing_target_ctx (omp_context
*ctx
)
2496 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2498 gcc_assert (ctx
!= NULL
);
2503 oacc_loop_or_target_p (gimple stmt
)
2505 enum gimple_code outer_type
= gimple_code (stmt
);
2506 return ((outer_type
== GIMPLE_OMP_TARGET
2507 && ((gimple_omp_target_kind (stmt
)
2508 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2509 || (gimple_omp_target_kind (stmt
)
2510 == GF_OMP_TARGET_KIND_OACC_KERNELS
)))
2511 || (outer_type
== GIMPLE_OMP_FOR
2512 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
));
2515 /* Scan a GIMPLE_OMP_FOR. */
2518 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2520 enum gimple_code outer_type
= GIMPLE_ERROR_MARK
;
2523 tree clauses
= gimple_omp_for_clauses (stmt
);
2526 outer_type
= gimple_code (outer_ctx
->stmt
);
2528 ctx
= new_omp_context (stmt
, outer_ctx
);
2530 if (is_gimple_omp_oacc (stmt
))
2532 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2533 ctx
->gwv_this
= outer_ctx
->gwv_this
;
2534 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2537 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_GANG
)
2539 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WORKER
)
2541 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR
)
2545 ctx
->gwv_this
|= val
;
2548 /* Skip; not nested inside a region. */
2551 if (!oacc_loop_or_target_p (outer_ctx
->stmt
))
2553 /* Skip; not nested inside an OpenACC region. */
2556 if (outer_type
== GIMPLE_OMP_FOR
)
2557 outer_ctx
->gwv_below
|= val
;
2558 if (OMP_CLAUSE_OPERAND (c
, 0) != NULL_TREE
)
2560 omp_context
*enclosing
= enclosing_target_ctx (outer_ctx
);
2561 if (gimple_omp_target_kind (enclosing
->stmt
)
2562 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2563 error_at (gimple_location (stmt
),
2564 "no arguments allowed to gang, worker and vector clauses inside parallel");
2569 scan_sharing_clauses (clauses
, ctx
);
2571 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2572 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2574 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2575 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2576 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2577 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2579 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2581 if (is_gimple_omp_oacc (stmt
))
2583 if (ctx
->gwv_this
& ctx
->gwv_below
)
2584 error_at (gimple_location (stmt
),
2585 "gang, worker and vector may occur only once in a loop nest");
2586 else if (ctx
->gwv_below
!= 0
2587 && ctx
->gwv_this
> ctx
->gwv_below
)
2588 error_at (gimple_location (stmt
),
2589 "gang, worker and vector must occur in this order in a loop nest");
2590 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2591 outer_ctx
->gwv_below
|= ctx
->gwv_below
;
2595 /* Scan an OpenMP sections directive. */
2598 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
2602 ctx
= new_omp_context (stmt
, outer_ctx
);
2603 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2604 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2607 /* Scan an OpenMP single directive. */
2610 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
2615 ctx
= new_omp_context (stmt
, outer_ctx
);
2616 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2617 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2618 name
= create_tmp_var_name (".omp_copy_s");
2619 name
= build_decl (gimple_location (stmt
),
2620 TYPE_DECL
, name
, ctx
->record_type
);
2621 TYPE_NAME (ctx
->record_type
) = name
;
2623 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2624 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2626 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2627 ctx
->record_type
= NULL
;
2629 layout_type (ctx
->record_type
);
2632 /* Scan a GIMPLE_OMP_TARGET. */
2635 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
2639 bool offloaded
= is_gimple_omp_offloaded (stmt
);
2640 tree clauses
= gimple_omp_target_clauses (stmt
);
2642 ctx
= new_omp_context (stmt
, outer_ctx
);
2643 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2644 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2645 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2646 name
= create_tmp_var_name (".omp_data_t");
2647 name
= build_decl (gimple_location (stmt
),
2648 TYPE_DECL
, name
, ctx
->record_type
);
2649 DECL_ARTIFICIAL (name
) = 1;
2650 DECL_NAMELESS (name
) = 1;
2651 TYPE_NAME (ctx
->record_type
) = name
;
2652 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2655 if (is_gimple_omp_oacc (stmt
))
2656 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
2659 create_omp_child_function (ctx
, false);
2660 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2663 if (is_gimple_omp_oacc (stmt
))
2665 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2667 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_GANGS
)
2668 ctx
->gwv_this
|= MASK_GANG
;
2669 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_WORKERS
)
2670 ctx
->gwv_this
|= MASK_WORKER
;
2671 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR_LENGTH
)
2672 ctx
->gwv_this
|= MASK_VECTOR
;
2676 scan_sharing_clauses (clauses
, ctx
);
2677 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2679 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2680 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2683 TYPE_FIELDS (ctx
->record_type
)
2684 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2685 #ifdef ENABLE_CHECKING
2687 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2688 for (field
= TYPE_FIELDS (ctx
->record_type
);
2690 field
= DECL_CHAIN (field
))
2691 gcc_assert (DECL_ALIGN (field
) == align
);
2693 layout_type (ctx
->record_type
);
2695 fixup_child_record_type (ctx
);
2699 /* Scan an OpenMP teams directive. */
2702 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
2704 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2705 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2706 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2709 /* Check nesting restrictions. */
2711 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2713 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2714 inside an OpenACC CTX. */
2715 if (!(is_gimple_omp (stmt
)
2716 && is_gimple_omp_oacc (stmt
)))
2718 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
2719 if (is_gimple_omp (ctx_
->stmt
)
2720 && is_gimple_omp_oacc (ctx_
->stmt
))
2722 error_at (gimple_location (stmt
),
2723 "non-OpenACC construct inside of OpenACC region");
2730 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2731 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2733 error_at (gimple_location (stmt
),
2734 "OpenMP constructs may not be nested inside simd region");
2737 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2739 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2740 || (gimple_omp_for_kind (stmt
)
2741 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2742 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2744 error_at (gimple_location (stmt
),
2745 "only distribute or parallel constructs are allowed to "
2746 "be closely nested inside teams construct");
2751 switch (gimple_code (stmt
))
2753 case GIMPLE_OMP_FOR
:
2754 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2756 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2758 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2760 error_at (gimple_location (stmt
),
2761 "distribute construct must be closely nested inside "
2769 if (is_gimple_call (stmt
)
2770 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2771 == BUILT_IN_GOMP_CANCEL
2772 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2773 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2775 const char *bad
= NULL
;
2776 const char *kind
= NULL
;
2779 error_at (gimple_location (stmt
), "orphaned %qs construct",
2780 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2781 == BUILT_IN_GOMP_CANCEL
2782 ? "#pragma omp cancel"
2783 : "#pragma omp cancellation point");
2786 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2787 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2791 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2792 bad
= "#pragma omp parallel";
2793 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2794 == BUILT_IN_GOMP_CANCEL
2795 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2796 ctx
->cancellable
= true;
2800 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2801 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2802 bad
= "#pragma omp for";
2803 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2804 == BUILT_IN_GOMP_CANCEL
2805 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2807 ctx
->cancellable
= true;
2808 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2810 warning_at (gimple_location (stmt
), 0,
2811 "%<#pragma omp cancel for%> inside "
2812 "%<nowait%> for construct");
2813 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2814 OMP_CLAUSE_ORDERED
))
2815 warning_at (gimple_location (stmt
), 0,
2816 "%<#pragma omp cancel for%> inside "
2817 "%<ordered%> for construct");
2822 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2823 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2824 bad
= "#pragma omp sections";
2825 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2826 == BUILT_IN_GOMP_CANCEL
2827 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2829 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2831 ctx
->cancellable
= true;
2832 if (find_omp_clause (gimple_omp_sections_clauses
2835 warning_at (gimple_location (stmt
), 0,
2836 "%<#pragma omp cancel sections%> inside "
2837 "%<nowait%> sections construct");
2841 gcc_assert (ctx
->outer
2842 && gimple_code (ctx
->outer
->stmt
)
2843 == GIMPLE_OMP_SECTIONS
);
2844 ctx
->outer
->cancellable
= true;
2845 if (find_omp_clause (gimple_omp_sections_clauses
2848 warning_at (gimple_location (stmt
), 0,
2849 "%<#pragma omp cancel sections%> inside "
2850 "%<nowait%> sections construct");
2856 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2857 bad
= "#pragma omp task";
2859 ctx
->cancellable
= true;
2863 error_at (gimple_location (stmt
), "invalid arguments");
2868 error_at (gimple_location (stmt
),
2869 "%<%s %s%> construct not closely nested inside of %qs",
2870 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2871 == BUILT_IN_GOMP_CANCEL
2872 ? "#pragma omp cancel"
2873 : "#pragma omp cancellation point", kind
, bad
);
2878 case GIMPLE_OMP_SECTIONS
:
2879 case GIMPLE_OMP_SINGLE
:
2880 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2881 switch (gimple_code (ctx
->stmt
))
2883 case GIMPLE_OMP_FOR
:
2884 case GIMPLE_OMP_SECTIONS
:
2885 case GIMPLE_OMP_SINGLE
:
2886 case GIMPLE_OMP_ORDERED
:
2887 case GIMPLE_OMP_MASTER
:
2888 case GIMPLE_OMP_TASK
:
2889 case GIMPLE_OMP_CRITICAL
:
2890 if (is_gimple_call (stmt
))
2892 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2893 != BUILT_IN_GOMP_BARRIER
)
2895 error_at (gimple_location (stmt
),
2896 "barrier region may not be closely nested inside "
2897 "of work-sharing, critical, ordered, master or "
2898 "explicit task region");
2901 error_at (gimple_location (stmt
),
2902 "work-sharing region may not be closely nested inside "
2903 "of work-sharing, critical, ordered, master or explicit "
2906 case GIMPLE_OMP_PARALLEL
:
2912 case GIMPLE_OMP_MASTER
:
2913 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2914 switch (gimple_code (ctx
->stmt
))
2916 case GIMPLE_OMP_FOR
:
2917 case GIMPLE_OMP_SECTIONS
:
2918 case GIMPLE_OMP_SINGLE
:
2919 case GIMPLE_OMP_TASK
:
2920 error_at (gimple_location (stmt
),
2921 "master region may not be closely nested inside "
2922 "of work-sharing or explicit task region");
2924 case GIMPLE_OMP_PARALLEL
:
2930 case GIMPLE_OMP_ORDERED
:
2931 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2932 switch (gimple_code (ctx
->stmt
))
2934 case GIMPLE_OMP_CRITICAL
:
2935 case GIMPLE_OMP_TASK
:
2936 error_at (gimple_location (stmt
),
2937 "ordered region may not be closely nested inside "
2938 "of critical or explicit task region");
2940 case GIMPLE_OMP_FOR
:
2941 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2942 OMP_CLAUSE_ORDERED
) == NULL
)
2944 error_at (gimple_location (stmt
),
2945 "ordered region must be closely nested inside "
2946 "a loop region with an ordered clause");
2950 case GIMPLE_OMP_PARALLEL
:
2951 error_at (gimple_location (stmt
),
2952 "ordered region must be closely nested inside "
2953 "a loop region with an ordered clause");
2959 case GIMPLE_OMP_CRITICAL
:
2962 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
2963 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2964 if (gomp_critical
*other_crit
2965 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
2966 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
2968 error_at (gimple_location (stmt
),
2969 "critical region may not be nested inside a critical "
2970 "region with the same name");
2975 case GIMPLE_OMP_TEAMS
:
2977 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2978 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2980 error_at (gimple_location (stmt
),
2981 "teams construct not closely nested inside of target "
2986 case GIMPLE_OMP_TARGET
:
2987 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2989 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2991 if (is_gimple_omp (stmt
)
2992 && is_gimple_omp_oacc (stmt
)
2993 && is_gimple_omp (ctx
->stmt
))
2995 error_at (gimple_location (stmt
),
2996 "OpenACC construct inside of non-OpenACC region");
3002 const char *stmt_name
, *ctx_stmt_name
;
3003 switch (gimple_omp_target_kind (stmt
))
3005 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3006 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3007 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3008 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3009 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3010 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3011 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3012 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
: stmt_name
= "enter/exit data"; break;
3013 default: gcc_unreachable ();
3015 switch (gimple_omp_target_kind (ctx
->stmt
))
3017 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3018 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3019 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: ctx_stmt_name
= "parallel"; break;
3020 case GF_OMP_TARGET_KIND_OACC_KERNELS
: ctx_stmt_name
= "kernels"; break;
3021 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3022 default: gcc_unreachable ();
3025 /* OpenACC/OpenMP mismatch? */
3026 if (is_gimple_omp_oacc (stmt
)
3027 != is_gimple_omp_oacc (ctx
->stmt
))
3029 error_at (gimple_location (stmt
),
3030 "%s %s construct inside of %s %s region",
3031 (is_gimple_omp_oacc (stmt
)
3032 ? "OpenACC" : "OpenMP"), stmt_name
,
3033 (is_gimple_omp_oacc (ctx
->stmt
)
3034 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3037 if (is_gimple_omp_offloaded (ctx
->stmt
))
3039 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3040 if (is_gimple_omp_oacc (ctx
->stmt
))
3042 error_at (gimple_location (stmt
),
3043 "%s construct inside of %s region",
3044 stmt_name
, ctx_stmt_name
);
3049 gcc_checking_assert (!is_gimple_omp_oacc (stmt
));
3050 warning_at (gimple_location (stmt
), 0,
3051 "%s construct inside of %s region",
3052 stmt_name
, ctx_stmt_name
);
3064 /* Helper function scan_omp.
3066 Callback for walk_tree or operators in walk_gimple_stmt used to
3067 scan for OMP directives in TP. */
3070 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3072 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3073 omp_context
*ctx
= (omp_context
*) wi
->info
;
3076 switch (TREE_CODE (t
))
3083 *tp
= remap_decl (t
, &ctx
->cb
);
3087 if (ctx
&& TYPE_P (t
))
3088 *tp
= remap_type (t
, &ctx
->cb
);
3089 else if (!DECL_P (t
))
3094 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3095 if (tem
!= TREE_TYPE (t
))
3097 if (TREE_CODE (t
) == INTEGER_CST
)
3098 *tp
= wide_int_to_tree (tem
, t
);
3100 TREE_TYPE (t
) = tem
;
3110 /* Return true if FNDECL is a setjmp or a longjmp. */
3113 setjmp_or_longjmp_p (const_tree fndecl
)
3115 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3116 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3117 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3120 tree declname
= DECL_NAME (fndecl
);
3123 const char *name
= IDENTIFIER_POINTER (declname
);
3124 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3128 /* Helper function for scan_omp.
3130 Callback for walk_gimple_stmt used to scan for OMP directives in
3131 the current statement in GSI. */
3134 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3135 struct walk_stmt_info
*wi
)
3137 gimple stmt
= gsi_stmt (*gsi
);
3138 omp_context
*ctx
= (omp_context
*) wi
->info
;
3140 if (gimple_has_location (stmt
))
3141 input_location
= gimple_location (stmt
);
3143 /* Check the nesting restrictions. */
3144 bool remove
= false;
3145 if (is_gimple_omp (stmt
))
3146 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3147 else if (is_gimple_call (stmt
))
3149 tree fndecl
= gimple_call_fndecl (stmt
);
3152 if (setjmp_or_longjmp_p (fndecl
)
3154 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3155 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3158 error_at (gimple_location (stmt
),
3159 "setjmp/longjmp inside simd construct");
3161 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3162 switch (DECL_FUNCTION_CODE (fndecl
))
3164 case BUILT_IN_GOMP_BARRIER
:
3165 case BUILT_IN_GOMP_CANCEL
:
3166 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3167 case BUILT_IN_GOMP_TASKYIELD
:
3168 case BUILT_IN_GOMP_TASKWAIT
:
3169 case BUILT_IN_GOMP_TASKGROUP_START
:
3170 case BUILT_IN_GOMP_TASKGROUP_END
:
3171 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3180 stmt
= gimple_build_nop ();
3181 gsi_replace (gsi
, stmt
, false);
3184 *handled_ops_p
= true;
3186 switch (gimple_code (stmt
))
3188 case GIMPLE_OMP_PARALLEL
:
3189 taskreg_nesting_level
++;
3190 scan_omp_parallel (gsi
, ctx
);
3191 taskreg_nesting_level
--;
3194 case GIMPLE_OMP_TASK
:
3195 taskreg_nesting_level
++;
3196 scan_omp_task (gsi
, ctx
);
3197 taskreg_nesting_level
--;
3200 case GIMPLE_OMP_FOR
:
3201 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3204 case GIMPLE_OMP_SECTIONS
:
3205 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3208 case GIMPLE_OMP_SINGLE
:
3209 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3212 case GIMPLE_OMP_SECTION
:
3213 case GIMPLE_OMP_MASTER
:
3214 case GIMPLE_OMP_TASKGROUP
:
3215 case GIMPLE_OMP_ORDERED
:
3216 case GIMPLE_OMP_CRITICAL
:
3217 ctx
= new_omp_context (stmt
, ctx
);
3218 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3221 case GIMPLE_OMP_TARGET
:
3222 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3225 case GIMPLE_OMP_TEAMS
:
3226 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3233 *handled_ops_p
= false;
3235 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3237 var
= DECL_CHAIN (var
))
3238 insert_decl_map (&ctx
->cb
, var
, var
);
3242 *handled_ops_p
= false;
3250 /* Scan all the statements starting at the current statement. CTX
3251 contains context information about the OMP directives and
3252 clauses found during the scan. */
3255 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3257 location_t saved_location
;
3258 struct walk_stmt_info wi
;
3260 memset (&wi
, 0, sizeof (wi
));
3262 wi
.want_locations
= true;
3264 saved_location
= input_location
;
3265 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3266 input_location
= saved_location
;
3269 /* Re-gimplification and code generation routines. */
3271 /* Build a call to GOMP_barrier. */
3274 build_omp_barrier (tree lhs
)
3276 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3277 : BUILT_IN_GOMP_BARRIER
);
3278 gcall
*g
= gimple_build_call (fndecl
, 0);
3280 gimple_call_set_lhs (g
, lhs
);
3284 /* If a context was created for STMT when it was scanned, return it. */
3286 static omp_context
*
3287 maybe_lookup_ctx (gimple stmt
)
3290 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3291 return n
? (omp_context
*) n
->value
: NULL
;
3295 /* Find the mapping for DECL in CTX or the immediately enclosing
3296 context that has a mapping for DECL.
3298 If CTX is a nested parallel directive, we may have to use the decl
3299 mappings created in CTX's parent context. Suppose that we have the
3300 following parallel nesting (variable UIDs showed for clarity):
3303 #omp parallel shared(iD.1562) -> outer parallel
3304 iD.1562 = iD.1562 + 1;
3306 #omp parallel shared (iD.1562) -> inner parallel
3307 iD.1562 = iD.1562 - 1;
3309 Each parallel structure will create a distinct .omp_data_s structure
3310 for copying iD.1562 in/out of the directive:
3312 outer parallel .omp_data_s.1.i -> iD.1562
3313 inner parallel .omp_data_s.2.i -> iD.1562
3315 A shared variable mapping will produce a copy-out operation before
3316 the parallel directive and a copy-in operation after it. So, in
3317 this case we would have:
3320 .omp_data_o.1.i = iD.1562;
3321 #omp parallel shared(iD.1562) -> outer parallel
3322 .omp_data_i.1 = &.omp_data_o.1
3323 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3325 .omp_data_o.2.i = iD.1562; -> **
3326 #omp parallel shared(iD.1562) -> inner parallel
3327 .omp_data_i.2 = &.omp_data_o.2
3328 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3331 ** This is a problem. The symbol iD.1562 cannot be referenced
3332 inside the body of the outer parallel region. But since we are
3333 emitting this copy operation while expanding the inner parallel
3334 directive, we need to access the CTX structure of the outer
3335 parallel directive to get the correct mapping:
3337 .omp_data_o.2.i = .omp_data_i.1->i
3339 Since there may be other workshare or parallel directives enclosing
3340 the parallel directive, it may be necessary to walk up the context
3341 parent chain. This is not a problem in general because nested
3342 parallelism happens only rarely. */
3345 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3350 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3351 t
= maybe_lookup_decl (decl
, up
);
3353 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3355 return t
? t
: decl
;
3359 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3360 in outer contexts. */
3363 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3368 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3369 t
= maybe_lookup_decl (decl
, up
);
3371 return t
? t
: decl
;
3375 /* Construct the initialization value for reduction CLAUSE. */
3378 omp_reduction_init (tree clause
, tree type
)
3380 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3381 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3388 case TRUTH_ORIF_EXPR
:
3389 case TRUTH_XOR_EXPR
:
3391 return build_zero_cst (type
);
3394 case TRUTH_AND_EXPR
:
3395 case TRUTH_ANDIF_EXPR
:
3397 return fold_convert_loc (loc
, type
, integer_one_node
);
3400 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3403 if (SCALAR_FLOAT_TYPE_P (type
))
3405 REAL_VALUE_TYPE max
, min
;
3406 if (HONOR_INFINITIES (type
))
3409 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3412 real_maxval (&min
, 1, TYPE_MODE (type
));
3413 return build_real (type
, min
);
3417 gcc_assert (INTEGRAL_TYPE_P (type
));
3418 return TYPE_MIN_VALUE (type
);
3422 if (SCALAR_FLOAT_TYPE_P (type
))
3424 REAL_VALUE_TYPE max
;
3425 if (HONOR_INFINITIES (type
))
3428 real_maxval (&max
, 0, TYPE_MODE (type
));
3429 return build_real (type
, max
);
3433 gcc_assert (INTEGRAL_TYPE_P (type
));
3434 return TYPE_MAX_VALUE (type
);
3442 /* Return alignment to be assumed for var in CLAUSE, which should be
3443 OMP_CLAUSE_ALIGNED. */
3446 omp_clause_aligned_alignment (tree clause
)
3448 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3449 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3451 /* Otherwise return implementation defined alignment. */
3452 unsigned int al
= 1;
3453 machine_mode mode
, vmode
;
3454 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3456 vs
= 1 << floor_log2 (vs
);
3457 static enum mode_class classes
[]
3458 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3459 for (int i
= 0; i
< 4; i
+= 2)
3460 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3462 mode
= GET_MODE_WIDER_MODE (mode
))
3464 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3465 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3468 && GET_MODE_SIZE (vmode
) < vs
3469 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3470 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3472 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3473 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3475 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3476 / GET_MODE_SIZE (mode
));
3477 if (TYPE_MODE (type
) != vmode
)
3479 if (TYPE_ALIGN_UNIT (type
) > al
)
3480 al
= TYPE_ALIGN_UNIT (type
);
3482 return build_int_cst (integer_type_node
, al
);
3485 /* Return maximum possible vectorization factor for the target. */
3492 || !flag_tree_loop_optimize
3493 || (!flag_tree_loop_vectorize
3494 && (global_options_set
.x_flag_tree_loop_vectorize
3495 || global_options_set
.x_flag_tree_vectorize
)))
3498 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3501 vs
= 1 << floor_log2 (vs
);
3504 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3505 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3506 return GET_MODE_NUNITS (vqimode
);
3510 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3514 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3515 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3519 max_vf
= omp_max_vf ();
3522 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3523 OMP_CLAUSE_SAFELEN
);
3524 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3526 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3528 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3532 idx
= create_tmp_var (unsigned_type_node
);
3533 lane
= create_tmp_var (unsigned_type_node
);
3539 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3540 tree avar
= create_tmp_var_raw (atype
);
3541 if (TREE_ADDRESSABLE (new_var
))
3542 TREE_ADDRESSABLE (avar
) = 1;
3543 DECL_ATTRIBUTES (avar
)
3544 = tree_cons (get_identifier ("omp simd array"), NULL
,
3545 DECL_ATTRIBUTES (avar
));
3546 gimple_add_tmp_var (avar
);
3547 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3548 NULL_TREE
, NULL_TREE
);
3549 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3550 NULL_TREE
, NULL_TREE
);
3551 if (DECL_P (new_var
))
3553 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3554 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3559 /* Helper function of lower_rec_input_clauses. For a reference
3560 in simd reduction, add an underlying variable it will reference. */
3563 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3565 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3566 if (TREE_CONSTANT (z
))
3568 const char *name
= NULL
;
3569 if (DECL_NAME (new_vard
))
3570 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3572 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3573 gimple_add_tmp_var (z
);
3574 TREE_ADDRESSABLE (z
) = 1;
3575 z
= build_fold_addr_expr_loc (loc
, z
);
3576 gimplify_assign (new_vard
, z
, ilist
);
3580 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3581 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3582 private variables. Initialization statements go in ILIST, while calls
3583 to destructors go in DLIST. */
3586 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3587 omp_context
*ctx
, struct omp_for_data
*fd
)
3589 tree c
, dtor
, copyin_seq
, x
, ptr
;
3590 bool copyin_by_ref
= false;
3591 bool lastprivate_firstprivate
= false;
3592 bool reduction_omp_orig_ref
= false;
3594 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3595 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3597 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3598 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3599 gimple_seq llist
[2] = { NULL
, NULL
};
3603 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3604 with data sharing clauses referencing variable sized vars. That
3605 is unnecessarily hard to support and very unlikely to result in
3606 vectorized code anyway. */
3608 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3609 switch (OMP_CLAUSE_CODE (c
))
3611 case OMP_CLAUSE_LINEAR
:
3612 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3615 case OMP_CLAUSE_REDUCTION
:
3616 case OMP_CLAUSE_PRIVATE
:
3617 case OMP_CLAUSE_FIRSTPRIVATE
:
3618 case OMP_CLAUSE_LASTPRIVATE
:
3619 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3626 /* Do all the fixed sized types in the first pass, and the variable sized
3627 types in the second pass. This makes sure that the scalar arguments to
3628 the variable sized types are processed before we use them in the
3629 variable sized operations. */
3630 for (pass
= 0; pass
< 2; ++pass
)
3632 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3634 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3637 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3641 case OMP_CLAUSE_PRIVATE
:
3642 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3645 case OMP_CLAUSE_SHARED
:
3646 /* Ignore shared directives in teams construct. */
3647 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3649 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3651 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3654 case OMP_CLAUSE_FIRSTPRIVATE
:
3655 case OMP_CLAUSE_COPYIN
:
3656 case OMP_CLAUSE_LINEAR
:
3658 case OMP_CLAUSE_REDUCTION
:
3659 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3660 reduction_omp_orig_ref
= true;
3662 case OMP_CLAUSE__LOOPTEMP_
:
3663 /* Handle _looptemp_ clauses only on parallel. */
3667 case OMP_CLAUSE_LASTPRIVATE
:
3668 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3670 lastprivate_firstprivate
= true;
3674 /* Even without corresponding firstprivate, if
3675 decl is Fortran allocatable, it needs outer var
3678 && lang_hooks
.decls
.omp_private_outer_ref
3679 (OMP_CLAUSE_DECL (c
)))
3680 lastprivate_firstprivate
= true;
3682 case OMP_CLAUSE_ALIGNED
:
3685 var
= OMP_CLAUSE_DECL (c
);
3686 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3687 && !is_global_var (var
))
3689 new_var
= maybe_lookup_decl (var
, ctx
);
3690 if (new_var
== NULL_TREE
)
3691 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3692 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3693 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3694 omp_clause_aligned_alignment (c
));
3695 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3696 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3697 gimplify_and_add (x
, ilist
);
3699 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3700 && is_global_var (var
))
3702 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3703 new_var
= lookup_decl (var
, ctx
);
3704 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3705 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3706 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3707 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3708 omp_clause_aligned_alignment (c
));
3709 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3710 x
= create_tmp_var (ptype
);
3711 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3712 gimplify_and_add (t
, ilist
);
3713 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3714 SET_DECL_VALUE_EXPR (new_var
, t
);
3715 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3722 new_var
= var
= OMP_CLAUSE_DECL (c
);
3723 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3724 new_var
= lookup_decl (var
, ctx
);
3726 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3731 else if (is_variable_sized (var
))
3733 /* For variable sized types, we need to allocate the
3734 actual storage here. Call alloca and store the
3735 result in the pointer decl that we created elsewhere. */
3739 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3744 ptr
= DECL_VALUE_EXPR (new_var
);
3745 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3746 ptr
= TREE_OPERAND (ptr
, 0);
3747 gcc_assert (DECL_P (ptr
));
3748 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3750 /* void *tmp = __builtin_alloca */
3751 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3752 stmt
= gimple_build_call (atmp
, 1, x
);
3753 tmp
= create_tmp_var_raw (ptr_type_node
);
3754 gimple_add_tmp_var (tmp
);
3755 gimple_call_set_lhs (stmt
, tmp
);
3757 gimple_seq_add_stmt (ilist
, stmt
);
3759 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3760 gimplify_assign (ptr
, x
, ilist
);
3763 else if (is_reference (var
))
3765 /* For references that are being privatized for Fortran,
3766 allocate new backing storage for the new pointer
3767 variable. This allows us to avoid changing all the
3768 code that expects a pointer to something that expects
3769 a direct variable. */
3773 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3774 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3776 x
= build_receiver_ref (var
, false, ctx
);
3777 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3779 else if (TREE_CONSTANT (x
))
3781 /* For reduction in SIMD loop, defer adding the
3782 initialization of the reference, because if we decide
3783 to use SIMD array for it, the initilization could cause
3785 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3789 const char *name
= NULL
;
3790 if (DECL_NAME (var
))
3791 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3793 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3795 gimple_add_tmp_var (x
);
3796 TREE_ADDRESSABLE (x
) = 1;
3797 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3802 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3803 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3808 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3809 gimplify_assign (new_var
, x
, ilist
);
3812 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3814 else if (c_kind
== OMP_CLAUSE_REDUCTION
3815 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3823 switch (OMP_CLAUSE_CODE (c
))
3825 case OMP_CLAUSE_SHARED
:
3826 /* Ignore shared directives in teams construct. */
3827 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3829 /* Shared global vars are just accessed directly. */
3830 if (is_global_var (new_var
))
3832 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3833 needs to be delayed until after fixup_child_record_type so
3834 that we get the correct type during the dereference. */
3835 by_ref
= use_pointer_for_field (var
, ctx
);
3836 x
= build_receiver_ref (var
, by_ref
, ctx
);
3837 SET_DECL_VALUE_EXPR (new_var
, x
);
3838 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3840 /* ??? If VAR is not passed by reference, and the variable
3841 hasn't been initialized yet, then we'll get a warning for
3842 the store into the omp_data_s structure. Ideally, we'd be
3843 able to notice this and not store anything at all, but
3844 we're generating code too early. Suppress the warning. */
3846 TREE_NO_WARNING (var
) = 1;
3849 case OMP_CLAUSE_LASTPRIVATE
:
3850 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3854 case OMP_CLAUSE_PRIVATE
:
3855 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3856 x
= build_outer_var_ref (var
, ctx
);
3857 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3859 if (is_task_ctx (ctx
))
3860 x
= build_receiver_ref (var
, false, ctx
);
3862 x
= build_outer_var_ref (var
, ctx
);
3868 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3871 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3872 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3873 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3874 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3875 idx
, lane
, ivar
, lvar
))
3878 x
= lang_hooks
.decls
.omp_clause_default_ctor
3879 (c
, unshare_expr (ivar
), x
);
3881 gimplify_and_add (x
, &llist
[0]);
3884 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3887 gimple_seq tseq
= NULL
;
3890 gimplify_stmt (&dtor
, &tseq
);
3891 gimple_seq_add_seq (&llist
[1], tseq
);
3898 gimplify_and_add (nx
, ilist
);
3902 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3905 gimple_seq tseq
= NULL
;
3908 gimplify_stmt (&dtor
, &tseq
);
3909 gimple_seq_add_seq (dlist
, tseq
);
3913 case OMP_CLAUSE_LINEAR
:
3914 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3915 goto do_firstprivate
;
3916 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3919 x
= build_outer_var_ref (var
, ctx
);
3922 case OMP_CLAUSE_FIRSTPRIVATE
:
3923 if (is_task_ctx (ctx
))
3925 if (is_reference (var
) || is_variable_sized (var
))
3927 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3929 || use_pointer_for_field (var
, NULL
))
3931 x
= build_receiver_ref (var
, false, ctx
);
3932 SET_DECL_VALUE_EXPR (new_var
, x
);
3933 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3938 x
= build_outer_var_ref (var
, ctx
);
3941 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3942 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3944 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3945 tree stept
= TREE_TYPE (t
);
3946 tree ct
= find_omp_clause (clauses
,
3947 OMP_CLAUSE__LOOPTEMP_
);
3949 tree l
= OMP_CLAUSE_DECL (ct
);
3950 tree n1
= fd
->loop
.n1
;
3951 tree step
= fd
->loop
.step
;
3952 tree itype
= TREE_TYPE (l
);
3953 if (POINTER_TYPE_P (itype
))
3954 itype
= signed_type_for (itype
);
3955 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3956 if (TYPE_UNSIGNED (itype
)
3957 && fd
->loop
.cond_code
== GT_EXPR
)
3958 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3959 fold_build1 (NEGATE_EXPR
, itype
, l
),
3960 fold_build1 (NEGATE_EXPR
,
3963 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3964 t
= fold_build2 (MULT_EXPR
, stept
,
3965 fold_convert (stept
, l
), t
);
3967 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3969 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3971 gimplify_and_add (x
, ilist
);
3975 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3976 x
= fold_build2 (POINTER_PLUS_EXPR
,
3977 TREE_TYPE (x
), x
, t
);
3979 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3982 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3983 || TREE_ADDRESSABLE (new_var
))
3984 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3985 idx
, lane
, ivar
, lvar
))
3987 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3989 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
3990 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3991 gimplify_and_add (x
, ilist
);
3992 gimple_stmt_iterator gsi
3993 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3995 = gimple_build_assign (unshare_expr (lvar
), iv
);
3996 gsi_insert_before_without_update (&gsi
, g
,
3998 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3999 enum tree_code code
= PLUS_EXPR
;
4000 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4001 code
= POINTER_PLUS_EXPR
;
4002 g
= gimple_build_assign (iv
, code
, iv
, t
);
4003 gsi_insert_before_without_update (&gsi
, g
,
4007 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4008 (c
, unshare_expr (ivar
), x
);
4009 gimplify_and_add (x
, &llist
[0]);
4010 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4013 gimple_seq tseq
= NULL
;
4016 gimplify_stmt (&dtor
, &tseq
);
4017 gimple_seq_add_seq (&llist
[1], tseq
);
4022 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
4023 gimplify_and_add (x
, ilist
);
4026 case OMP_CLAUSE__LOOPTEMP_
:
4027 gcc_assert (is_parallel_ctx (ctx
));
4028 x
= build_outer_var_ref (var
, ctx
);
4029 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4030 gimplify_and_add (x
, ilist
);
4033 case OMP_CLAUSE_COPYIN
:
4034 by_ref
= use_pointer_for_field (var
, NULL
);
4035 x
= build_receiver_ref (var
, by_ref
, ctx
);
4036 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4037 append_to_statement_list (x
, ©in_seq
);
4038 copyin_by_ref
|= by_ref
;
4041 case OMP_CLAUSE_REDUCTION
:
4042 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4044 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4046 x
= build_outer_var_ref (var
, ctx
);
4048 if (is_reference (var
)
4049 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4051 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4052 SET_DECL_VALUE_EXPR (placeholder
, x
);
4053 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4054 tree new_vard
= new_var
;
4055 if (is_reference (var
))
4057 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4058 new_vard
= TREE_OPERAND (new_var
, 0);
4059 gcc_assert (DECL_P (new_vard
));
4062 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4063 idx
, lane
, ivar
, lvar
))
4065 if (new_vard
== new_var
)
4067 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4068 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4072 SET_DECL_VALUE_EXPR (new_vard
,
4073 build_fold_addr_expr (ivar
));
4074 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4076 x
= lang_hooks
.decls
.omp_clause_default_ctor
4077 (c
, unshare_expr (ivar
),
4078 build_outer_var_ref (var
, ctx
));
4080 gimplify_and_add (x
, &llist
[0]);
4081 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4083 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4084 lower_omp (&tseq
, ctx
);
4085 gimple_seq_add_seq (&llist
[0], tseq
);
4087 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4088 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4089 lower_omp (&tseq
, ctx
);
4090 gimple_seq_add_seq (&llist
[1], tseq
);
4091 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4092 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4093 if (new_vard
== new_var
)
4094 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4096 SET_DECL_VALUE_EXPR (new_vard
,
4097 build_fold_addr_expr (lvar
));
4098 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4103 gimplify_stmt (&dtor
, &tseq
);
4104 gimple_seq_add_seq (&llist
[1], tseq
);
4108 /* If this is a reference to constant size reduction var
4109 with placeholder, we haven't emitted the initializer
4110 for it because it is undesirable if SIMD arrays are used.
4111 But if they aren't used, we need to emit the deferred
4112 initialization now. */
4113 else if (is_reference (var
) && is_simd
)
4114 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4115 x
= lang_hooks
.decls
.omp_clause_default_ctor
4116 (c
, unshare_expr (new_var
),
4117 build_outer_var_ref (var
, ctx
));
4119 gimplify_and_add (x
, ilist
);
4120 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4122 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4123 lower_omp (&tseq
, ctx
);
4124 gimple_seq_add_seq (ilist
, tseq
);
4126 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4129 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4130 lower_omp (&tseq
, ctx
);
4131 gimple_seq_add_seq (dlist
, tseq
);
4132 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4134 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4139 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4140 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4141 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4143 /* reduction(-:var) sums up the partial results, so it
4144 acts identically to reduction(+:var). */
4145 if (code
== MINUS_EXPR
)
4148 tree new_vard
= new_var
;
4149 if (is_simd
&& is_reference (var
))
4151 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4152 new_vard
= TREE_OPERAND (new_var
, 0);
4153 gcc_assert (DECL_P (new_vard
));
4156 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4157 idx
, lane
, ivar
, lvar
))
4159 tree ref
= build_outer_var_ref (var
, ctx
);
4161 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
4163 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
4164 ref
= build_outer_var_ref (var
, ctx
);
4165 gimplify_assign (ref
, x
, &llist
[1]);
4167 if (new_vard
!= new_var
)
4169 SET_DECL_VALUE_EXPR (new_vard
,
4170 build_fold_addr_expr (lvar
));
4171 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4176 if (is_reference (var
) && is_simd
)
4177 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4178 gimplify_assign (new_var
, x
, ilist
);
4181 tree ref
= build_outer_var_ref (var
, ctx
);
4183 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4184 ref
= build_outer_var_ref (var
, ctx
);
4185 gimplify_assign (ref
, x
, dlist
);
4199 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
4200 /* Don't want uninit warnings on simduid, it is always uninitialized,
4201 but we use it not for the value, but for the DECL_UID only. */
4202 TREE_NO_WARNING (uid
) = 1;
4204 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
4205 gimple_call_set_lhs (g
, lane
);
4206 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4207 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
4208 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
4209 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
4210 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4211 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4212 g
= gimple_build_assign (lane
, INTEGER_CST
,
4213 build_int_cst (unsigned_type_node
, 0));
4214 gimple_seq_add_stmt (ilist
, g
);
4215 for (int i
= 0; i
< 2; i
++)
4218 tree vf
= create_tmp_var (unsigned_type_node
);
4219 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
4220 gimple_call_set_lhs (g
, vf
);
4221 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
4222 gimple_seq_add_stmt (seq
, g
);
4223 tree t
= build_int_cst (unsigned_type_node
, 0);
4224 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
4225 gimple_seq_add_stmt (seq
, g
);
4226 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4227 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
4228 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4229 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
4230 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
4231 gimple_seq_add_seq (seq
, llist
[i
]);
4232 t
= build_int_cst (unsigned_type_node
, 1);
4233 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
4234 gimple_seq_add_stmt (seq
, g
);
4235 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
4236 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
4237 gimple_seq_add_stmt (seq
, g
);
4238 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
4242 /* The copyin sequence is not to be executed by the main thread, since
4243 that would result in self-copies. Perhaps not visible to scalars,
4244 but it certainly is to C++ operator=. */
4247 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
4249 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
4250 build_int_cst (TREE_TYPE (x
), 0));
4251 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
4252 gimplify_and_add (x
, ilist
);
4255 /* If any copyin variable is passed by reference, we must ensure the
4256 master thread doesn't modify it before it is copied over in all
4257 threads. Similarly for variables in both firstprivate and
4258 lastprivate clauses we need to ensure the lastprivate copying
4259 happens after firstprivate copying in all threads. And similarly
4260 for UDRs if initializer expression refers to omp_orig. */
4261 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
4263 /* Don't add any barrier for #pragma omp simd or
4264 #pragma omp distribute. */
4265 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
4266 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
4267 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
4270 /* If max_vf is non-zero, then we can use only a vectorization factor
4271 up to the max_vf we chose. So stick it into the safelen clause. */
4274 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4275 OMP_CLAUSE_SAFELEN
);
4277 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
4278 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4281 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
4282 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
4284 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4285 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4291 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4292 both parallel and workshare constructs. PREDICATE may be NULL if it's
4296 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
4299 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
4300 bool par_clauses
= false;
4301 tree simduid
= NULL
, lastlane
= NULL
;
4303 /* Early exit if there are no lastprivate or linear clauses. */
4304 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
4305 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
4306 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
4307 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
4309 if (clauses
== NULL
)
4311 /* If this was a workshare clause, see if it had been combined
4312 with its parallel. In that case, look for the clauses on the
4313 parallel statement itself. */
4314 if (is_parallel_ctx (ctx
))
4318 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4321 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4322 OMP_CLAUSE_LASTPRIVATE
);
4323 if (clauses
== NULL
)
4331 tree label_true
, arm1
, arm2
;
4333 label
= create_artificial_label (UNKNOWN_LOCATION
);
4334 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
4335 arm1
= TREE_OPERAND (predicate
, 0);
4336 arm2
= TREE_OPERAND (predicate
, 1);
4337 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4338 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4339 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
4341 gimple_seq_add_stmt (stmt_list
, stmt
);
4342 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
4345 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4346 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4348 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
4350 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
4353 for (c
= clauses
; c
;)
4356 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4358 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4359 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4360 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
4362 var
= OMP_CLAUSE_DECL (c
);
4363 new_var
= lookup_decl (var
, ctx
);
4365 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
4367 tree val
= DECL_VALUE_EXPR (new_var
);
4368 if (TREE_CODE (val
) == ARRAY_REF
4369 && VAR_P (TREE_OPERAND (val
, 0))
4370 && lookup_attribute ("omp simd array",
4371 DECL_ATTRIBUTES (TREE_OPERAND (val
,
4374 if (lastlane
== NULL
)
4376 lastlane
= create_tmp_var (unsigned_type_node
);
4378 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4380 TREE_OPERAND (val
, 1));
4381 gimple_call_set_lhs (g
, lastlane
);
4382 gimple_seq_add_stmt (stmt_list
, g
);
4384 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4385 TREE_OPERAND (val
, 0), lastlane
,
4386 NULL_TREE
, NULL_TREE
);
4390 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4391 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4393 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4394 gimple_seq_add_seq (stmt_list
,
4395 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4396 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4398 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4399 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4401 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4402 gimple_seq_add_seq (stmt_list
,
4403 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4404 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4407 x
= build_outer_var_ref (var
, ctx
);
4408 if (is_reference (var
))
4409 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4410 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4411 gimplify_and_add (x
, stmt_list
);
4413 c
= OMP_CLAUSE_CHAIN (c
);
4414 if (c
== NULL
&& !par_clauses
)
4416 /* If this was a workshare clause, see if it had been combined
4417 with its parallel. In that case, continue looking for the
4418 clauses also on the parallel statement itself. */
4419 if (is_parallel_ctx (ctx
))
4423 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4426 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4427 OMP_CLAUSE_LASTPRIVATE
);
4433 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4437 oacc_lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
,
4438 tree tid
, tree var
, tree new_var
)
4440 /* The atomic add at the end of the sum creates unnecessary
4441 write contention on accelerators. To work around this,
4442 create an array to store the partial reductions. Later, in
4443 lower_omp_for (for openacc), the values of array will be
4446 tree t
= NULL_TREE
, array
, x
;
4447 tree type
= get_base_type (var
);
4450 /* Now insert the partial reductions into the array. */
4452 /* Find the reduction array. */
4454 tree ptype
= build_pointer_type (type
);
4456 t
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
), ctx
);
4457 t
= build_receiver_ref (t
, false, ctx
->outer
);
4459 array
= create_tmp_var (ptype
);
4460 gimplify_assign (array
, t
, stmt_seqp
);
4462 tree ptr
= create_tmp_var (TREE_TYPE (array
));
4464 /* Find the reduction array. */
4466 /* testing a unary conversion. */
4467 tree offset
= create_tmp_var (sizetype
);
4468 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
4470 t
= create_tmp_var (sizetype
);
4471 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
4473 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, t
);
4474 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4476 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4477 of adding sizeof(var) to the array? */
4478 ptr
= create_tmp_var (ptype
);
4479 stmt
= gimple_build_assign (unshare_expr (ptr
), POINTER_PLUS_EXPR
, array
,
4481 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4483 /* Move the local sum to gfc$sum[i]. */
4484 x
= unshare_expr (build_simple_mem_ref (ptr
));
4485 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
4488 /* Generate code to implement the REDUCTION clauses. */
4491 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4493 gimple_seq sub_seq
= NULL
;
4495 tree x
, c
, tid
= NULL_TREE
;
4498 /* SIMD reductions are handled in lower_rec_input_clauses. */
4499 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4500 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4503 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4504 update in that case, otherwise use a lock. */
4505 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4506 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4508 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4510 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4520 /* Initialize thread info for OpenACC. */
4521 if (is_gimple_omp_oacc (ctx
->stmt
))
4523 /* Get the current thread id. */
4524 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
4525 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)));
4526 gimple stmt
= gimple_build_call (call
, 0);
4527 gimple_call_set_lhs (stmt
, tid
);
4528 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4531 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4533 tree var
, ref
, new_var
;
4534 enum tree_code code
;
4535 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4537 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4540 var
= OMP_CLAUSE_DECL (c
);
4541 new_var
= lookup_decl (var
, ctx
);
4542 if (is_reference (var
))
4543 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4544 ref
= build_outer_var_ref (var
, ctx
);
4545 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4547 /* reduction(-:var) sums up the partial results, so it acts
4548 identically to reduction(+:var). */
4549 if (code
== MINUS_EXPR
)
4552 if (is_gimple_omp_oacc (ctx
->stmt
))
4554 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
));
4556 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
4558 else if (count
== 1)
4560 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4562 addr
= save_expr (addr
);
4563 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4564 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4565 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4566 gimplify_and_add (x
, stmt_seqp
);
4569 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4571 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4573 if (is_reference (var
)
4574 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4576 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4577 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4578 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4579 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4580 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4581 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4582 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4586 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4587 ref
= build_outer_var_ref (var
, ctx
);
4588 gimplify_assign (ref
, x
, &sub_seq
);
4592 if (is_gimple_omp_oacc (ctx
->stmt
))
4595 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4597 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4599 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4601 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4603 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4607 /* Generate code to implement the COPYPRIVATE clauses. */
4610 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4615 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4617 tree var
, new_var
, ref
, x
;
4619 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4621 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4624 var
= OMP_CLAUSE_DECL (c
);
4625 by_ref
= use_pointer_for_field (var
, NULL
);
4627 ref
= build_sender_ref (var
, ctx
);
4628 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4631 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4632 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4634 gimplify_assign (ref
, x
, slist
);
4636 ref
= build_receiver_ref (var
, false, ctx
);
4639 ref
= fold_convert_loc (clause_loc
,
4640 build_pointer_type (TREE_TYPE (new_var
)),
4642 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4644 if (is_reference (var
))
4646 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4647 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4648 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4650 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4651 gimplify_and_add (x
, rlist
);
4656 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4657 and REDUCTION from the sender (aka parent) side. */
4660 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4665 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4667 tree val
, ref
, x
, var
;
4668 bool by_ref
, do_in
= false, do_out
= false;
4669 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4671 switch (OMP_CLAUSE_CODE (c
))
4673 case OMP_CLAUSE_PRIVATE
:
4674 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4677 case OMP_CLAUSE_FIRSTPRIVATE
:
4678 case OMP_CLAUSE_COPYIN
:
4679 case OMP_CLAUSE_LASTPRIVATE
:
4680 case OMP_CLAUSE_REDUCTION
:
4681 case OMP_CLAUSE__LOOPTEMP_
:
4687 val
= OMP_CLAUSE_DECL (c
);
4688 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4690 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4691 && is_global_var (var
))
4693 if (is_variable_sized (val
))
4695 by_ref
= use_pointer_for_field (val
, NULL
);
4697 switch (OMP_CLAUSE_CODE (c
))
4699 case OMP_CLAUSE_PRIVATE
:
4700 case OMP_CLAUSE_FIRSTPRIVATE
:
4701 case OMP_CLAUSE_COPYIN
:
4702 case OMP_CLAUSE__LOOPTEMP_
:
4706 case OMP_CLAUSE_LASTPRIVATE
:
4707 if (by_ref
|| is_reference (val
))
4709 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4716 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4721 case OMP_CLAUSE_REDUCTION
:
4723 do_out
= !(by_ref
|| is_reference (val
));
4732 ref
= build_sender_ref (val
, ctx
);
4733 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4734 gimplify_assign (ref
, x
, ilist
);
4735 if (is_task_ctx (ctx
))
4736 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4741 ref
= build_sender_ref (val
, ctx
);
4742 gimplify_assign (var
, ref
, olist
);
4747 /* Generate code to implement SHARED from the sender (aka parent)
4748 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4749 list things that got automatically shared. */
4752 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4754 tree var
, ovar
, nvar
, f
, x
, record_type
;
4756 if (ctx
->record_type
== NULL
)
4759 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4760 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4762 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4763 nvar
= maybe_lookup_decl (ovar
, ctx
);
4764 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4767 /* If CTX is a nested parallel directive. Find the immediately
4768 enclosing parallel or workshare construct that contains a
4769 mapping for OVAR. */
4770 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4772 if (use_pointer_for_field (ovar
, ctx
))
4774 x
= build_sender_ref (ovar
, ctx
);
4775 var
= build_fold_addr_expr (var
);
4776 gimplify_assign (x
, var
, ilist
);
4780 x
= build_sender_ref (ovar
, ctx
);
4781 gimplify_assign (x
, var
, ilist
);
4783 if (!TREE_READONLY (var
)
4784 /* We don't need to receive a new reference to a result
4785 or parm decl. In fact we may not store to it as we will
4786 invalidate any pending RSO and generate wrong gimple
4788 && !((TREE_CODE (var
) == RESULT_DECL
4789 || TREE_CODE (var
) == PARM_DECL
)
4790 && DECL_BY_REFERENCE (var
)))
4792 x
= build_sender_ref (ovar
, ctx
);
4793 gimplify_assign (var
, x
, olist
);
4800 /* A convenience function to build an empty GIMPLE_COND with just the
4804 gimple_build_cond_empty (tree cond
)
4806 enum tree_code pred_code
;
4809 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4810 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4814 /* Build the function calls to GOMP_parallel_start etc to actually
4815 generate the parallel operation. REGION is the parallel region
4816 being expanded. BB is the block where to insert the code. WS_ARGS
4817 will be set if this is a call to a combined parallel+workshare
4818 construct, it contains the list of additional arguments needed by
4819 the workshare construct. */
4822 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4823 gomp_parallel
*entry_stmt
,
4824 vec
<tree
, va_gc
> *ws_args
)
4826 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4827 gimple_stmt_iterator gsi
;
4829 enum built_in_function start_ix
;
4831 location_t clause_loc
;
4832 vec
<tree
, va_gc
> *args
;
4834 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4836 /* Determine what flavor of GOMP_parallel we will be
4838 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4839 if (is_combined_parallel (region
))
4841 switch (region
->inner
->type
)
4843 case GIMPLE_OMP_FOR
:
4844 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4845 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4846 + (region
->inner
->sched_kind
4847 == OMP_CLAUSE_SCHEDULE_RUNTIME
4848 ? 3 : region
->inner
->sched_kind
));
4849 start_ix
= (enum built_in_function
)start_ix2
;
4851 case GIMPLE_OMP_SECTIONS
:
4852 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4859 /* By default, the value of NUM_THREADS is zero (selected at run time)
4860 and there is no conditional. */
4862 val
= build_int_cst (unsigned_type_node
, 0);
4863 flags
= build_int_cst (unsigned_type_node
, 0);
4865 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4867 cond
= OMP_CLAUSE_IF_EXPR (c
);
4869 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4872 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4873 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4876 clause_loc
= gimple_location (entry_stmt
);
4878 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4880 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4882 /* Ensure 'val' is of the correct type. */
4883 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4885 /* If we found the clause 'if (cond)', build either
4886 (cond != 0) or (cond ? val : 1u). */
4889 cond
= gimple_boolify (cond
);
4891 if (integer_zerop (val
))
4892 val
= fold_build2_loc (clause_loc
,
4893 EQ_EXPR
, unsigned_type_node
, cond
,
4894 build_int_cst (TREE_TYPE (cond
), 0));
4897 basic_block cond_bb
, then_bb
, else_bb
;
4898 edge e
, e_then
, e_else
;
4899 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4901 tmp_var
= create_tmp_var (TREE_TYPE (val
));
4902 if (gimple_in_ssa_p (cfun
))
4904 tmp_then
= make_ssa_name (tmp_var
);
4905 tmp_else
= make_ssa_name (tmp_var
);
4906 tmp_join
= make_ssa_name (tmp_var
);
4915 e
= split_block_after_labels (bb
);
4920 then_bb
= create_empty_bb (cond_bb
);
4921 else_bb
= create_empty_bb (then_bb
);
4922 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4923 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4925 stmt
= gimple_build_cond_empty (cond
);
4926 gsi
= gsi_start_bb (cond_bb
);
4927 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4929 gsi
= gsi_start_bb (then_bb
);
4930 stmt
= gimple_build_assign (tmp_then
, val
);
4931 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4933 gsi
= gsi_start_bb (else_bb
);
4934 stmt
= gimple_build_assign
4935 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4936 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4938 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4939 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4940 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4941 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4942 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4943 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4945 if (gimple_in_ssa_p (cfun
))
4947 gphi
*phi
= create_phi_node (tmp_join
, bb
);
4948 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4949 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4955 gsi
= gsi_start_bb (bb
);
4956 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4957 false, GSI_CONTINUE_LINKING
);
4960 gsi
= gsi_last_bb (bb
);
4961 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4963 t1
= null_pointer_node
;
4965 t1
= build_fold_addr_expr (t
);
4966 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4968 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4969 args
->quick_push (t2
);
4970 args
->quick_push (t1
);
4971 args
->quick_push (val
);
4973 args
->splice (*ws_args
);
4974 args
->quick_push (flags
);
4976 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4977 builtin_decl_explicit (start_ix
), args
);
4979 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4980 false, GSI_CONTINUE_LINKING
);
4983 /* Insert a function call whose name is FUNC_NAME with the information from
4984 ENTRY_STMT into the basic_block BB. */
4987 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
4988 vec
<tree
, va_gc
> *ws_args
)
4991 gimple_stmt_iterator gsi
;
4992 vec
<tree
, va_gc
> *args
;
4994 gcc_assert (vec_safe_length (ws_args
) == 2);
4995 tree func_name
= (*ws_args
)[0];
4996 tree grain
= (*ws_args
)[1];
4998 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4999 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
5000 gcc_assert (count
!= NULL_TREE
);
5001 count
= OMP_CLAUSE_OPERAND (count
, 0);
5003 gsi
= gsi_last_bb (bb
);
5004 t
= gimple_omp_parallel_data_arg (entry_stmt
);
5006 t1
= null_pointer_node
;
5008 t1
= build_fold_addr_expr (t
);
5009 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5011 vec_alloc (args
, 4);
5012 args
->quick_push (t2
);
5013 args
->quick_push (t1
);
5014 args
->quick_push (count
);
5015 args
->quick_push (grain
);
5016 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
5018 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
5019 GSI_CONTINUE_LINKING
);
5022 /* Build the function call to GOMP_task to actually
5023 generate the task operation. BB is the block where to insert the code. */
5026 expand_task_call (basic_block bb
, gomp_task
*entry_stmt
)
5028 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
5029 gimple_stmt_iterator gsi
;
5030 location_t loc
= gimple_location (entry_stmt
);
5032 clauses
= gimple_omp_task_clauses (entry_stmt
);
5034 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
5036 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
5038 cond
= boolean_true_node
;
5040 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
5041 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
5042 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
5043 flags
= build_int_cst (unsigned_type_node
,
5044 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
5046 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
5049 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
5050 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
5051 build_int_cst (unsigned_type_node
, 2),
5052 build_int_cst (unsigned_type_node
, 0));
5053 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
5056 depend
= OMP_CLAUSE_DECL (depend
);
5058 depend
= build_int_cst (ptr_type_node
, 0);
5060 gsi
= gsi_last_bb (bb
);
5061 t
= gimple_omp_task_data_arg (entry_stmt
);
5063 t2
= null_pointer_node
;
5065 t2
= build_fold_addr_expr_loc (loc
, t
);
5066 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
5067 t
= gimple_omp_task_copy_fn (entry_stmt
);
5069 t3
= null_pointer_node
;
5071 t3
= build_fold_addr_expr_loc (loc
, t
);
5073 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
5075 gimple_omp_task_arg_size (entry_stmt
),
5076 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
5079 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5080 false, GSI_CONTINUE_LINKING
);
5084 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5085 catch handler and return it. This prevents programs from violating the
5086 structured block semantics with throws. */
5089 maybe_catch_exception (gimple_seq body
)
5094 if (!flag_exceptions
)
5097 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
5098 decl
= lang_hooks
.eh_protect_cleanup_actions ();
5100 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
5102 g
= gimple_build_eh_must_not_throw (decl
);
5103 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
5106 return gimple_seq_alloc_with_stmt (g
);
5109 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5112 vec2chain (vec
<tree
, va_gc
> *v
)
5114 tree chain
= NULL_TREE
, t
;
5117 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
5119 DECL_CHAIN (t
) = chain
;
5127 /* Remove barriers in REGION->EXIT's block. Note that this is only
5128 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5129 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5130 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5134 remove_exit_barrier (struct omp_region
*region
)
5136 gimple_stmt_iterator gsi
;
5137 basic_block exit_bb
;
5141 int any_addressable_vars
= -1;
5143 exit_bb
= region
->exit
;
5145 /* If the parallel region doesn't return, we don't have REGION->EXIT
5150 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5151 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5152 statements that can appear in between are extremely limited -- no
5153 memory operations at all. Here, we allow nothing at all, so the
5154 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5155 gsi
= gsi_last_bb (exit_bb
);
5156 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5158 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
5161 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
5163 gsi
= gsi_last_bb (e
->src
);
5164 if (gsi_end_p (gsi
))
5166 stmt
= gsi_stmt (gsi
);
5167 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
5168 && !gimple_omp_return_nowait_p (stmt
))
5170 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5171 in many cases. If there could be tasks queued, the barrier
5172 might be needed to let the tasks run before some local
5173 variable of the parallel that the task uses as shared
5174 runs out of scope. The task can be spawned either
5175 from within current function (this would be easy to check)
5176 or from some function it calls and gets passed an address
5177 of such a variable. */
5178 if (any_addressable_vars
< 0)
5180 gomp_parallel
*parallel_stmt
5181 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
5182 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
5183 tree local_decls
, block
, decl
;
5186 any_addressable_vars
= 0;
5187 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
5188 if (TREE_ADDRESSABLE (decl
))
5190 any_addressable_vars
= 1;
5193 for (block
= gimple_block (stmt
);
5194 !any_addressable_vars
5196 && TREE_CODE (block
) == BLOCK
;
5197 block
= BLOCK_SUPERCONTEXT (block
))
5199 for (local_decls
= BLOCK_VARS (block
);
5201 local_decls
= DECL_CHAIN (local_decls
))
5202 if (TREE_ADDRESSABLE (local_decls
))
5204 any_addressable_vars
= 1;
5207 if (block
== gimple_block (parallel_stmt
))
5211 if (!any_addressable_vars
)
5212 gimple_omp_return_set_nowait (stmt
);
5218 remove_exit_barriers (struct omp_region
*region
)
5220 if (region
->type
== GIMPLE_OMP_PARALLEL
)
5221 remove_exit_barrier (region
);
5225 region
= region
->inner
;
5226 remove_exit_barriers (region
);
5227 while (region
->next
)
5229 region
= region
->next
;
5230 remove_exit_barriers (region
);
5235 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5236 calls. These can't be declared as const functions, but
5237 within one parallel body they are constant, so they can be
5238 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5239 which are declared const. Similarly for task body, except
5240 that in untied task omp_get_thread_num () can change at any task
5241 scheduling point. */
5244 optimize_omp_library_calls (gimple entry_stmt
)
5247 gimple_stmt_iterator gsi
;
5248 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5249 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
5250 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5251 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
5252 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
5253 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
5254 OMP_CLAUSE_UNTIED
) != NULL
);
5256 FOR_EACH_BB_FN (bb
, cfun
)
5257 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5259 gimple call
= gsi_stmt (gsi
);
5262 if (is_gimple_call (call
)
5263 && (decl
= gimple_call_fndecl (call
))
5264 && DECL_EXTERNAL (decl
)
5265 && TREE_PUBLIC (decl
)
5266 && DECL_INITIAL (decl
) == NULL
)
5270 if (DECL_NAME (decl
) == thr_num_id
)
5272 /* In #pragma omp task untied omp_get_thread_num () can change
5273 during the execution of the task region. */
5276 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5278 else if (DECL_NAME (decl
) == num_thr_id
)
5279 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5283 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
5284 || gimple_call_num_args (call
) != 0)
5287 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
5290 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
5291 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
5292 TREE_TYPE (TREE_TYPE (built_in
))))
5295 gimple_call_set_fndecl (call
, built_in
);
5300 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5304 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
5308 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5309 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
5312 if (TREE_CODE (t
) == ADDR_EXPR
)
5313 recompute_tree_invariant_for_addr_expr (t
);
5315 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
5319 /* Prepend TO = FROM assignment before *GSI_P. */
5322 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
5324 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
5325 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
5326 true, GSI_SAME_STMT
);
5327 gimple stmt
= gimple_build_assign (to
, from
);
5328 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
5329 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
5330 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
5332 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
5333 gimple_regimplify_operands (stmt
, &gsi
);
5337 /* Expand the OpenMP parallel or task directive starting at REGION. */
5340 expand_omp_taskreg (struct omp_region
*region
)
5342 basic_block entry_bb
, exit_bb
, new_bb
;
5343 struct function
*child_cfun
;
5344 tree child_fn
, block
, t
;
5345 gimple_stmt_iterator gsi
;
5346 gimple entry_stmt
, stmt
;
5348 vec
<tree
, va_gc
> *ws_args
;
5350 entry_stmt
= last_stmt (region
->entry
);
5351 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
5352 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
5354 entry_bb
= region
->entry
;
5355 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
5356 exit_bb
= region
->cont
;
5358 exit_bb
= region
->exit
;
5362 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
5363 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
5364 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
5367 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5368 and the inner statement contains the name of the built-in function
5370 ws_args
= region
->inner
->ws_args
;
5371 else if (is_combined_parallel (region
))
5372 ws_args
= region
->ws_args
;
5376 if (child_cfun
->cfg
)
5378 /* Due to inlining, it may happen that we have already outlined
5379 the region, in which case all we need to do is make the
5380 sub-graph unreachable and emit the parallel call. */
5381 edge entry_succ_e
, exit_succ_e
;
5383 entry_succ_e
= single_succ_edge (entry_bb
);
5385 gsi
= gsi_last_bb (entry_bb
);
5386 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
5387 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
5388 gsi_remove (&gsi
, true);
5393 exit_succ_e
= single_succ_edge (exit_bb
);
5394 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
5396 remove_edge_and_dominated_blocks (entry_succ_e
);
5400 unsigned srcidx
, dstidx
, num
;
5402 /* If the parallel region needs data sent from the parent
5403 function, then the very first statement (except possible
5404 tree profile counter updates) of the parallel body
5405 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5406 &.OMP_DATA_O is passed as an argument to the child function,
5407 we need to replace it with the argument as seen by the child
5410 In most cases, this will end up being the identity assignment
5411 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5412 a function call that has been inlined, the original PARM_DECL
5413 .OMP_DATA_I may have been converted into a different local
5414 variable. In which case, we need to keep the assignment. */
5415 if (gimple_omp_taskreg_data_arg (entry_stmt
))
5417 basic_block entry_succ_bb
5418 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
5419 : FALLTHRU_EDGE (entry_bb
)->dest
;
5421 gimple parcopy_stmt
= NULL
;
5423 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
5427 gcc_assert (!gsi_end_p (gsi
));
5428 stmt
= gsi_stmt (gsi
);
5429 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
5432 if (gimple_num_ops (stmt
) == 2)
5434 tree arg
= gimple_assign_rhs1 (stmt
);
5436 /* We're ignore the subcode because we're
5437 effectively doing a STRIP_NOPS. */
5439 if (TREE_CODE (arg
) == ADDR_EXPR
5440 && TREE_OPERAND (arg
, 0)
5441 == gimple_omp_taskreg_data_arg (entry_stmt
))
5443 parcopy_stmt
= stmt
;
5449 gcc_assert (parcopy_stmt
!= NULL
);
5450 arg
= DECL_ARGUMENTS (child_fn
);
5452 if (!gimple_in_ssa_p (cfun
))
5454 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5455 gsi_remove (&gsi
, true);
5458 /* ?? Is setting the subcode really necessary ?? */
5459 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5460 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5465 /* If we are in ssa form, we must load the value from the default
5466 definition of the argument. That should not be defined now,
5467 since the argument is not used uninitialized. */
5468 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5469 narg
= make_ssa_name (arg
, gimple_build_nop ());
5470 set_ssa_default_def (cfun
, arg
, narg
);
5471 /* ?? Is setting the subcode really necessary ?? */
5472 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5473 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5474 update_stmt (parcopy_stmt
);
5478 /* Declare local variables needed in CHILD_CFUN. */
5479 block
= DECL_INITIAL (child_fn
);
5480 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5481 /* The gimplifier could record temporaries in parallel/task block
5482 rather than in containing function's local_decls chain,
5483 which would mean cgraph missed finalizing them. Do it now. */
5484 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5485 if (TREE_CODE (t
) == VAR_DECL
5487 && !DECL_EXTERNAL (t
))
5488 varpool_node::finalize_decl (t
);
5489 DECL_SAVED_TREE (child_fn
) = NULL
;
5490 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5491 gimple_set_body (child_fn
, NULL
);
5492 TREE_USED (block
) = 1;
5494 /* Reset DECL_CONTEXT on function arguments. */
5495 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5496 DECL_CONTEXT (t
) = child_fn
;
5498 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5499 so that it can be moved to the child function. */
5500 gsi
= gsi_last_bb (entry_bb
);
5501 stmt
= gsi_stmt (gsi
);
5502 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5503 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5504 e
= split_block (entry_bb
, stmt
);
5505 gsi_remove (&gsi
, true);
5508 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5509 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5512 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
5513 gcc_assert (e2
->dest
== region
->exit
);
5514 remove_edge (BRANCH_EDGE (entry_bb
));
5515 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
5516 gsi
= gsi_last_bb (region
->exit
);
5517 gcc_assert (!gsi_end_p (gsi
)
5518 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5519 gsi_remove (&gsi
, true);
5522 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
5525 gsi
= gsi_last_bb (exit_bb
);
5526 gcc_assert (!gsi_end_p (gsi
)
5527 && (gimple_code (gsi_stmt (gsi
))
5528 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
5529 stmt
= gimple_build_return (NULL
);
5530 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5531 gsi_remove (&gsi
, true);
5534 /* Move the parallel region into CHILD_CFUN. */
5536 if (gimple_in_ssa_p (cfun
))
5538 init_tree_ssa (child_cfun
);
5539 init_ssa_operands (child_cfun
);
5540 child_cfun
->gimple_df
->in_ssa_p
= true;
5544 block
= gimple_block (entry_stmt
);
5546 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5548 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5551 basic_block dest_bb
= e2
->dest
;
5553 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
5555 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
5557 /* When the OMP expansion process cannot guarantee an up-to-date
5558 loop tree arrange for the child function to fixup loops. */
5559 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5560 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5562 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5563 num
= vec_safe_length (child_cfun
->local_decls
);
5564 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5566 t
= (*child_cfun
->local_decls
)[srcidx
];
5567 if (DECL_CONTEXT (t
) == cfun
->decl
)
5569 if (srcidx
!= dstidx
)
5570 (*child_cfun
->local_decls
)[dstidx
] = t
;
5574 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5576 /* Inform the callgraph about the new function. */
5577 child_cfun
->curr_properties
= cfun
->curr_properties
;
5578 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
5579 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
5580 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
5581 node
->parallelized_function
= 1;
5582 cgraph_node::add_new_function (child_fn
, true);
5584 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5585 fixed in a following pass. */
5586 push_cfun (child_cfun
);
5588 optimize_omp_library_calls (entry_stmt
);
5589 cgraph_edge::rebuild_edges ();
5591 /* Some EH regions might become dead, see PR34608. If
5592 pass_cleanup_cfg isn't the first pass to happen with the
5593 new child, these dead EH edges might cause problems.
5594 Clean them up now. */
5595 if (flag_exceptions
)
5598 bool changed
= false;
5600 FOR_EACH_BB_FN (bb
, cfun
)
5601 changed
|= gimple_purge_dead_eh_edges (bb
);
5603 cleanup_tree_cfg ();
5605 if (gimple_in_ssa_p (cfun
))
5606 update_ssa (TODO_update_ssa
);
5610 /* Emit a library call to launch the children threads. */
5612 expand_cilk_for_call (new_bb
,
5613 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5614 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5615 expand_parallel_call (region
, new_bb
,
5616 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5618 expand_task_call (new_bb
, as_a
<gomp_task
*> (entry_stmt
));
5619 if (gimple_in_ssa_p (cfun
))
5620 update_ssa (TODO_update_ssa_only_virtuals
);
5624 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5625 of the combined collapse > 1 loop constructs, generate code like:
5626 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5631 count3 = (adj + N32 - N31) / STEP3;
5632 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5637 count2 = (adj + N22 - N21) / STEP2;
5638 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5643 count1 = (adj + N12 - N11) / STEP1;
5644 count = count1 * count2 * count3;
5645 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5647 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5648 of the combined loop constructs, just initialize COUNTS array
5649 from the _looptemp_ clauses. */
5651 /* NOTE: It *could* be better to moosh all of the BBs together,
5652 creating one larger BB with all the computation and the unexpected
5653 jump at the end. I.e.
5655 bool zero3, zero2, zero1, zero;
5658 count3 = (N32 - N31) /[cl] STEP3;
5660 count2 = (N22 - N21) /[cl] STEP2;
5662 count1 = (N12 - N11) /[cl] STEP1;
5663 zero = zero3 || zero2 || zero1;
5664 count = count1 * count2 * count3;
5665 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5667 After all, we expect the zero=false, and thus we expect to have to
5668 evaluate all of the comparison expressions, so short-circuiting
5669 oughtn't be a win. Since the condition isn't protecting a
5670 denominator, we're not concerned about divide-by-zero, so we can
5671 fully evaluate count even if a numerator turned out to be wrong.
5673 It seems like putting this all together would create much better
5674 scheduling opportunities, and less pressure on the chip's branch
5678 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5679 basic_block
&entry_bb
, tree
*counts
,
5680 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5681 basic_block
&l2_dom_bb
)
5683 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5687 /* Collapsed loops need work for expansion into SSA form. */
5688 gcc_assert (!gimple_in_ssa_p (cfun
));
5690 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5691 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5693 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5694 isn't supposed to be handled, as the inner loop doesn't
5696 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5697 OMP_CLAUSE__LOOPTEMP_
);
5698 gcc_assert (innerc
);
5699 for (i
= 0; i
< fd
->collapse
; i
++)
5701 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5702 OMP_CLAUSE__LOOPTEMP_
);
5703 gcc_assert (innerc
);
5705 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5707 counts
[0] = NULL_TREE
;
5712 for (i
= 0; i
< fd
->collapse
; i
++)
5714 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5716 if (SSA_VAR_P (fd
->loop
.n2
)
5717 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5718 fold_convert (itype
, fd
->loops
[i
].n1
),
5719 fold_convert (itype
, fd
->loops
[i
].n2
)))
5720 == NULL_TREE
|| !integer_onep (t
)))
5724 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5725 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5726 true, GSI_SAME_STMT
);
5727 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5728 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5729 true, GSI_SAME_STMT
);
5730 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5731 NULL_TREE
, NULL_TREE
);
5732 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
5733 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
5734 expand_omp_regimplify_p
, NULL
, NULL
)
5735 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
5736 expand_omp_regimplify_p
, NULL
, NULL
))
5738 *gsi
= gsi_for_stmt (cond_stmt
);
5739 gimple_regimplify_operands (cond_stmt
, gsi
);
5741 e
= split_block (entry_bb
, cond_stmt
);
5742 if (zero_iter_bb
== NULL
)
5744 gassign
*assign_stmt
;
5745 first_zero_iter
= i
;
5746 zero_iter_bb
= create_empty_bb (entry_bb
);
5747 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5748 *gsi
= gsi_after_labels (zero_iter_bb
);
5749 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
5750 build_zero_cst (type
));
5751 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
5752 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5755 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5756 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5757 e
->flags
= EDGE_TRUE_VALUE
;
5758 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5759 if (l2_dom_bb
== NULL
)
5760 l2_dom_bb
= entry_bb
;
5762 *gsi
= gsi_last_bb (entry_bb
);
5765 if (POINTER_TYPE_P (itype
))
5766 itype
= signed_type_for (itype
);
5767 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5769 t
= fold_build2 (PLUS_EXPR
, itype
,
5770 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5771 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5772 fold_convert (itype
, fd
->loops
[i
].n2
));
5773 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5774 fold_convert (itype
, fd
->loops
[i
].n1
));
5775 /* ?? We could probably use CEIL_DIV_EXPR instead of
5776 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5777 generate the same code in the end because generically we
5778 don't know that the values involved must be negative for
5780 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5781 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5782 fold_build1 (NEGATE_EXPR
, itype
, t
),
5783 fold_build1 (NEGATE_EXPR
, itype
,
5784 fold_convert (itype
,
5785 fd
->loops
[i
].step
)));
5787 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5788 fold_convert (itype
, fd
->loops
[i
].step
));
5789 t
= fold_convert (type
, t
);
5790 if (TREE_CODE (t
) == INTEGER_CST
)
5794 counts
[i
] = create_tmp_reg (type
, ".count");
5795 expand_omp_build_assign (gsi
, counts
[i
], t
);
5797 if (SSA_VAR_P (fd
->loop
.n2
))
5802 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5803 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5809 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5811 V3 = N31 + (T % count3) * STEP3;
5813 V2 = N21 + (T % count2) * STEP2;
5815 V1 = N11 + T * STEP1;
5816 if this loop doesn't have an inner loop construct combined with it.
5817 If it does have an inner loop construct combined with it and the
5818 iteration count isn't known constant, store values from counts array
5819 into its _looptemp_ temporaries instead. */
5822 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5823 tree
*counts
, gimple inner_stmt
, tree startvar
)
5826 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5828 /* If fd->loop.n2 is constant, then no propagation of the counts
5829 is needed, they are constant. */
5830 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5833 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5834 ? gimple_omp_parallel_clauses (inner_stmt
)
5835 : gimple_omp_for_clauses (inner_stmt
);
5836 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5837 isn't supposed to be handled, as the inner loop doesn't
5839 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5840 gcc_assert (innerc
);
5841 for (i
= 0; i
< fd
->collapse
; i
++)
5843 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5844 OMP_CLAUSE__LOOPTEMP_
);
5845 gcc_assert (innerc
);
5848 tree tem
= OMP_CLAUSE_DECL (innerc
);
5849 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5850 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5851 false, GSI_CONTINUE_LINKING
);
5852 gassign
*stmt
= gimple_build_assign (tem
, t
);
5853 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5859 tree type
= TREE_TYPE (fd
->loop
.v
);
5860 tree tem
= create_tmp_reg (type
, ".tem");
5861 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
5862 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5864 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5866 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5868 if (POINTER_TYPE_P (vtype
))
5869 itype
= signed_type_for (vtype
);
5871 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5874 t
= fold_convert (itype
, t
);
5875 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5876 fold_convert (itype
, fd
->loops
[i
].step
));
5877 if (POINTER_TYPE_P (vtype
))
5878 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5880 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5881 t
= force_gimple_operand_gsi (gsi
, t
,
5882 DECL_P (fd
->loops
[i
].v
)
5883 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5885 GSI_CONTINUE_LINKING
);
5886 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5887 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5890 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5891 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5892 false, GSI_CONTINUE_LINKING
);
5893 stmt
= gimple_build_assign (tem
, t
);
5894 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5900 /* Helper function for expand_omp_for_*. Generate code like:
5903 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5907 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5914 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5915 basic_block body_bb
)
5917 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5919 gimple_stmt_iterator gsi
;
5925 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5927 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5929 bb
= create_empty_bb (last_bb
);
5930 add_bb_to_loop (bb
, last_bb
->loop_father
);
5931 gsi
= gsi_start_bb (bb
);
5933 if (i
< fd
->collapse
- 1)
5935 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5936 e
->probability
= REG_BR_PROB_BASE
/ 8;
5938 t
= fd
->loops
[i
+ 1].n1
;
5939 t
= force_gimple_operand_gsi (&gsi
, t
,
5940 DECL_P (fd
->loops
[i
+ 1].v
)
5941 && TREE_ADDRESSABLE (fd
->loops
[i
5944 GSI_CONTINUE_LINKING
);
5945 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5946 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5951 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5953 if (POINTER_TYPE_P (vtype
))
5954 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5956 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5957 t
= force_gimple_operand_gsi (&gsi
, t
,
5958 DECL_P (fd
->loops
[i
].v
)
5959 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5960 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5961 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5962 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5966 t
= fd
->loops
[i
].n2
;
5967 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5968 false, GSI_CONTINUE_LINKING
);
5969 tree v
= fd
->loops
[i
].v
;
5970 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5971 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5972 false, GSI_CONTINUE_LINKING
);
5973 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5974 stmt
= gimple_build_cond_empty (t
);
5975 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5976 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5977 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5980 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5988 /* A subroutine of expand_omp_for. Generate code for a parallel
5989 loop with any schedule. Given parameters:
5991 for (V = N1; V cond N2; V += STEP) BODY;
5993 where COND is "<" or ">", we generate pseudocode
5995 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5996 if (more) goto L0; else goto L3;
6003 if (V cond iend) goto L1; else goto L2;
6005 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6008 If this is a combined omp parallel loop, instead of the call to
6009 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
6010 If this is gimple_omp_for_combined_p loop, then instead of assigning
6011 V and iend in L0 we assign the first two _looptemp_ clause decls of the
6012 inner GIMPLE_OMP_FOR and V += STEP; and
6013 if (V cond iend) goto L1; else goto L2; are removed.
6015 For collapsed loops, given parameters:
6017 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6018 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6019 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6022 we generate pseudocode
6024 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6029 count3 = (adj + N32 - N31) / STEP3;
6030 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6035 count2 = (adj + N22 - N21) / STEP2;
6036 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6041 count1 = (adj + N12 - N11) / STEP1;
6042 count = count1 * count2 * count3;
6047 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6048 if (more) goto L0; else goto L3;
6052 V3 = N31 + (T % count3) * STEP3;
6054 V2 = N21 + (T % count2) * STEP2;
6056 V1 = N11 + T * STEP1;
6061 if (V < iend) goto L10; else goto L2;
6064 if (V3 cond3 N32) goto L1; else goto L11;
6068 if (V2 cond2 N22) goto L1; else goto L12;
6074 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6080 expand_omp_for_generic (struct omp_region
*region
,
6081 struct omp_for_data
*fd
,
6082 enum built_in_function start_fn
,
6083 enum built_in_function next_fn
,
6086 tree type
, istart0
, iend0
, iend
;
6087 tree t
, vmain
, vback
, bias
= NULL_TREE
;
6088 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
6089 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
6090 gimple_stmt_iterator gsi
;
6091 gassign
*assign_stmt
;
6092 bool in_combined_parallel
= is_combined_parallel (region
);
6093 bool broken_loop
= region
->cont
== NULL
;
6095 tree
*counts
= NULL
;
6098 gcc_assert (!broken_loop
|| !in_combined_parallel
);
6099 gcc_assert (fd
->iter_type
== long_integer_type_node
6100 || !in_combined_parallel
);
6102 type
= TREE_TYPE (fd
->loop
.v
);
6103 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
6104 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
6105 TREE_ADDRESSABLE (istart0
) = 1;
6106 TREE_ADDRESSABLE (iend0
) = 1;
6108 /* See if we need to bias by LLONG_MIN. */
6109 if (fd
->iter_type
== long_long_unsigned_type_node
6110 && TREE_CODE (type
) == INTEGER_TYPE
6111 && !TYPE_UNSIGNED (type
))
6115 if (fd
->loop
.cond_code
== LT_EXPR
)
6118 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6122 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6125 if (TREE_CODE (n1
) != INTEGER_CST
6126 || TREE_CODE (n2
) != INTEGER_CST
6127 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
6128 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
6131 entry_bb
= region
->entry
;
6132 cont_bb
= region
->cont
;
6134 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6135 gcc_assert (broken_loop
6136 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6137 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6138 l1_bb
= single_succ (l0_bb
);
6141 l2_bb
= create_empty_bb (cont_bb
);
6142 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
6143 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6147 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6148 exit_bb
= region
->exit
;
6150 gsi
= gsi_last_bb (entry_bb
);
6152 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6153 if (fd
->collapse
> 1)
6155 int first_zero_iter
= -1;
6156 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
6158 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6159 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6160 zero_iter_bb
, first_zero_iter
,
6165 /* Some counts[i] vars might be uninitialized if
6166 some loop has zero iterations. But the body shouldn't
6167 be executed in that case, so just avoid uninit warnings. */
6168 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
6169 if (SSA_VAR_P (counts
[i
]))
6170 TREE_NO_WARNING (counts
[i
]) = 1;
6172 e
= split_block (entry_bb
, gsi_stmt (gsi
));
6174 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
6175 gsi
= gsi_last_bb (entry_bb
);
6176 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
6177 get_immediate_dominator (CDI_DOMINATORS
,
6181 if (in_combined_parallel
)
6183 /* In a combined parallel loop, emit a call to
6184 GOMP_loop_foo_next. */
6185 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6186 build_fold_addr_expr (istart0
),
6187 build_fold_addr_expr (iend0
));
6191 tree t0
, t1
, t2
, t3
, t4
;
6192 /* If this is not a combined parallel loop, emit a call to
6193 GOMP_loop_foo_start in ENTRY_BB. */
6194 t4
= build_fold_addr_expr (iend0
);
6195 t3
= build_fold_addr_expr (istart0
);
6196 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
6199 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6201 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6202 OMP_CLAUSE__LOOPTEMP_
);
6203 gcc_assert (innerc
);
6204 t0
= OMP_CLAUSE_DECL (innerc
);
6205 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6206 OMP_CLAUSE__LOOPTEMP_
);
6207 gcc_assert (innerc
);
6208 t1
= OMP_CLAUSE_DECL (innerc
);
6210 if (POINTER_TYPE_P (TREE_TYPE (t0
))
6211 && TYPE_PRECISION (TREE_TYPE (t0
))
6212 != TYPE_PRECISION (fd
->iter_type
))
6214 /* Avoid casting pointers to integer of a different size. */
6215 tree itype
= signed_type_for (type
);
6216 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
6217 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
6221 t1
= fold_convert (fd
->iter_type
, t1
);
6222 t0
= fold_convert (fd
->iter_type
, t0
);
6226 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
6227 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
6229 if (fd
->iter_type
== long_integer_type_node
)
6233 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6234 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6235 6, t0
, t1
, t2
, t
, t3
, t4
);
6238 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6239 5, t0
, t1
, t2
, t3
, t4
);
6247 /* The GOMP_loop_ull_*start functions have additional boolean
6248 argument, true for < loops and false for > loops.
6249 In Fortran, the C bool type can be different from
6250 boolean_type_node. */
6251 bfn_decl
= builtin_decl_explicit (start_fn
);
6252 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
6253 t5
= build_int_cst (c_bool_type
,
6254 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
6257 tree bfn_decl
= builtin_decl_explicit (start_fn
);
6258 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6259 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
6262 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6263 6, t5
, t0
, t1
, t2
, t3
, t4
);
6266 if (TREE_TYPE (t
) != boolean_type_node
)
6267 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6268 t
, build_int_cst (TREE_TYPE (t
), 0));
6269 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6270 true, GSI_SAME_STMT
);
6271 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6273 /* Remove the GIMPLE_OMP_FOR statement. */
6274 gsi_remove (&gsi
, true);
6276 /* Iteration setup for sequential loop goes in L0_BB. */
6277 tree startvar
= fd
->loop
.v
;
6278 tree endvar
= NULL_TREE
;
6280 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6282 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
6283 && gimple_omp_for_kind (inner_stmt
)
6284 == GF_OMP_FOR_KIND_SIMD
);
6285 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
6286 OMP_CLAUSE__LOOPTEMP_
);
6287 gcc_assert (innerc
);
6288 startvar
= OMP_CLAUSE_DECL (innerc
);
6289 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6290 OMP_CLAUSE__LOOPTEMP_
);
6291 gcc_assert (innerc
);
6292 endvar
= OMP_CLAUSE_DECL (innerc
);
6295 gsi
= gsi_start_bb (l0_bb
);
6298 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6299 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6300 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6301 t
= fold_convert (TREE_TYPE (startvar
), t
);
6302 t
= force_gimple_operand_gsi (&gsi
, t
,
6304 && TREE_ADDRESSABLE (startvar
),
6305 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6306 assign_stmt
= gimple_build_assign (startvar
, t
);
6307 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6311 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6312 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6313 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6314 t
= fold_convert (TREE_TYPE (startvar
), t
);
6315 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6316 false, GSI_CONTINUE_LINKING
);
6319 assign_stmt
= gimple_build_assign (endvar
, iend
);
6320 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6321 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
6322 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
6324 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
6325 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6327 if (fd
->collapse
> 1)
6328 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6332 /* Code to control the increment and predicate for the sequential
6333 loop goes in the CONT_BB. */
6334 gsi
= gsi_last_bb (cont_bb
);
6335 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6336 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6337 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6338 vback
= gimple_omp_continue_control_def (cont_stmt
);
6340 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6342 if (POINTER_TYPE_P (type
))
6343 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
6345 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
6346 t
= force_gimple_operand_gsi (&gsi
, t
,
6348 && TREE_ADDRESSABLE (vback
),
6349 NULL_TREE
, true, GSI_SAME_STMT
);
6350 assign_stmt
= gimple_build_assign (vback
, t
);
6351 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6353 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6354 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
6356 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6357 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6360 /* Remove GIMPLE_OMP_CONTINUE. */
6361 gsi_remove (&gsi
, true);
6363 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6364 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
6366 /* Emit code to get the next parallel iteration in L2_BB. */
6367 gsi
= gsi_start_bb (l2_bb
);
6369 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6370 build_fold_addr_expr (istart0
),
6371 build_fold_addr_expr (iend0
));
6372 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6373 false, GSI_CONTINUE_LINKING
);
6374 if (TREE_TYPE (t
) != boolean_type_node
)
6375 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6376 t
, build_int_cst (TREE_TYPE (t
), 0));
6377 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6378 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
6381 /* Add the loop cleanup function. */
6382 gsi
= gsi_last_bb (exit_bb
);
6383 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6384 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
6385 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6386 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
6388 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
6389 gcall
*call_stmt
= gimple_build_call (t
, 0);
6390 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6391 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
6392 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
6393 gsi_remove (&gsi
, true);
6395 /* Connect the new blocks. */
6396 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
6397 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
6403 e
= find_edge (cont_bb
, l3_bb
);
6404 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
6406 phis
= phi_nodes (l3_bb
);
6407 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6409 gimple phi
= gsi_stmt (gsi
);
6410 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
6411 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
6415 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
6416 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
6417 e
= find_edge (cont_bb
, l1_bb
);
6418 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6423 else if (fd
->collapse
> 1)
6426 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6429 e
->flags
= EDGE_TRUE_VALUE
;
6432 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6433 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
6437 e
= find_edge (cont_bb
, l2_bb
);
6438 e
->flags
= EDGE_FALLTHRU
;
6440 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6442 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
6443 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
6444 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
6445 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
6446 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
6447 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
6448 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
6449 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
6451 struct loop
*outer_loop
= alloc_loop ();
6452 outer_loop
->header
= l0_bb
;
6453 outer_loop
->latch
= l2_bb
;
6454 add_loop (outer_loop
, l0_bb
->loop_father
);
6456 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6458 struct loop
*loop
= alloc_loop ();
6459 loop
->header
= l1_bb
;
6460 /* The loop may have multiple latches. */
6461 add_loop (loop
, outer_loop
);
6467 /* A subroutine of expand_omp_for. Generate code for a parallel
6468 loop with static schedule and no specified chunk size. Given
6471 for (V = N1; V cond N2; V += STEP) BODY;
6473 where COND is "<" or ">", we generate pseudocode
6475 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6480 if ((__typeof (V)) -1 > 0 && cond is >)
6481 n = -(adj + N2 - N1) / -STEP;
6483 n = (adj + N2 - N1) / STEP;
6486 if (threadid < tt) goto L3; else goto L4;
6491 s0 = q * threadid + tt;
6494 if (s0 >= e0) goto L2; else goto L0;
6500 if (V cond e) goto L1;
6505 expand_omp_for_static_nochunk (struct omp_region
*region
,
6506 struct omp_for_data
*fd
,
6509 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6510 tree type
, itype
, vmain
, vback
;
6511 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6512 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6514 gimple_stmt_iterator gsi
;
6516 bool broken_loop
= region
->cont
== NULL
;
6517 tree
*counts
= NULL
;
6520 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6521 != GF_OMP_FOR_KIND_OACC_LOOP
)
6524 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6525 if (POINTER_TYPE_P (type
))
6526 itype
= signed_type_for (type
);
6528 entry_bb
= region
->entry
;
6529 cont_bb
= region
->cont
;
6530 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6531 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6532 gcc_assert (broken_loop
6533 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6534 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6535 body_bb
= single_succ (seq_start_bb
);
6538 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6539 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6541 exit_bb
= region
->exit
;
6543 /* Iteration space partitioning goes in ENTRY_BB. */
6544 gsi
= gsi_last_bb (entry_bb
);
6545 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6547 if (fd
->collapse
> 1)
6549 int first_zero_iter
= -1;
6550 basic_block l2_dom_bb
= NULL
;
6552 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6553 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6554 fin_bb
, first_zero_iter
,
6558 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6559 t
= integer_one_node
;
6561 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6562 fold_convert (type
, fd
->loop
.n1
),
6563 fold_convert (type
, fd
->loop
.n2
));
6564 if (fd
->collapse
== 1
6565 && TYPE_UNSIGNED (type
)
6566 && (t
== NULL_TREE
|| !integer_onep (t
)))
6568 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6569 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6570 true, GSI_SAME_STMT
);
6571 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6572 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6573 true, GSI_SAME_STMT
);
6574 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6575 NULL_TREE
, NULL_TREE
);
6576 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6577 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6578 expand_omp_regimplify_p
, NULL
, NULL
)
6579 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6580 expand_omp_regimplify_p
, NULL
, NULL
))
6582 gsi
= gsi_for_stmt (cond_stmt
);
6583 gimple_regimplify_operands (cond_stmt
, &gsi
);
6585 ep
= split_block (entry_bb
, cond_stmt
);
6586 ep
->flags
= EDGE_TRUE_VALUE
;
6587 entry_bb
= ep
->dest
;
6588 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6589 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6590 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6591 if (gimple_in_ssa_p (cfun
))
6593 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6594 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6595 !gsi_end_p (gpi
); gsi_next (&gpi
))
6597 gphi
*phi
= gpi
.phi ();
6598 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6599 ep
, UNKNOWN_LOCATION
);
6602 gsi
= gsi_last_bb (entry_bb
);
6605 switch (gimple_omp_for_kind (fd
->for_stmt
))
6607 case GF_OMP_FOR_KIND_FOR
:
6608 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6609 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6611 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6612 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6613 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6615 case GF_OMP_FOR_KIND_OACC_LOOP
:
6616 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6617 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
6622 nthreads
= build_call_expr (nthreads
, 0);
6623 nthreads
= fold_convert (itype
, nthreads
);
6624 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
6625 true, GSI_SAME_STMT
);
6626 threadid
= build_call_expr (threadid
, 0);
6627 threadid
= fold_convert (itype
, threadid
);
6628 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
6629 true, GSI_SAME_STMT
);
6633 step
= fd
->loop
.step
;
6634 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6636 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6637 OMP_CLAUSE__LOOPTEMP_
);
6638 gcc_assert (innerc
);
6639 n1
= OMP_CLAUSE_DECL (innerc
);
6640 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6641 OMP_CLAUSE__LOOPTEMP_
);
6642 gcc_assert (innerc
);
6643 n2
= OMP_CLAUSE_DECL (innerc
);
6645 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6646 true, NULL_TREE
, true, GSI_SAME_STMT
);
6647 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6648 true, NULL_TREE
, true, GSI_SAME_STMT
);
6649 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6650 true, NULL_TREE
, true, GSI_SAME_STMT
);
6652 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6653 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6654 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6655 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6656 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6657 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6658 fold_build1 (NEGATE_EXPR
, itype
, t
),
6659 fold_build1 (NEGATE_EXPR
, itype
, step
));
6661 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6662 t
= fold_convert (itype
, t
);
6663 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6665 q
= create_tmp_reg (itype
, "q");
6666 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6667 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6668 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6670 tt
= create_tmp_reg (itype
, "tt");
6671 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6672 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6673 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6675 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6676 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6677 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6679 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
6680 gsi
= gsi_last_bb (second_bb
);
6681 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6683 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6685 gassign
*assign_stmt
6686 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
6687 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6689 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
6690 gsi
= gsi_last_bb (third_bb
);
6691 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6693 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6694 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6695 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6697 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6698 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6700 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6701 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6703 /* Remove the GIMPLE_OMP_FOR statement. */
6704 gsi_remove (&gsi
, true);
6706 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6707 gsi
= gsi_start_bb (seq_start_bb
);
6709 tree startvar
= fd
->loop
.v
;
6710 tree endvar
= NULL_TREE
;
6712 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6714 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6715 ? gimple_omp_parallel_clauses (inner_stmt
)
6716 : gimple_omp_for_clauses (inner_stmt
);
6717 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6718 gcc_assert (innerc
);
6719 startvar
= OMP_CLAUSE_DECL (innerc
);
6720 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6721 OMP_CLAUSE__LOOPTEMP_
);
6722 gcc_assert (innerc
);
6723 endvar
= OMP_CLAUSE_DECL (innerc
);
6725 t
= fold_convert (itype
, s0
);
6726 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6727 if (POINTER_TYPE_P (type
))
6728 t
= fold_build_pointer_plus (n1
, t
);
6730 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6731 t
= fold_convert (TREE_TYPE (startvar
), t
);
6732 t
= force_gimple_operand_gsi (&gsi
, t
,
6734 && TREE_ADDRESSABLE (startvar
),
6735 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6736 assign_stmt
= gimple_build_assign (startvar
, t
);
6737 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6739 t
= fold_convert (itype
, e0
);
6740 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6741 if (POINTER_TYPE_P (type
))
6742 t
= fold_build_pointer_plus (n1
, t
);
6744 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6745 t
= fold_convert (TREE_TYPE (startvar
), t
);
6746 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6747 false, GSI_CONTINUE_LINKING
);
6750 assign_stmt
= gimple_build_assign (endvar
, e
);
6751 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6752 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6753 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6755 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
6756 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6758 if (fd
->collapse
> 1)
6759 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6763 /* The code controlling the sequential loop replaces the
6764 GIMPLE_OMP_CONTINUE. */
6765 gsi
= gsi_last_bb (cont_bb
);
6766 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6767 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6768 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6769 vback
= gimple_omp_continue_control_def (cont_stmt
);
6771 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6773 if (POINTER_TYPE_P (type
))
6774 t
= fold_build_pointer_plus (vmain
, step
);
6776 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6777 t
= force_gimple_operand_gsi (&gsi
, t
,
6779 && TREE_ADDRESSABLE (vback
),
6780 NULL_TREE
, true, GSI_SAME_STMT
);
6781 assign_stmt
= gimple_build_assign (vback
, t
);
6782 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6784 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6785 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6787 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6790 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6791 gsi_remove (&gsi
, true);
6793 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6794 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6797 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6798 gsi
= gsi_last_bb (exit_bb
);
6799 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6801 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6802 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
6803 gcc_checking_assert (t
== NULL_TREE
);
6805 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6807 gsi_remove (&gsi
, true);
6809 /* Connect all the blocks. */
6810 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6811 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6812 ep
= find_edge (entry_bb
, second_bb
);
6813 ep
->flags
= EDGE_TRUE_VALUE
;
6814 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6815 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6816 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6820 ep
= find_edge (cont_bb
, body_bb
);
6821 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6826 else if (fd
->collapse
> 1)
6829 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6832 ep
->flags
= EDGE_TRUE_VALUE
;
6833 find_edge (cont_bb
, fin_bb
)->flags
6834 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6837 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6838 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6839 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6841 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6842 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6843 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6844 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6846 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6848 struct loop
*loop
= alloc_loop ();
6849 loop
->header
= body_bb
;
6850 if (collapse_bb
== NULL
)
6851 loop
->latch
= cont_bb
;
6852 add_loop (loop
, body_bb
->loop_father
);
6857 /* A subroutine of expand_omp_for. Generate code for a parallel
6858 loop with static schedule and a specified chunk size. Given
6861 for (V = N1; V cond N2; V += STEP) BODY;
6863 where COND is "<" or ">", we generate pseudocode
6865 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6870 if ((__typeof (V)) -1 > 0 && cond is >)
6871 n = -(adj + N2 - N1) / -STEP;
6873 n = (adj + N2 - N1) / STEP;
6875 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6876 here so that V is defined
6877 if the loop is not entered
6879 s0 = (trip * nthreads + threadid) * CHUNK;
6880 e0 = min(s0 + CHUNK, n);
6881 if (s0 < n) goto L1; else goto L4;
6888 if (V cond e) goto L2; else goto L3;
6896 expand_omp_for_static_chunk (struct omp_region
*region
,
6897 struct omp_for_data
*fd
, gimple inner_stmt
)
6899 tree n
, s0
, e0
, e
, t
;
6900 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6901 tree type
, itype
, vmain
, vback
, vextra
;
6902 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6903 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6904 gimple_stmt_iterator gsi
;
6906 bool broken_loop
= region
->cont
== NULL
;
6907 tree
*counts
= NULL
;
6910 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6911 != GF_OMP_FOR_KIND_OACC_LOOP
)
6914 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6915 if (POINTER_TYPE_P (type
))
6916 itype
= signed_type_for (type
);
6918 entry_bb
= region
->entry
;
6919 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6921 iter_part_bb
= se
->dest
;
6922 cont_bb
= region
->cont
;
6923 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6924 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6925 gcc_assert (broken_loop
6926 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6927 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6928 body_bb
= single_succ (seq_start_bb
);
6931 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6932 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6933 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6935 exit_bb
= region
->exit
;
6937 /* Trip and adjustment setup goes in ENTRY_BB. */
6938 gsi
= gsi_last_bb (entry_bb
);
6939 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6941 if (fd
->collapse
> 1)
6943 int first_zero_iter
= -1;
6944 basic_block l2_dom_bb
= NULL
;
6946 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6947 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6948 fin_bb
, first_zero_iter
,
6952 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6953 t
= integer_one_node
;
6955 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6956 fold_convert (type
, fd
->loop
.n1
),
6957 fold_convert (type
, fd
->loop
.n2
));
6958 if (fd
->collapse
== 1
6959 && TYPE_UNSIGNED (type
)
6960 && (t
== NULL_TREE
|| !integer_onep (t
)))
6962 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6963 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6964 true, GSI_SAME_STMT
);
6965 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6966 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6967 true, GSI_SAME_STMT
);
6968 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6969 NULL_TREE
, NULL_TREE
);
6970 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6971 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6972 expand_omp_regimplify_p
, NULL
, NULL
)
6973 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6974 expand_omp_regimplify_p
, NULL
, NULL
))
6976 gsi
= gsi_for_stmt (cond_stmt
);
6977 gimple_regimplify_operands (cond_stmt
, &gsi
);
6979 se
= split_block (entry_bb
, cond_stmt
);
6980 se
->flags
= EDGE_TRUE_VALUE
;
6981 entry_bb
= se
->dest
;
6982 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6983 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6984 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6985 if (gimple_in_ssa_p (cfun
))
6987 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6988 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6989 !gsi_end_p (gpi
); gsi_next (&gpi
))
6991 gphi
*phi
= gpi
.phi ();
6992 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6993 se
, UNKNOWN_LOCATION
);
6996 gsi
= gsi_last_bb (entry_bb
);
6999 switch (gimple_omp_for_kind (fd
->for_stmt
))
7001 case GF_OMP_FOR_KIND_FOR
:
7002 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7003 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7005 case GF_OMP_FOR_KIND_DISTRIBUTE
:
7006 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
7007 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
7009 case GF_OMP_FOR_KIND_OACC_LOOP
:
7010 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
7011 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
7016 nthreads
= build_call_expr (nthreads
, 0);
7017 nthreads
= fold_convert (itype
, nthreads
);
7018 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
7019 true, GSI_SAME_STMT
);
7020 threadid
= build_call_expr (threadid
, 0);
7021 threadid
= fold_convert (itype
, threadid
);
7022 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
7023 true, GSI_SAME_STMT
);
7027 step
= fd
->loop
.step
;
7028 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7030 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7031 OMP_CLAUSE__LOOPTEMP_
);
7032 gcc_assert (innerc
);
7033 n1
= OMP_CLAUSE_DECL (innerc
);
7034 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7035 OMP_CLAUSE__LOOPTEMP_
);
7036 gcc_assert (innerc
);
7037 n2
= OMP_CLAUSE_DECL (innerc
);
7039 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
7040 true, NULL_TREE
, true, GSI_SAME_STMT
);
7041 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
7042 true, NULL_TREE
, true, GSI_SAME_STMT
);
7043 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
7044 true, NULL_TREE
, true, GSI_SAME_STMT
);
7046 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
7047 true, NULL_TREE
, true, GSI_SAME_STMT
);
7049 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
7050 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
7051 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
7052 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
7053 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
7054 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7055 fold_build1 (NEGATE_EXPR
, itype
, t
),
7056 fold_build1 (NEGATE_EXPR
, itype
, step
));
7058 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
7059 t
= fold_convert (itype
, t
);
7060 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7061 true, GSI_SAME_STMT
);
7063 trip_var
= create_tmp_reg (itype
, ".trip");
7064 if (gimple_in_ssa_p (cfun
))
7066 trip_init
= make_ssa_name (trip_var
);
7067 trip_main
= make_ssa_name (trip_var
);
7068 trip_back
= make_ssa_name (trip_var
);
7072 trip_init
= trip_var
;
7073 trip_main
= trip_var
;
7074 trip_back
= trip_var
;
7077 gassign
*assign_stmt
7078 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
7079 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7081 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
7082 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7083 if (POINTER_TYPE_P (type
))
7084 t
= fold_build_pointer_plus (n1
, t
);
7086 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7087 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7088 true, GSI_SAME_STMT
);
7090 /* Remove the GIMPLE_OMP_FOR. */
7091 gsi_remove (&gsi
, true);
7093 /* Iteration space partitioning goes in ITER_PART_BB. */
7094 gsi
= gsi_last_bb (iter_part_bb
);
7096 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
7097 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
7098 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
7099 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7100 false, GSI_CONTINUE_LINKING
);
7102 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
7103 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
7104 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7105 false, GSI_CONTINUE_LINKING
);
7107 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
7108 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
7110 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7111 gsi
= gsi_start_bb (seq_start_bb
);
7113 tree startvar
= fd
->loop
.v
;
7114 tree endvar
= NULL_TREE
;
7116 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7118 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
7119 ? gimple_omp_parallel_clauses (inner_stmt
)
7120 : gimple_omp_for_clauses (inner_stmt
);
7121 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7122 gcc_assert (innerc
);
7123 startvar
= OMP_CLAUSE_DECL (innerc
);
7124 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7125 OMP_CLAUSE__LOOPTEMP_
);
7126 gcc_assert (innerc
);
7127 endvar
= OMP_CLAUSE_DECL (innerc
);
7130 t
= fold_convert (itype
, s0
);
7131 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7132 if (POINTER_TYPE_P (type
))
7133 t
= fold_build_pointer_plus (n1
, t
);
7135 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7136 t
= fold_convert (TREE_TYPE (startvar
), t
);
7137 t
= force_gimple_operand_gsi (&gsi
, t
,
7139 && TREE_ADDRESSABLE (startvar
),
7140 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7141 assign_stmt
= gimple_build_assign (startvar
, t
);
7142 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7144 t
= fold_convert (itype
, e0
);
7145 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7146 if (POINTER_TYPE_P (type
))
7147 t
= fold_build_pointer_plus (n1
, t
);
7149 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7150 t
= fold_convert (TREE_TYPE (startvar
), t
);
7151 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7152 false, GSI_CONTINUE_LINKING
);
7155 assign_stmt
= gimple_build_assign (endvar
, e
);
7156 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7157 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
7158 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
7160 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
7161 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7163 if (fd
->collapse
> 1)
7164 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
7168 /* The code controlling the sequential loop goes in CONT_BB,
7169 replacing the GIMPLE_OMP_CONTINUE. */
7170 gsi
= gsi_last_bb (cont_bb
);
7171 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
7172 vmain
= gimple_omp_continue_control_use (cont_stmt
);
7173 vback
= gimple_omp_continue_control_def (cont_stmt
);
7175 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7177 if (POINTER_TYPE_P (type
))
7178 t
= fold_build_pointer_plus (vmain
, step
);
7180 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
7181 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
7182 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7183 true, GSI_SAME_STMT
);
7184 assign_stmt
= gimple_build_assign (vback
, t
);
7185 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7187 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
7188 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
7190 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
7193 /* Remove GIMPLE_OMP_CONTINUE. */
7194 gsi_remove (&gsi
, true);
7196 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
7197 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
7199 /* Trip update code goes into TRIP_UPDATE_BB. */
7200 gsi
= gsi_start_bb (trip_update_bb
);
7202 t
= build_int_cst (itype
, 1);
7203 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
7204 assign_stmt
= gimple_build_assign (trip_back
, t
);
7205 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7208 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7209 gsi
= gsi_last_bb (exit_bb
);
7210 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
7212 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
7213 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
7214 gcc_checking_assert (t
== NULL_TREE
);
7216 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
7218 gsi_remove (&gsi
, true);
7220 /* Connect the new blocks. */
7221 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
7222 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
7226 se
= find_edge (cont_bb
, body_bb
);
7227 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7232 else if (fd
->collapse
> 1)
7235 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
7238 se
->flags
= EDGE_TRUE_VALUE
;
7239 find_edge (cont_bb
, trip_update_bb
)->flags
7240 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
7242 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
7245 if (gimple_in_ssa_p (cfun
))
7253 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
7255 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7256 remove arguments of the phi nodes in fin_bb. We need to create
7257 appropriate phi nodes in iter_part_bb instead. */
7258 se
= single_pred_edge (fin_bb
);
7259 re
= single_succ_edge (trip_update_bb
);
7260 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
7261 ene
= single_succ_edge (entry_bb
);
7263 psi
= gsi_start_phis (fin_bb
);
7264 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
7265 gsi_next (&psi
), ++i
)
7268 source_location locus
;
7271 t
= gimple_phi_result (phi
);
7272 gcc_assert (t
== redirect_edge_var_map_result (vm
));
7273 nphi
= create_phi_node (t
, iter_part_bb
);
7275 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
7276 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
7278 /* A special case -- fd->loop.v is not yet computed in
7279 iter_part_bb, we need to use vextra instead. */
7280 if (t
== fd
->loop
.v
)
7282 add_phi_arg (nphi
, t
, ene
, locus
);
7283 locus
= redirect_edge_var_map_location (vm
);
7284 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
7286 gcc_assert (gsi_end_p (psi
) && i
== head
->length ());
7287 redirect_edge_var_map_clear (re
);
7290 psi
= gsi_start_phis (fin_bb
);
7291 if (gsi_end_p (psi
))
7293 remove_phi_node (&psi
, false);
7296 /* Make phi node for trip. */
7297 phi
= create_phi_node (trip_main
, iter_part_bb
);
7298 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
7300 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
7305 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
7306 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
7307 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
7308 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
7309 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
7310 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
7311 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
7312 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
7313 recompute_dominator (CDI_DOMINATORS
, body_bb
));
7317 struct loop
*trip_loop
= alloc_loop ();
7318 trip_loop
->header
= iter_part_bb
;
7319 trip_loop
->latch
= trip_update_bb
;
7320 add_loop (trip_loop
, iter_part_bb
->loop_father
);
7322 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7324 struct loop
*loop
= alloc_loop ();
7325 loop
->header
= body_bb
;
7326 if (collapse_bb
== NULL
)
7327 loop
->latch
= cont_bb
;
7328 add_loop (loop
, trip_loop
);
7333 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7335 for (V = N1; V cond N2; V += STEP) BODY;
7337 where COND is "<" or ">" or "!=", we generate pseudocode
7339 for (ind_var = low; ind_var < high; ind_var++)
7341 V = n1 + (ind_var * STEP)
7346 In the above pseudocode, low and high are function parameters of the
7347 child function. In the function below, we are inserting a temp.
7348 variable that will be making a call to two OMP functions that will not be
7349 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7350 with _Cilk_for). These functions are replaced with low and high
7351 by the function that handles taskreg. */
7355 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
7357 bool broken_loop
= region
->cont
== NULL
;
7358 basic_block entry_bb
= region
->entry
;
7359 basic_block cont_bb
= region
->cont
;
7361 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7362 gcc_assert (broken_loop
7363 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7364 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7365 basic_block l1_bb
, l2_bb
;
7369 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7370 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7371 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7372 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7376 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7377 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7378 l2_bb
= single_succ (l1_bb
);
7380 basic_block exit_bb
= region
->exit
;
7381 basic_block l2_dom_bb
= NULL
;
7383 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
7385 /* Below statements until the "tree high_val = ..." are pseudo statements
7386 used to pass information to be used by expand_omp_taskreg.
7387 low_val and high_val will be replaced by the __low and __high
7388 parameter from the child function.
7390 The call_exprs part is a place-holder, it is mainly used
7391 to distinctly identify to the top-level part that this is
7392 where we should put low and high (reasoning given in header
7396 = gimple_omp_parallel_child_fn (
7397 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
7398 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
7399 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
7401 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
7403 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
7406 gcc_assert (low_val
&& high_val
);
7408 tree type
= TREE_TYPE (low_val
);
7409 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
7410 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7412 /* Not needed in SSA form right now. */
7413 gcc_assert (!gimple_in_ssa_p (cfun
));
7414 if (l2_dom_bb
== NULL
)
7420 gimple stmt
= gimple_build_assign (ind_var
, n1
);
7422 /* Replace the GIMPLE_OMP_FOR statement. */
7423 gsi_replace (&gsi
, stmt
, true);
7427 /* Code to control the increment goes in the CONT_BB. */
7428 gsi
= gsi_last_bb (cont_bb
);
7429 stmt
= gsi_stmt (gsi
);
7430 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7431 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
7432 build_one_cst (type
));
7434 /* Replace GIMPLE_OMP_CONTINUE. */
7435 gsi_replace (&gsi
, stmt
, true);
7438 /* Emit the condition in L1_BB. */
7439 gsi
= gsi_after_labels (l1_bb
);
7440 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
7441 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
7443 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
7444 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7445 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7447 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7448 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
7449 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
7450 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7452 /* The condition is always '<' since the runtime will fill in the low
7454 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
7455 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7457 /* Remove GIMPLE_OMP_RETURN. */
7458 gsi
= gsi_last_bb (exit_bb
);
7459 gsi_remove (&gsi
, true);
7461 /* Connect the new blocks. */
7462 remove_edge (FALLTHRU_EDGE (entry_bb
));
7467 remove_edge (BRANCH_EDGE (entry_bb
));
7468 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7470 e
= BRANCH_EDGE (l1_bb
);
7471 ne
= FALLTHRU_EDGE (l1_bb
);
7472 e
->flags
= EDGE_TRUE_VALUE
;
7476 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7478 ne
= single_succ_edge (l1_bb
);
7479 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7482 ne
->flags
= EDGE_FALSE_VALUE
;
7483 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7484 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7486 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7487 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7488 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7492 struct loop
*loop
= alloc_loop ();
7493 loop
->header
= l1_bb
;
7494 loop
->latch
= cont_bb
;
7495 add_loop (loop
, l1_bb
->loop_father
);
7496 loop
->safelen
= INT_MAX
;
7499 /* Pick the correct library function based on the precision of the
7500 induction variable type. */
7501 tree lib_fun
= NULL_TREE
;
7502 if (TYPE_PRECISION (type
) == 32)
7503 lib_fun
= cilk_for_32_fndecl
;
7504 else if (TYPE_PRECISION (type
) == 64)
7505 lib_fun
= cilk_for_64_fndecl
;
7509 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7511 /* WS_ARGS contains the library function flavor to call:
7512 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7513 user-defined grain value. If the user does not define one, then zero
7514 is passed in by the parser. */
7515 vec_alloc (region
->ws_args
, 2);
7516 region
->ws_args
->quick_push (lib_fun
);
7517 region
->ws_args
->quick_push (fd
->chunk_size
);
7520 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7521 loop. Given parameters:
7523 for (V = N1; V cond N2; V += STEP) BODY;
7525 where COND is "<" or ">", we generate pseudocode
7533 if (V cond N2) goto L0; else goto L2;
7536 For collapsed loops, given parameters:
7538 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7539 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7540 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7543 we generate pseudocode
7549 count3 = (adj + N32 - N31) / STEP3;
7554 count2 = (adj + N22 - N21) / STEP2;
7559 count1 = (adj + N12 - N11) / STEP1;
7560 count = count1 * count2 * count3;
7570 V2 += (V3 cond3 N32) ? 0 : STEP2;
7571 V3 = (V3 cond3 N32) ? V3 : N31;
7572 V1 += (V2 cond2 N22) ? 0 : STEP1;
7573 V2 = (V2 cond2 N22) ? V2 : N21;
7575 if (V < count) goto L0; else goto L2;
7581 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7584 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7585 gimple_stmt_iterator gsi
;
7588 bool broken_loop
= region
->cont
== NULL
;
7590 tree
*counts
= NULL
;
7592 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7593 OMP_CLAUSE_SAFELEN
);
7594 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7595 OMP_CLAUSE__SIMDUID_
);
7598 type
= TREE_TYPE (fd
->loop
.v
);
7599 entry_bb
= region
->entry
;
7600 cont_bb
= region
->cont
;
7601 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7602 gcc_assert (broken_loop
7603 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7604 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7607 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7608 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7609 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7610 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7614 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7615 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7616 l2_bb
= single_succ (l1_bb
);
7618 exit_bb
= region
->exit
;
7621 gsi
= gsi_last_bb (entry_bb
);
7623 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7624 /* Not needed in SSA form right now. */
7625 gcc_assert (!gimple_in_ssa_p (cfun
));
7626 if (fd
->collapse
> 1)
7628 int first_zero_iter
= -1;
7629 basic_block zero_iter_bb
= l2_bb
;
7631 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7632 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7633 zero_iter_bb
, first_zero_iter
,
7636 if (l2_dom_bb
== NULL
)
7641 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7643 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7644 OMP_CLAUSE__LOOPTEMP_
);
7645 gcc_assert (innerc
);
7646 n1
= OMP_CLAUSE_DECL (innerc
);
7647 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7648 OMP_CLAUSE__LOOPTEMP_
);
7649 gcc_assert (innerc
);
7650 n2
= OMP_CLAUSE_DECL (innerc
);
7651 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7652 fold_convert (type
, n1
));
7653 if (fd
->collapse
> 1)
7656 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7662 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7663 fold_convert (type
, fd
->loop
.n1
));
7664 if (fd
->collapse
> 1)
7665 for (i
= 0; i
< fd
->collapse
; i
++)
7667 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7668 if (POINTER_TYPE_P (itype
))
7669 itype
= signed_type_for (itype
);
7670 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7671 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7675 /* Remove the GIMPLE_OMP_FOR statement. */
7676 gsi_remove (&gsi
, true);
7680 /* Code to control the increment goes in the CONT_BB. */
7681 gsi
= gsi_last_bb (cont_bb
);
7682 stmt
= gsi_stmt (gsi
);
7683 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7685 if (POINTER_TYPE_P (type
))
7686 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7688 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7689 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7691 if (fd
->collapse
> 1)
7693 i
= fd
->collapse
- 1;
7694 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7696 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7697 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7701 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7703 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7706 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7708 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7710 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7711 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7712 if (POINTER_TYPE_P (itype2
))
7713 itype2
= signed_type_for (itype2
);
7714 t
= build3 (COND_EXPR
, itype2
,
7715 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7717 fold_convert (itype
, fd
->loops
[i
].n2
)),
7718 build_int_cst (itype2
, 0),
7719 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7720 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7721 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7723 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7724 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7726 t
= build3 (COND_EXPR
, itype
,
7727 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7729 fold_convert (itype
, fd
->loops
[i
].n2
)),
7731 fold_convert (itype
, fd
->loops
[i
].n1
));
7732 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7736 /* Remove GIMPLE_OMP_CONTINUE. */
7737 gsi_remove (&gsi
, true);
7740 /* Emit the condition in L1_BB. */
7741 gsi
= gsi_start_bb (l1_bb
);
7743 t
= fold_convert (type
, n2
);
7744 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7745 false, GSI_CONTINUE_LINKING
);
7746 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7747 cond_stmt
= gimple_build_cond_empty (t
);
7748 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
7749 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7751 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7754 gsi
= gsi_for_stmt (cond_stmt
);
7755 gimple_regimplify_operands (cond_stmt
, &gsi
);
7758 /* Remove GIMPLE_OMP_RETURN. */
7759 gsi
= gsi_last_bb (exit_bb
);
7760 gsi_remove (&gsi
, true);
7762 /* Connect the new blocks. */
7763 remove_edge (FALLTHRU_EDGE (entry_bb
));
7767 remove_edge (BRANCH_EDGE (entry_bb
));
7768 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7770 e
= BRANCH_EDGE (l1_bb
);
7771 ne
= FALLTHRU_EDGE (l1_bb
);
7772 e
->flags
= EDGE_TRUE_VALUE
;
7776 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7778 ne
= single_succ_edge (l1_bb
);
7779 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7782 ne
->flags
= EDGE_FALSE_VALUE
;
7783 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7784 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7786 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7787 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7788 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7792 struct loop
*loop
= alloc_loop ();
7793 loop
->header
= l1_bb
;
7794 loop
->latch
= cont_bb
;
7795 add_loop (loop
, l1_bb
->loop_father
);
7796 if (safelen
== NULL_TREE
)
7797 loop
->safelen
= INT_MAX
;
7800 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7801 if (TREE_CODE (safelen
) != INTEGER_CST
)
7803 else if (!tree_fits_uhwi_p (safelen
)
7804 || tree_to_uhwi (safelen
) > INT_MAX
)
7805 loop
->safelen
= INT_MAX
;
7807 loop
->safelen
= tree_to_uhwi (safelen
);
7808 if (loop
->safelen
== 1)
7813 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7814 cfun
->has_simduid_loops
= true;
7816 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7818 if ((flag_tree_loop_vectorize
7819 || (!global_options_set
.x_flag_tree_loop_vectorize
7820 && !global_options_set
.x_flag_tree_vectorize
))
7821 && flag_tree_loop_optimize
7822 && loop
->safelen
> 1)
7824 loop
->force_vectorize
= true;
7825 cfun
->has_force_vectorize_loops
= true;
7829 cfun
->has_simduid_loops
= true;
7833 /* Expand the OMP loop defined by REGION. */
7836 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7838 struct omp_for_data fd
;
7839 struct omp_for_data_loop
*loops
;
7842 = (struct omp_for_data_loop
*)
7843 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7844 * sizeof (struct omp_for_data_loop
));
7845 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
7847 region
->sched_kind
= fd
.sched_kind
;
7849 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7850 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7851 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7854 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7855 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7856 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7859 /* If there isn't a continue then this is a degerate case where
7860 the introduction of abnormal edges during lowering will prevent
7861 original loops from being detected. Fix that up. */
7862 loops_state_set (LOOPS_NEED_FIXUP
);
7864 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7865 expand_omp_simd (region
, &fd
);
7866 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7867 expand_cilk_for (region
, &fd
);
7868 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7869 && !fd
.have_ordered
)
7871 if (fd
.chunk_size
== NULL
)
7872 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7874 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7878 int fn_index
, start_ix
, next_ix
;
7880 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7881 == GF_OMP_FOR_KIND_FOR
);
7882 if (fd
.chunk_size
== NULL
7883 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7884 fd
.chunk_size
= integer_zero_node
;
7885 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7886 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7887 ? 3 : fd
.sched_kind
;
7888 fn_index
+= fd
.have_ordered
* 4;
7889 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7890 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7891 if (fd
.iter_type
== long_long_unsigned_type_node
)
7893 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7894 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7895 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7896 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7898 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7899 (enum built_in_function
) next_ix
, inner_stmt
);
7902 if (gimple_in_ssa_p (cfun
))
7903 update_ssa (TODO_update_ssa_only_virtuals
);
7907 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7909 v = GOMP_sections_start (n);
7926 v = GOMP_sections_next ();
7931 If this is a combined parallel sections, replace the call to
7932 GOMP_sections_start with call to GOMP_sections_next. */
7935 expand_omp_sections (struct omp_region
*region
)
7937 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7939 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7940 gimple_stmt_iterator si
, switch_si
;
7941 gomp_sections
*sections_stmt
;
7943 gomp_continue
*cont
;
7946 struct omp_region
*inner
;
7948 bool exit_reachable
= region
->cont
!= NULL
;
7950 gcc_assert (region
->exit
!= NULL
);
7951 entry_bb
= region
->entry
;
7952 l0_bb
= single_succ (entry_bb
);
7953 l1_bb
= region
->cont
;
7954 l2_bb
= region
->exit
;
7955 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7956 l2
= gimple_block_label (l2_bb
);
7959 /* This can happen if there are reductions. */
7960 len
= EDGE_COUNT (l0_bb
->succs
);
7961 gcc_assert (len
> 0);
7962 e
= EDGE_SUCC (l0_bb
, len
- 1);
7963 si
= gsi_last_bb (e
->dest
);
7966 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7967 l2
= gimple_block_label (e
->dest
);
7969 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7971 si
= gsi_last_bb (e
->dest
);
7973 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7975 l2
= gimple_block_label (e
->dest
);
7981 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7983 default_bb
= create_empty_bb (l0_bb
);
7985 /* We will build a switch() with enough cases for all the
7986 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7987 and a default case to abort if something goes wrong. */
7988 len
= EDGE_COUNT (l0_bb
->succs
);
7990 /* Use vec::quick_push on label_vec throughout, since we know the size
7992 auto_vec
<tree
> label_vec (len
);
7994 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7995 GIMPLE_OMP_SECTIONS statement. */
7996 si
= gsi_last_bb (entry_bb
);
7997 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
7998 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7999 vin
= gimple_omp_sections_control (sections_stmt
);
8000 if (!is_combined_parallel (region
))
8002 /* If we are not inside a combined parallel+sections region,
8003 call GOMP_sections_start. */
8004 t
= build_int_cst (unsigned_type_node
, len
- 1);
8005 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
8006 stmt
= gimple_build_call (u
, 1, t
);
8010 /* Otherwise, call GOMP_sections_next. */
8011 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8012 stmt
= gimple_build_call (u
, 0);
8014 gimple_call_set_lhs (stmt
, vin
);
8015 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8016 gsi_remove (&si
, true);
8018 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8020 switch_si
= gsi_last_bb (l0_bb
);
8021 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
8024 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
8025 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
8026 vmain
= gimple_omp_continue_control_use (cont
);
8027 vnext
= gimple_omp_continue_control_def (cont
);
8035 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
8036 label_vec
.quick_push (t
);
8039 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8040 for (inner
= region
->inner
, casei
= 1;
8042 inner
= inner
->next
, i
++, casei
++)
8044 basic_block s_entry_bb
, s_exit_bb
;
8046 /* Skip optional reduction region. */
8047 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
8054 s_entry_bb
= inner
->entry
;
8055 s_exit_bb
= inner
->exit
;
8057 t
= gimple_block_label (s_entry_bb
);
8058 u
= build_int_cst (unsigned_type_node
, casei
);
8059 u
= build_case_label (u
, NULL
, t
);
8060 label_vec
.quick_push (u
);
8062 si
= gsi_last_bb (s_entry_bb
);
8063 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
8064 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
8065 gsi_remove (&si
, true);
8066 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
8068 if (s_exit_bb
== NULL
)
8071 si
= gsi_last_bb (s_exit_bb
);
8072 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8073 gsi_remove (&si
, true);
8075 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
8078 /* Error handling code goes in DEFAULT_BB. */
8079 t
= gimple_block_label (default_bb
);
8080 u
= build_case_label (NULL
, NULL
, t
);
8081 make_edge (l0_bb
, default_bb
, 0);
8082 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
8084 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
8085 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
8086 gsi_remove (&switch_si
, true);
8088 si
= gsi_start_bb (default_bb
);
8089 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
8090 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
8096 /* Code to get the next section goes in L1_BB. */
8097 si
= gsi_last_bb (l1_bb
);
8098 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
8100 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8101 stmt
= gimple_build_call (bfn_decl
, 0);
8102 gimple_call_set_lhs (stmt
, vnext
);
8103 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8104 gsi_remove (&si
, true);
8106 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
8109 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8110 si
= gsi_last_bb (l2_bb
);
8111 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
8112 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
8113 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
8114 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
8116 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
8117 stmt
= gimple_build_call (t
, 0);
8118 if (gimple_omp_return_lhs (gsi_stmt (si
)))
8119 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
8120 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8121 gsi_remove (&si
, true);
8123 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
8127 /* Expand code for an OpenMP single directive. We've already expanded
8128 much of the code, here we simply place the GOMP_barrier call. */
8131 expand_omp_single (struct omp_region
*region
)
8133 basic_block entry_bb
, exit_bb
;
8134 gimple_stmt_iterator si
;
8136 entry_bb
= region
->entry
;
8137 exit_bb
= region
->exit
;
8139 si
= gsi_last_bb (entry_bb
);
8140 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
8141 gsi_remove (&si
, true);
8142 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8144 si
= gsi_last_bb (exit_bb
);
8145 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
8147 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
8148 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
8150 gsi_remove (&si
, true);
8151 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8155 /* Generic expansion for OpenMP synchronization directives: master,
8156 ordered and critical. All we need to do here is remove the entry
8157 and exit markers for REGION. */
8160 expand_omp_synch (struct omp_region
*region
)
8162 basic_block entry_bb
, exit_bb
;
8163 gimple_stmt_iterator si
;
8165 entry_bb
= region
->entry
;
8166 exit_bb
= region
->exit
;
8168 si
= gsi_last_bb (entry_bb
);
8169 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
8170 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
8171 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
8172 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
8173 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
8174 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
8175 gsi_remove (&si
, true);
8176 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8180 si
= gsi_last_bb (exit_bb
);
8181 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8182 gsi_remove (&si
, true);
8183 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8187 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8188 operation as a normal volatile load. */
8191 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
8192 tree loaded_val
, int index
)
8194 enum built_in_function tmpbase
;
8195 gimple_stmt_iterator gsi
;
8196 basic_block store_bb
;
8199 tree decl
, call
, type
, itype
;
8201 gsi
= gsi_last_bb (load_bb
);
8202 stmt
= gsi_stmt (gsi
);
8203 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8204 loc
= gimple_location (stmt
);
8206 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8207 is smaller than word size, then expand_atomic_load assumes that the load
8208 is atomic. We could avoid the builtin entirely in this case. */
8210 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8211 decl
= builtin_decl_explicit (tmpbase
);
8212 if (decl
== NULL_TREE
)
8215 type
= TREE_TYPE (loaded_val
);
8216 itype
= TREE_TYPE (TREE_TYPE (decl
));
8218 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
8219 build_int_cst (NULL
,
8220 gimple_omp_atomic_seq_cst_p (stmt
)
8222 : MEMMODEL_RELAXED
));
8223 if (!useless_type_conversion_p (type
, itype
))
8224 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8225 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8227 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8228 gsi_remove (&gsi
, true);
8230 store_bb
= single_succ (load_bb
);
8231 gsi
= gsi_last_bb (store_bb
);
8232 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8233 gsi_remove (&gsi
, true);
8235 if (gimple_in_ssa_p (cfun
))
8236 update_ssa (TODO_update_ssa_no_phi
);
8241 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8242 operation as a normal volatile store. */
8245 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
8246 tree loaded_val
, tree stored_val
, int index
)
8248 enum built_in_function tmpbase
;
8249 gimple_stmt_iterator gsi
;
8250 basic_block store_bb
= single_succ (load_bb
);
8253 tree decl
, call
, type
, itype
;
8257 gsi
= gsi_last_bb (load_bb
);
8258 stmt
= gsi_stmt (gsi
);
8259 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8261 /* If the load value is needed, then this isn't a store but an exchange. */
8262 exchange
= gimple_omp_atomic_need_value_p (stmt
);
8264 gsi
= gsi_last_bb (store_bb
);
8265 stmt
= gsi_stmt (gsi
);
8266 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
8267 loc
= gimple_location (stmt
);
8269 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8270 is smaller than word size, then expand_atomic_store assumes that the store
8271 is atomic. We could avoid the builtin entirely in this case. */
8273 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
8274 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
8275 decl
= builtin_decl_explicit (tmpbase
);
8276 if (decl
== NULL_TREE
)
8279 type
= TREE_TYPE (stored_val
);
8281 /* Dig out the type of the function's second argument. */
8282 itype
= TREE_TYPE (decl
);
8283 itype
= TYPE_ARG_TYPES (itype
);
8284 itype
= TREE_CHAIN (itype
);
8285 itype
= TREE_VALUE (itype
);
8286 imode
= TYPE_MODE (itype
);
8288 if (exchange
&& !can_atomic_exchange_p (imode
, true))
8291 if (!useless_type_conversion_p (itype
, type
))
8292 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
8293 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
8294 build_int_cst (NULL
,
8295 gimple_omp_atomic_seq_cst_p (stmt
)
8297 : MEMMODEL_RELAXED
));
8300 if (!useless_type_conversion_p (type
, itype
))
8301 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8302 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8305 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8306 gsi_remove (&gsi
, true);
8308 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8309 gsi
= gsi_last_bb (load_bb
);
8310 gsi_remove (&gsi
, true);
8312 if (gimple_in_ssa_p (cfun
))
8313 update_ssa (TODO_update_ssa_no_phi
);
8318 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8319 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8320 size of the data type, and thus usable to find the index of the builtin
8321 decl. Returns false if the expression is not of the proper form. */
8324 expand_omp_atomic_fetch_op (basic_block load_bb
,
8325 tree addr
, tree loaded_val
,
8326 tree stored_val
, int index
)
8328 enum built_in_function oldbase
, newbase
, tmpbase
;
8329 tree decl
, itype
, call
;
8331 basic_block store_bb
= single_succ (load_bb
);
8332 gimple_stmt_iterator gsi
;
8335 enum tree_code code
;
8336 bool need_old
, need_new
;
8340 /* We expect to find the following sequences:
8343 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8346 val = tmp OP something; (or: something OP tmp)
8347 GIMPLE_OMP_STORE (val)
8349 ???FIXME: Allow a more flexible sequence.
8350 Perhaps use data flow to pick the statements.
8354 gsi
= gsi_after_labels (store_bb
);
8355 stmt
= gsi_stmt (gsi
);
8356 loc
= gimple_location (stmt
);
8357 if (!is_gimple_assign (stmt
))
8360 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
8362 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
8363 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
8364 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
8365 gcc_checking_assert (!need_old
|| !need_new
);
8367 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
8370 /* Check for one of the supported fetch-op operations. */
8371 code
= gimple_assign_rhs_code (stmt
);
8375 case POINTER_PLUS_EXPR
:
8376 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
8377 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
8380 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
8381 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
8384 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
8385 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
8388 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
8389 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
8392 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
8393 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
8399 /* Make sure the expression is of the proper form. */
8400 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
8401 rhs
= gimple_assign_rhs2 (stmt
);
8402 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
8403 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
8404 rhs
= gimple_assign_rhs1 (stmt
);
8408 tmpbase
= ((enum built_in_function
)
8409 ((need_new
? newbase
: oldbase
) + index
+ 1));
8410 decl
= builtin_decl_explicit (tmpbase
);
8411 if (decl
== NULL_TREE
)
8413 itype
= TREE_TYPE (TREE_TYPE (decl
));
8414 imode
= TYPE_MODE (itype
);
8416 /* We could test all of the various optabs involved, but the fact of the
8417 matter is that (with the exception of i486 vs i586 and xadd) all targets
8418 that support any atomic operaton optab also implements compare-and-swap.
8419 Let optabs.c take care of expanding any compare-and-swap loop. */
8420 if (!can_compare_and_swap_p (imode
, true))
8423 gsi
= gsi_last_bb (load_bb
);
8424 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8426 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8427 It only requires that the operation happen atomically. Thus we can
8428 use the RELAXED memory model. */
8429 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
8430 fold_convert_loc (loc
, itype
, rhs
),
8431 build_int_cst (NULL
,
8432 seq_cst
? MEMMODEL_SEQ_CST
8433 : MEMMODEL_RELAXED
));
8435 if (need_old
|| need_new
)
8437 lhs
= need_old
? loaded_val
: stored_val
;
8438 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
8439 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
8442 call
= fold_convert_loc (loc
, void_type_node
, call
);
8443 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8444 gsi_remove (&gsi
, true);
8446 gsi
= gsi_last_bb (store_bb
);
8447 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8448 gsi_remove (&gsi
, true);
8449 gsi
= gsi_last_bb (store_bb
);
8450 gsi_remove (&gsi
, true);
8452 if (gimple_in_ssa_p (cfun
))
8453 update_ssa (TODO_update_ssa_no_phi
);
8458 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8462 newval = rhs; // with oldval replacing *addr in rhs
8463 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8464 if (oldval != newval)
8467 INDEX is log2 of the size of the data type, and thus usable to find the
8468 index of the builtin decl. */
8471 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
8472 tree addr
, tree loaded_val
, tree stored_val
,
8475 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
8476 tree type
, itype
, cmpxchg
, iaddr
;
8477 gimple_stmt_iterator si
;
8478 basic_block loop_header
= single_succ (load_bb
);
8481 enum built_in_function fncode
;
8483 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8484 order to use the RELAXED memory model effectively. */
8485 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8487 cmpxchg
= builtin_decl_explicit (fncode
);
8488 if (cmpxchg
== NULL_TREE
)
8490 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8491 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
8493 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
8496 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8497 si
= gsi_last_bb (load_bb
);
8498 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8500 /* For floating-point values, we'll need to view-convert them to integers
8501 so that we can perform the atomic compare and swap. Simplify the
8502 following code by always setting up the "i"ntegral variables. */
8503 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
8507 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
8510 = force_gimple_operand_gsi (&si
,
8511 fold_convert (TREE_TYPE (iaddr
), addr
),
8512 false, NULL_TREE
, true, GSI_SAME_STMT
);
8513 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
8514 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8515 loadedi
= create_tmp_var (itype
);
8516 if (gimple_in_ssa_p (cfun
))
8517 loadedi
= make_ssa_name (loadedi
);
8522 loadedi
= loaded_val
;
8525 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8526 tree loaddecl
= builtin_decl_explicit (fncode
);
8529 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8530 build_call_expr (loaddecl
, 2, iaddr
,
8531 build_int_cst (NULL_TREE
,
8532 MEMMODEL_RELAXED
)));
8534 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8535 build_int_cst (TREE_TYPE (iaddr
), 0));
8538 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8541 /* Move the value to the LOADEDI temporary. */
8542 if (gimple_in_ssa_p (cfun
))
8544 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8545 phi
= create_phi_node (loadedi
, loop_header
);
8546 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8550 gsi_insert_before (&si
,
8551 gimple_build_assign (loadedi
, initial
),
8553 if (loadedi
!= loaded_val
)
8555 gimple_stmt_iterator gsi2
;
8558 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8559 gsi2
= gsi_start_bb (loop_header
);
8560 if (gimple_in_ssa_p (cfun
))
8563 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8564 true, GSI_SAME_STMT
);
8565 stmt
= gimple_build_assign (loaded_val
, x
);
8566 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8570 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8571 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8572 true, GSI_SAME_STMT
);
8575 gsi_remove (&si
, true);
8577 si
= gsi_last_bb (store_bb
);
8578 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8581 storedi
= stored_val
;
8584 force_gimple_operand_gsi (&si
,
8585 build1 (VIEW_CONVERT_EXPR
, itype
,
8586 stored_val
), true, NULL_TREE
, true,
8589 /* Build the compare&swap statement. */
8590 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8591 new_storedi
= force_gimple_operand_gsi (&si
,
8592 fold_convert (TREE_TYPE (loadedi
),
8595 true, GSI_SAME_STMT
);
8597 if (gimple_in_ssa_p (cfun
))
8601 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
8602 stmt
= gimple_build_assign (old_vali
, loadedi
);
8603 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8605 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8606 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8609 /* Note that we always perform the comparison as an integer, even for
8610 floating point. This allows the atomic operation to properly
8611 succeed even with NaNs and -0.0. */
8612 stmt
= gimple_build_cond_empty
8613 (build2 (NE_EXPR
, boolean_type_node
,
8614 new_storedi
, old_vali
));
8615 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8618 e
= single_succ_edge (store_bb
);
8619 e
->flags
&= ~EDGE_FALLTHRU
;
8620 e
->flags
|= EDGE_FALSE_VALUE
;
8622 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8624 /* Copy the new value to loadedi (we already did that before the condition
8625 if we are not in SSA). */
8626 if (gimple_in_ssa_p (cfun
))
8628 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8629 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8632 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8633 gsi_remove (&si
, true);
8635 struct loop
*loop
= alloc_loop ();
8636 loop
->header
= loop_header
;
8637 loop
->latch
= store_bb
;
8638 add_loop (loop
, loop_header
->loop_father
);
8640 if (gimple_in_ssa_p (cfun
))
8641 update_ssa (TODO_update_ssa_no_phi
);
8646 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8648 GOMP_atomic_start ();
8652 The result is not globally atomic, but works so long as all parallel
8653 references are within #pragma omp atomic directives. According to
8654 responses received from omp@openmp.org, appears to be within spec.
8655 Which makes sense, since that's how several other compilers handle
8656 this situation as well.
8657 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8658 expanding. STORED_VAL is the operand of the matching
8659 GIMPLE_OMP_ATOMIC_STORE.
8662 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8666 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8671 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8672 tree addr
, tree loaded_val
, tree stored_val
)
8674 gimple_stmt_iterator si
;
8678 si
= gsi_last_bb (load_bb
);
8679 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8681 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8682 t
= build_call_expr (t
, 0);
8683 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8685 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8686 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8687 gsi_remove (&si
, true);
8689 si
= gsi_last_bb (store_bb
);
8690 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8692 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8694 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8696 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8697 t
= build_call_expr (t
, 0);
8698 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8699 gsi_remove (&si
, true);
8701 if (gimple_in_ssa_p (cfun
))
8702 update_ssa (TODO_update_ssa_no_phi
);
8706 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8707 using expand_omp_atomic_fetch_op. If it failed, we try to
8708 call expand_omp_atomic_pipeline, and if it fails too, the
8709 ultimate fallback is wrapping the operation in a mutex
8710 (expand_omp_atomic_mutex). REGION is the atomic region built
8711 by build_omp_regions_1(). */
8714 expand_omp_atomic (struct omp_region
*region
)
8716 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8717 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
8718 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
8719 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8720 tree addr
= gimple_omp_atomic_load_rhs (load
);
8721 tree stored_val
= gimple_omp_atomic_store_val (store
);
8722 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8723 HOST_WIDE_INT index
;
8725 /* Make sure the type is one of the supported sizes. */
8726 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8727 index
= exact_log2 (index
);
8728 if (index
>= 0 && index
<= 4)
8730 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8732 /* __sync builtins require strict data alignment. */
8733 if (exact_log2 (align
) >= index
)
8736 if (loaded_val
== stored_val
8737 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8738 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8739 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8740 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8744 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8745 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8746 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8747 && store_bb
== single_succ (load_bb
)
8748 && first_stmt (store_bb
) == store
8749 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8753 /* When possible, use specialized atomic update functions. */
8754 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8755 && store_bb
== single_succ (load_bb
)
8756 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8757 loaded_val
, stored_val
, index
))
8760 /* If we don't have specialized __sync builtins, try and implement
8761 as a compare and swap loop. */
8762 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8763 loaded_val
, stored_val
, index
))
8768 /* The ultimate fallback is wrapping the operation in a mutex. */
8769 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8773 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8776 expand_omp_target (struct omp_region
*region
)
8778 basic_block entry_bb
, exit_bb
, new_bb
;
8779 struct function
*child_cfun
;
8780 tree child_fn
, block
, t
;
8781 gimple_stmt_iterator gsi
;
8782 gomp_target
*entry_stmt
;
8785 bool offloaded
, data_region
;
8787 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
8788 new_bb
= region
->entry
;
8790 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
8791 switch (gimple_omp_target_kind (entry_stmt
))
8793 case GF_OMP_TARGET_KIND_REGION
:
8794 case GF_OMP_TARGET_KIND_UPDATE
:
8795 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8796 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8797 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8798 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8799 data_region
= false;
8801 case GF_OMP_TARGET_KIND_DATA
:
8802 case GF_OMP_TARGET_KIND_OACC_DATA
:
8809 child_fn
= NULL_TREE
;
8813 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8814 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8817 /* Supported by expand_omp_taskreg, but not here. */
8818 if (child_cfun
!= NULL
)
8819 gcc_checking_assert (!child_cfun
->cfg
);
8820 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
8822 entry_bb
= region
->entry
;
8823 exit_bb
= region
->exit
;
8827 unsigned srcidx
, dstidx
, num
;
8829 /* If the offloading region needs data sent from the parent
8830 function, then the very first statement (except possible
8831 tree profile counter updates) of the offloading body
8832 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8833 &.OMP_DATA_O is passed as an argument to the child function,
8834 we need to replace it with the argument as seen by the child
8837 In most cases, this will end up being the identity assignment
8838 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8839 a function call that has been inlined, the original PARM_DECL
8840 .OMP_DATA_I may have been converted into a different local
8841 variable. In which case, we need to keep the assignment. */
8842 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
8845 basic_block entry_succ_bb
= single_succ (entry_bb
);
8846 gimple_stmt_iterator gsi
;
8848 gimple tgtcopy_stmt
= NULL
;
8849 tree sender
= TREE_VEC_ELT (data_arg
, 0);
8851 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8853 gcc_assert (!gsi_end_p (gsi
));
8854 stmt
= gsi_stmt (gsi
);
8855 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8858 if (gimple_num_ops (stmt
) == 2)
8860 tree arg
= gimple_assign_rhs1 (stmt
);
8862 /* We're ignoring the subcode because we're
8863 effectively doing a STRIP_NOPS. */
8865 if (TREE_CODE (arg
) == ADDR_EXPR
8866 && TREE_OPERAND (arg
, 0) == sender
)
8868 tgtcopy_stmt
= stmt
;
8874 gcc_assert (tgtcopy_stmt
!= NULL
);
8875 arg
= DECL_ARGUMENTS (child_fn
);
8877 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8878 gsi_remove (&gsi
, true);
8881 /* Declare local variables needed in CHILD_CFUN. */
8882 block
= DECL_INITIAL (child_fn
);
8883 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8884 /* The gimplifier could record temporaries in the offloading block
8885 rather than in containing function's local_decls chain,
8886 which would mean cgraph missed finalizing them. Do it now. */
8887 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8888 if (TREE_CODE (t
) == VAR_DECL
8890 && !DECL_EXTERNAL (t
))
8891 varpool_node::finalize_decl (t
);
8892 DECL_SAVED_TREE (child_fn
) = NULL
;
8893 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8894 gimple_set_body (child_fn
, NULL
);
8895 TREE_USED (block
) = 1;
8897 /* Reset DECL_CONTEXT on function arguments. */
8898 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8899 DECL_CONTEXT (t
) = child_fn
;
8901 /* Split ENTRY_BB at GIMPLE_*,
8902 so that it can be moved to the child function. */
8903 gsi
= gsi_last_bb (entry_bb
);
8904 stmt
= gsi_stmt (gsi
);
8906 && gimple_code (stmt
) == gimple_code (entry_stmt
));
8907 e
= split_block (entry_bb
, stmt
);
8908 gsi_remove (&gsi
, true);
8910 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8912 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8915 gsi
= gsi_last_bb (exit_bb
);
8916 gcc_assert (!gsi_end_p (gsi
)
8917 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8918 stmt
= gimple_build_return (NULL
);
8919 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8920 gsi_remove (&gsi
, true);
8923 /* Move the offloading region into CHILD_CFUN. */
8925 block
= gimple_block (entry_stmt
);
8927 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8929 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8930 /* When the OMP expansion process cannot guarantee an up-to-date
8931 loop tree arrange for the child function to fixup loops. */
8932 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8933 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8935 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8936 num
= vec_safe_length (child_cfun
->local_decls
);
8937 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8939 t
= (*child_cfun
->local_decls
)[srcidx
];
8940 if (DECL_CONTEXT (t
) == cfun
->decl
)
8942 if (srcidx
!= dstidx
)
8943 (*child_cfun
->local_decls
)[dstidx
] = t
;
8947 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8949 /* Inform the callgraph about the new function. */
8950 child_cfun
->curr_properties
= cfun
->curr_properties
;
8951 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
8952 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
8953 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
8954 node
->parallelized_function
= 1;
8955 cgraph_node::add_new_function (child_fn
, true);
8957 #ifdef ENABLE_OFFLOADING
8958 /* Add the new function to the offload table. */
8959 vec_safe_push (offload_funcs
, child_fn
);
8962 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8963 fixed in a following pass. */
8964 push_cfun (child_cfun
);
8965 cgraph_edge::rebuild_edges ();
8967 #ifdef ENABLE_OFFLOADING
8968 /* Prevent IPA from removing child_fn as unreachable, since there are no
8969 refs from the parent function to child_fn in offload LTO mode. */
8970 cgraph_node::get (child_fn
)->mark_force_output ();
8973 /* Some EH regions might become dead, see PR34608. If
8974 pass_cleanup_cfg isn't the first pass to happen with the
8975 new child, these dead EH edges might cause problems.
8976 Clean them up now. */
8977 if (flag_exceptions
)
8980 bool changed
= false;
8982 FOR_EACH_BB_FN (bb
, cfun
)
8983 changed
|= gimple_purge_dead_eh_edges (bb
);
8985 cleanup_tree_cfg ();
8990 /* Emit a library call to launch the offloading region, or do data
8992 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8993 enum built_in_function start_ix
;
8994 location_t clause_loc
;
8996 switch (gimple_omp_target_kind (entry_stmt
))
8998 case GF_OMP_TARGET_KIND_REGION
:
8999 start_ix
= BUILT_IN_GOMP_TARGET
;
9001 case GF_OMP_TARGET_KIND_DATA
:
9002 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
9004 case GF_OMP_TARGET_KIND_UPDATE
:
9005 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
9007 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9008 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9009 start_ix
= BUILT_IN_GOACC_PARALLEL
;
9011 case GF_OMP_TARGET_KIND_OACC_DATA
:
9012 start_ix
= BUILT_IN_GOACC_DATA_START
;
9014 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9015 start_ix
= BUILT_IN_GOACC_UPDATE
;
9017 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9018 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
9024 clauses
= gimple_omp_target_clauses (entry_stmt
);
9026 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
9027 library choose) and there is no conditional. */
9029 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
9031 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
9033 cond
= OMP_CLAUSE_IF_EXPR (c
);
9035 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
9038 /* Even if we pass it to all library function calls, it is currently only
9039 defined/used for the OpenMP target ones. */
9040 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
9041 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
9042 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
);
9044 device
= OMP_CLAUSE_DEVICE_ID (c
);
9045 clause_loc
= OMP_CLAUSE_LOCATION (c
);
9048 clause_loc
= gimple_location (entry_stmt
);
9050 /* Ensure 'device' is of the correct type. */
9051 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
9053 /* If we found the clause 'if (cond)', build
9054 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
9057 cond
= gimple_boolify (cond
);
9059 basic_block cond_bb
, then_bb
, else_bb
;
9063 tmp_var
= create_tmp_var (TREE_TYPE (device
));
9065 e
= split_block_after_labels (new_bb
);
9068 gsi
= gsi_last_bb (new_bb
);
9070 e
= split_block (new_bb
, gsi_stmt (gsi
));
9076 then_bb
= create_empty_bb (cond_bb
);
9077 else_bb
= create_empty_bb (then_bb
);
9078 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
9079 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
9081 stmt
= gimple_build_cond_empty (cond
);
9082 gsi
= gsi_last_bb (cond_bb
);
9083 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9085 gsi
= gsi_start_bb (then_bb
);
9086 stmt
= gimple_build_assign (tmp_var
, device
);
9087 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9089 gsi
= gsi_start_bb (else_bb
);
9090 stmt
= gimple_build_assign (tmp_var
,
9091 build_int_cst (integer_type_node
,
9092 GOMP_DEVICE_HOST_FALLBACK
));
9093 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9095 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
9096 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
9097 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
9098 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
9099 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
9100 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
9105 gsi
= gsi_last_bb (new_bb
);
9106 t
= gimple_omp_target_data_arg (entry_stmt
);
9109 t1
= size_zero_node
;
9110 t2
= build_zero_cst (ptr_type_node
);
9116 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
9117 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
9118 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
9119 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
9120 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
9124 /* The maximum number used by any start_ix, without varargs. */
9125 auto_vec
<tree
, 11> args
;
9126 args
.quick_push (device
);
9128 args
.quick_push (build_fold_addr_expr (child_fn
));
9131 case BUILT_IN_GOMP_TARGET
:
9132 case BUILT_IN_GOMP_TARGET_DATA
:
9133 case BUILT_IN_GOMP_TARGET_UPDATE
:
9134 /* This const void * is part of the current ABI, but we're not actually
9136 args
.quick_push (build_zero_cst (ptr_type_node
));
9138 case BUILT_IN_GOACC_DATA_START
:
9139 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9140 case BUILT_IN_GOACC_PARALLEL
:
9141 case BUILT_IN_GOACC_UPDATE
:
9146 args
.quick_push (t1
);
9147 args
.quick_push (t2
);
9148 args
.quick_push (t3
);
9149 args
.quick_push (t4
);
9152 case BUILT_IN_GOACC_DATA_START
:
9153 case BUILT_IN_GOMP_TARGET
:
9154 case BUILT_IN_GOMP_TARGET_DATA
:
9155 case BUILT_IN_GOMP_TARGET_UPDATE
:
9157 case BUILT_IN_GOACC_PARALLEL
:
9159 tree t_num_gangs
, t_num_workers
, t_vector_length
;
9161 /* Default values for num_gangs, num_workers, and vector_length. */
9162 t_num_gangs
= t_num_workers
= t_vector_length
9163 = fold_convert_loc (gimple_location (entry_stmt
),
9164 integer_type_node
, integer_one_node
);
9165 /* ..., but if present, use the value specified by the respective
9166 clause, making sure that are of the correct type. */
9167 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
9169 t_num_gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9171 OMP_CLAUSE_NUM_GANGS_EXPR (c
));
9172 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_WORKERS
);
9174 t_num_workers
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9176 OMP_CLAUSE_NUM_WORKERS_EXPR (c
));
9177 c
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
9179 t_vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9181 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c
));
9182 args
.quick_push (t_num_gangs
);
9183 args
.quick_push (t_num_workers
);
9184 args
.quick_push (t_vector_length
);
9187 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9188 case BUILT_IN_GOACC_UPDATE
:
9193 /* Default values for t_async. */
9194 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
9196 build_int_cst (integer_type_node
,
9198 /* ..., but if present, use the value specified by the respective
9199 clause, making sure that is of the correct type. */
9200 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
9202 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9204 OMP_CLAUSE_ASYNC_EXPR (c
));
9206 args
.quick_push (t_async
);
9207 /* Save the index, and... */
9208 t_wait_idx
= args
.length ();
9209 /* ... push a default value. */
9210 args
.quick_push (fold_convert_loc (gimple_location (entry_stmt
),
9212 integer_zero_node
));
9213 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
9218 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
9220 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
9222 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9224 OMP_CLAUSE_WAIT_EXPR (c
)));
9229 /* Now that we know the number, replace the default value. */
9230 args
.ordered_remove (t_wait_idx
);
9231 args
.quick_insert (t_wait_idx
,
9232 fold_convert_loc (gimple_location (entry_stmt
),
9234 build_int_cst (integer_type_node
, n
)));
9242 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
9243 gimple_set_location (g
, gimple_location (entry_stmt
));
9244 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
9248 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
9249 gsi_remove (&gsi
, true);
9254 gsi
= gsi_last_bb (region
->exit
);
9256 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
9257 gsi_remove (&gsi
, true);
9262 /* Expand the parallel region tree rooted at REGION. Expansion
9263 proceeds in depth-first order. Innermost regions are expanded
9264 first. This way, parallel regions that require a new function to
9265 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9266 internal dependencies in their body. */
9269 expand_omp (struct omp_region
*region
)
9273 location_t saved_location
;
9274 gimple inner_stmt
= NULL
;
9276 /* First, determine whether this is a combined parallel+workshare
9278 if (region
->type
== GIMPLE_OMP_PARALLEL
)
9279 determine_parallel_type (region
);
9281 if (region
->type
== GIMPLE_OMP_FOR
9282 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
9283 inner_stmt
= last_stmt (region
->inner
->entry
);
9286 expand_omp (region
->inner
);
9288 saved_location
= input_location
;
9289 if (gimple_has_location (last_stmt (region
->entry
)))
9290 input_location
= gimple_location (last_stmt (region
->entry
));
9292 switch (region
->type
)
9294 case GIMPLE_OMP_PARALLEL
:
9295 case GIMPLE_OMP_TASK
:
9296 expand_omp_taskreg (region
);
9299 case GIMPLE_OMP_FOR
:
9300 expand_omp_for (region
, inner_stmt
);
9303 case GIMPLE_OMP_SECTIONS
:
9304 expand_omp_sections (region
);
9307 case GIMPLE_OMP_SECTION
:
9308 /* Individual omp sections are handled together with their
9309 parent GIMPLE_OMP_SECTIONS region. */
9312 case GIMPLE_OMP_SINGLE
:
9313 expand_omp_single (region
);
9316 case GIMPLE_OMP_MASTER
:
9317 case GIMPLE_OMP_TASKGROUP
:
9318 case GIMPLE_OMP_ORDERED
:
9319 case GIMPLE_OMP_CRITICAL
:
9320 case GIMPLE_OMP_TEAMS
:
9321 expand_omp_synch (region
);
9324 case GIMPLE_OMP_ATOMIC_LOAD
:
9325 expand_omp_atomic (region
);
9328 case GIMPLE_OMP_TARGET
:
9329 expand_omp_target (region
);
9336 input_location
= saved_location
;
9337 region
= region
->next
;
9342 /* Helper for build_omp_regions. Scan the dominator tree starting at
9343 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9344 true, the function ends once a single tree is built (otherwise, whole
9345 forest of OMP constructs may be built). */
9348 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
9351 gimple_stmt_iterator gsi
;
9355 gsi
= gsi_last_bb (bb
);
9356 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
9358 struct omp_region
*region
;
9359 enum gimple_code code
;
9361 stmt
= gsi_stmt (gsi
);
9362 code
= gimple_code (stmt
);
9363 if (code
== GIMPLE_OMP_RETURN
)
9365 /* STMT is the return point out of region PARENT. Mark it
9366 as the exit point and make PARENT the immediately
9367 enclosing region. */
9368 gcc_assert (parent
);
9371 parent
= parent
->outer
;
9373 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
9375 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9376 GIMPLE_OMP_RETURN, but matches with
9377 GIMPLE_OMP_ATOMIC_LOAD. */
9378 gcc_assert (parent
);
9379 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
9382 parent
= parent
->outer
;
9384 else if (code
== GIMPLE_OMP_CONTINUE
)
9386 gcc_assert (parent
);
9389 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
9391 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9392 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9396 region
= new_omp_region (bb
, code
, parent
);
9398 if (code
== GIMPLE_OMP_TARGET
)
9400 switch (gimple_omp_target_kind (stmt
))
9402 case GF_OMP_TARGET_KIND_REGION
:
9403 case GF_OMP_TARGET_KIND_DATA
:
9404 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9405 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9406 case GF_OMP_TARGET_KIND_OACC_DATA
:
9408 case GF_OMP_TARGET_KIND_UPDATE
:
9409 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9410 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9411 /* ..., other than for those stand-alone directives... */
9418 /* ..., this directive becomes the parent for a new region. */
9424 if (single_tree
&& !parent
)
9427 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
9429 son
= next_dom_son (CDI_DOMINATORS
, son
))
9430 build_omp_regions_1 (son
, parent
, single_tree
);
9433 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9437 build_omp_regions_root (basic_block root
)
9439 gcc_assert (root_omp_region
== NULL
);
9440 build_omp_regions_1 (root
, NULL
, true);
9441 gcc_assert (root_omp_region
!= NULL
);
9444 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9447 omp_expand_local (basic_block head
)
9449 build_omp_regions_root (head
);
9450 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
9452 fprintf (dump_file
, "\nOMP region tree\n\n");
9453 dump_omp_region (dump_file
, root_omp_region
, 0);
9454 fprintf (dump_file
, "\n");
9457 remove_exit_barriers (root_omp_region
);
9458 expand_omp (root_omp_region
);
9460 free_omp_regions ();
9463 /* Scan the CFG and build a tree of OMP regions. Return the root of
9464 the OMP region tree. */
9467 build_omp_regions (void)
9469 gcc_assert (root_omp_region
== NULL
);
9470 calculate_dominance_info (CDI_DOMINATORS
);
9471 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
9474 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9477 execute_expand_omp (void)
9479 build_omp_regions ();
9481 if (!root_omp_region
)
9486 fprintf (dump_file
, "\nOMP region tree\n\n");
9487 dump_omp_region (dump_file
, root_omp_region
, 0);
9488 fprintf (dump_file
, "\n");
9491 remove_exit_barriers (root_omp_region
);
9493 expand_omp (root_omp_region
);
9495 cleanup_tree_cfg ();
9497 free_omp_regions ();
9502 /* OMP expansion -- the default pass, run before creation of SSA form. */
9506 const pass_data pass_data_expand_omp
=
9508 GIMPLE_PASS
, /* type */
9509 "ompexp", /* name */
9510 OPTGROUP_NONE
, /* optinfo_flags */
9511 TV_NONE
, /* tv_id */
9512 PROP_gimple_any
, /* properties_required */
9513 PROP_gimple_eomp
, /* properties_provided */
9514 0, /* properties_destroyed */
9515 0, /* todo_flags_start */
9516 0, /* todo_flags_finish */
9519 class pass_expand_omp
: public gimple_opt_pass
9522 pass_expand_omp (gcc::context
*ctxt
)
9523 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
9526 /* opt_pass methods: */
9527 virtual unsigned int execute (function
*)
9529 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
9530 || flag_openmp_simd
!= 0)
9533 /* This pass always runs, to provide PROP_gimple_eomp.
9534 But often, there is nothing to do. */
9538 return execute_expand_omp ();
9541 }; // class pass_expand_omp
9546 make_pass_expand_omp (gcc::context
*ctxt
)
9548 return new pass_expand_omp (ctxt
);
9553 const pass_data pass_data_expand_omp_ssa
=
9555 GIMPLE_PASS
, /* type */
9556 "ompexpssa", /* name */
9557 OPTGROUP_NONE
, /* optinfo_flags */
9558 TV_NONE
, /* tv_id */
9559 PROP_cfg
| PROP_ssa
, /* properties_required */
9560 PROP_gimple_eomp
, /* properties_provided */
9561 0, /* properties_destroyed */
9562 0, /* todo_flags_start */
9563 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
9566 class pass_expand_omp_ssa
: public gimple_opt_pass
9569 pass_expand_omp_ssa (gcc::context
*ctxt
)
9570 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
9573 /* opt_pass methods: */
9574 virtual bool gate (function
*fun
)
9576 return !(fun
->curr_properties
& PROP_gimple_eomp
);
9578 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
9580 }; // class pass_expand_omp_ssa
9585 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
9587 return new pass_expand_omp_ssa (ctxt
);
9590 /* Routines to lower OMP directives into OMP-GIMPLE. */
9592 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9593 convert it to gimple. */
9595 oacc_gimple_assign (tree dest
, tree_code op
, tree src
, gimple_seq
*seq
)
9599 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
9601 stmt
= gimple_build_assign (dest
, op
, dest
, src
);
9602 gimple_seq_add_stmt (seq
, stmt
);
9606 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9607 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9608 gimplify_assign (t
, rdest
, seq
);
9611 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9612 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9613 gimplify_assign (t
, idest
, seq
);
9616 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9617 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9618 gimplify_assign (t
, rsrc
, seq
);
9621 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9622 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9623 gimplify_assign (t
, isrc
, seq
);
9626 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9627 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9630 if (op
== PLUS_EXPR
)
9632 stmt
= gimple_build_assign (r
, op
, rdest
, rsrc
);
9633 gimple_seq_add_stmt (seq
, stmt
);
9635 stmt
= gimple_build_assign (i
, op
, idest
, isrc
);
9636 gimple_seq_add_stmt (seq
, stmt
);
9638 else if (op
== MULT_EXPR
)
9640 /* Let x = a + ib = dest, y = c + id = src.
9641 x * y = (ac - bd) + i(ad + bc) */
9642 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9643 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9644 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9645 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9647 stmt
= gimple_build_assign (ac
, MULT_EXPR
, rdest
, rsrc
);
9648 gimple_seq_add_stmt (seq
, stmt
);
9650 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, isrc
);
9651 gimple_seq_add_stmt (seq
, stmt
);
9653 stmt
= gimple_build_assign (r
, MINUS_EXPR
, ac
, bd
);
9654 gimple_seq_add_stmt (seq
, stmt
);
9656 stmt
= gimple_build_assign (ad
, MULT_EXPR
, rdest
, isrc
);
9657 gimple_seq_add_stmt (seq
, stmt
);
9659 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, rsrc
);
9660 gimple_seq_add_stmt (seq
, stmt
);
9662 stmt
= gimple_build_assign (i
, PLUS_EXPR
, ad
, bc
);
9663 gimple_seq_add_stmt (seq
, stmt
);
9668 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
9669 gimplify_assign (dest
, result
, seq
);
9672 /* Helper function to initialize local data for the reduction arrays.
9673 The reduction arrays need to be placed inside the calling function
9674 for accelerators, or else the host won't be able to preform the final
9678 oacc_initialize_reduction_data (tree clauses
, tree nthreads
,
9679 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9685 /* Find the innermost OpenACC parallel context. */
9686 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
9687 && (gimple_omp_target_kind (ctx
->stmt
)
9688 == GF_OMP_TARGET_KIND_OACC_PARALLEL
))
9692 gcc_checking_assert (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
9693 && (gimple_omp_target_kind (octx
->stmt
)
9694 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
9696 /* Extract the clauses. */
9697 oc
= gimple_omp_target_clauses (octx
->stmt
);
9699 /* Find the last outer clause. */
9700 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
9703 /* Allocate arrays for each reduction variable. */
9704 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9706 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9709 tree var
= OMP_CLAUSE_DECL (c
);
9710 tree type
= get_base_type (var
);
9711 tree array
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
),
9715 /* Calculate size of the reduction array. */
9716 t
= create_tmp_var (TREE_TYPE (nthreads
));
9717 stmt
= gimple_build_assign (t
, MULT_EXPR
, nthreads
,
9718 fold_convert (TREE_TYPE (nthreads
),
9719 TYPE_SIZE_UNIT (type
)));
9720 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9722 size
= create_tmp_var (sizetype
);
9723 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
9725 /* Now allocate memory for it. */
9726 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
9727 stmt
= gimple_build_call (call
, 1, size
);
9728 gimple_call_set_lhs (stmt
, array
);
9729 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9731 /* Map this array into the accelerator. */
9733 /* Add the reduction array to the list of clauses. */
9735 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
9736 OMP_CLAUSE_SET_MAP_KIND (t
, GOMP_MAP_FORCE_FROM
);
9737 OMP_CLAUSE_DECL (t
) = x
;
9738 OMP_CLAUSE_CHAIN (t
) = NULL
;
9740 OMP_CLAUSE_CHAIN (oc
) = t
;
9742 gimple_omp_target_set_clauses (as_a
<gomp_target
*> (octx
->stmt
), t
);
9743 OMP_CLAUSE_SIZE (t
) = size
;
9748 /* Helper function to process the array of partial reductions. Nthreads
9749 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9750 cannot be used here, because nthreads on the host may be different than
9751 on the accelerator. */
9754 oacc_finalize_reduction_data (tree clauses
, tree nthreads
,
9755 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9757 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
9762 let var = the original reduction variable
9763 let array = reduction variable array
9765 for (i = 0; i < nthreads; i++)
9769 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
9770 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
9771 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
9773 /* Create and initialize an index variable. */
9774 tree ix
= create_tmp_var (sizetype
);
9775 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
9778 /* Insert the loop header label here. */
9779 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
9781 /* Exit loop if ix >= nthreads. */
9782 x
= create_tmp_var (sizetype
);
9783 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
9784 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
9785 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9787 /* Insert the loop body label here. */
9788 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
9790 /* Collapse each reduction array, one element at a time. */
9791 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9793 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9796 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
9798 /* reduction(-:var) sums up the partial results, so it acts
9799 identically to reduction(+:var). */
9800 if (reduction_code
== MINUS_EXPR
)
9801 reduction_code
= PLUS_EXPR
;
9803 /* Set up reduction variable var. */
9804 var
= OMP_CLAUSE_DECL (c
);
9805 type
= get_base_type (var
);
9806 array
= lookup_oacc_reduction (oacc_get_reduction_array_id
9807 (OMP_CLAUSE_DECL (c
)), ctx
);
9809 /* Calculate the array offset. */
9810 tree offset
= create_tmp_var (sizetype
);
9811 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
9812 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, ix
);
9813 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9815 tree ptr
= create_tmp_var (TREE_TYPE (array
));
9816 stmt
= gimple_build_assign (ptr
, POINTER_PLUS_EXPR
, array
, offset
);
9817 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9819 /* Extract array[ix] into mem. */
9820 tree mem
= create_tmp_var (type
);
9821 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
9823 /* Find the original reduction variable. */
9824 if (is_reference (var
))
9825 var
= build_simple_mem_ref (var
);
9827 tree t
= create_tmp_var (type
);
9829 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
9830 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9832 /* var = var op mem */
9833 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
9835 case TRUTH_ANDIF_EXPR
:
9836 case TRUTH_ORIF_EXPR
:
9837 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
9839 gimplify_and_add (t
, stmt_seqp
);
9842 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9843 oacc_gimple_assign (t
, OMP_CLAUSE_REDUCTION_CODE (c
), mem
,
9847 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
9848 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
9849 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9852 /* Increment the induction variable. */
9853 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
9854 stmt
= gimple_build_assign (ix
, PLUS_EXPR
, ix
, one
);
9855 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9857 /* Go back to the top of the loop. */
9858 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
9860 /* Place the loop exit label here. */
9861 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
9864 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9865 scan that for reductions. */
9868 oacc_process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
9869 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
9871 gimple_stmt_iterator gsi
;
9872 gimple_seq inner
= NULL
;
9874 /* A collapse clause may have inserted a new bind block. */
9875 gsi
= gsi_start (*body
);
9876 while (!gsi_end_p (gsi
))
9878 gimple stmt
= gsi_stmt (gsi
);
9879 if (gbind
*bind_stmt
= dyn_cast
<gbind
*> (stmt
))
9881 inner
= gimple_bind_body (bind_stmt
);
9883 gsi
= gsi_start (*body
);
9885 else if (dyn_cast
<gomp_for
*> (stmt
))
9891 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9893 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
9895 bool reduction_found
= false;
9897 gimple stmt
= gsi_stmt (gsi
);
9899 switch (gimple_code (stmt
))
9901 case GIMPLE_OMP_FOR
:
9902 clauses
= gimple_omp_for_clauses (stmt
);
9904 /* Search for a reduction clause. */
9905 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9906 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
9908 reduction_found
= true;
9912 if (!reduction_found
)
9915 ctx
= maybe_lookup_ctx (stmt
);
9918 /* Extract the number of threads. */
9919 nthreads
= create_tmp_var (sizetype
);
9920 t
= oacc_max_threads (ctx
);
9921 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
9923 /* Determine if this is kernel will be executed on the host. */
9924 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
9925 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
9926 stmt
= gimple_build_call (call
, 0);
9927 gimple_call_set_lhs (stmt
, acc_device
);
9928 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9930 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9931 acc_device_host
= create_tmp_var (integer_type_node
,
9932 ".acc_device_host");
9933 gimplify_assign (acc_device_host
,
9934 build_int_cst (integer_type_node
,
9938 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9939 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9941 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9943 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9944 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9945 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9948 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9950 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9951 gimplify_assign (acc_device_host
,
9952 build_int_cst (integer_type_node
,
9953 GOMP_DEVICE_HOST_NONSHM
),
9956 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9957 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9959 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9961 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9962 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9963 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9966 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9968 oacc_initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
,
9970 oacc_finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
9973 // Scan for other directives which support reduction here.
9979 /* If ctx is a worksharing context inside of a cancellable parallel
9980 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
9981 and conditional branch to parallel's cancel_label to handle
9982 cancellation in the implicit barrier. */
9985 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
9987 gimple omp_return
= gimple_seq_last_stmt (*body
);
9988 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
9989 if (gimple_omp_return_nowait_p (omp_return
))
9992 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
9993 && ctx
->outer
->cancellable
)
9995 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
9996 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
9997 tree lhs
= create_tmp_var (c_bool_type
);
9998 gimple_omp_return_set_lhs (omp_return
, lhs
);
9999 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10000 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
10001 fold_convert (c_bool_type
,
10002 boolean_false_node
),
10003 ctx
->outer
->cancel_label
, fallthru_label
);
10004 gimple_seq_add_stmt (body
, g
);
10005 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
10009 /* Lower the OpenMP sections directive in the current statement in GSI_P.
10010 CTX is the enclosing OMP context for the current statement. */
10013 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10015 tree block
, control
;
10016 gimple_stmt_iterator tgsi
;
10017 gomp_sections
*stmt
;
10019 gbind
*new_stmt
, *bind
;
10020 gimple_seq ilist
, dlist
, olist
, new_body
;
10022 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
10024 push_gimplify_context ();
10028 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
10029 &ilist
, &dlist
, ctx
, NULL
);
10031 new_body
= gimple_omp_body (stmt
);
10032 gimple_omp_set_body (stmt
, NULL
);
10033 tgsi
= gsi_start (new_body
);
10034 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
10039 sec_start
= gsi_stmt (tgsi
);
10040 sctx
= maybe_lookup_ctx (sec_start
);
10043 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
10044 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
10045 GSI_CONTINUE_LINKING
);
10046 gimple_omp_set_body (sec_start
, NULL
);
10048 if (gsi_one_before_end_p (tgsi
))
10050 gimple_seq l
= NULL
;
10051 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
10053 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
10054 gimple_omp_section_set_last (sec_start
);
10057 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
10058 GSI_CONTINUE_LINKING
);
10061 block
= make_node (BLOCK
);
10062 bind
= gimple_build_bind (NULL
, new_body
, block
);
10065 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
10067 block
= make_node (BLOCK
);
10068 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10069 gsi_replace (gsi_p
, new_stmt
, true);
10071 pop_gimplify_context (new_stmt
);
10072 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10073 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10074 if (BLOCK_VARS (block
))
10075 TREE_USED (block
) = 1;
10078 gimple_seq_add_seq (&new_body
, ilist
);
10079 gimple_seq_add_stmt (&new_body
, stmt
);
10080 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
10081 gimple_seq_add_stmt (&new_body
, bind
);
10083 control
= create_tmp_var (unsigned_type_node
, ".section");
10084 t
= gimple_build_omp_continue (control
, control
);
10085 gimple_omp_sections_set_control (stmt
, control
);
10086 gimple_seq_add_stmt (&new_body
, t
);
10088 gimple_seq_add_seq (&new_body
, olist
);
10089 if (ctx
->cancellable
)
10090 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10091 gimple_seq_add_seq (&new_body
, dlist
);
10093 new_body
= maybe_catch_exception (new_body
);
10095 t
= gimple_build_omp_return
10096 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
10097 OMP_CLAUSE_NOWAIT
));
10098 gimple_seq_add_stmt (&new_body
, t
);
10099 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
10101 gimple_bind_set_body (new_stmt
, new_body
);
10105 /* A subroutine of lower_omp_single. Expand the simple form of
10106 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10108 if (GOMP_single_start ())
10110 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10112 FIXME. It may be better to delay expanding the logic of this until
10113 pass_expand_omp. The expanded logic may make the job more difficult
10114 to a synchronization analysis pass. */
10117 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
10119 location_t loc
= gimple_location (single_stmt
);
10120 tree tlabel
= create_artificial_label (loc
);
10121 tree flabel
= create_artificial_label (loc
);
10125 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
10126 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
10127 call
= gimple_build_call (decl
, 0);
10128 gimple_call_set_lhs (call
, lhs
);
10129 gimple_seq_add_stmt (pre_p
, call
);
10131 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
10132 fold_convert_loc (loc
, TREE_TYPE (lhs
),
10133 boolean_true_node
),
10135 gimple_seq_add_stmt (pre_p
, cond
);
10136 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
10137 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10138 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
10142 /* A subroutine of lower_omp_single. Expand the simple form of
10143 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10145 #pragma omp single copyprivate (a, b, c)
10147 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10150 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10156 GOMP_single_copy_end (©out);
10167 FIXME. It may be better to delay expanding the logic of this until
10168 pass_expand_omp. The expanded logic may make the job more difficult
10169 to a synchronization analysis pass. */
10172 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
10175 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
10176 gimple_seq copyin_seq
;
10177 location_t loc
= gimple_location (single_stmt
);
10179 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
10181 ptr_type
= build_pointer_type (ctx
->record_type
);
10182 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
10184 l0
= create_artificial_label (loc
);
10185 l1
= create_artificial_label (loc
);
10186 l2
= create_artificial_label (loc
);
10188 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
10189 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
10190 t
= fold_convert_loc (loc
, ptr_type
, t
);
10191 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
10193 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
10194 build_int_cst (ptr_type
, 0));
10195 t
= build3 (COND_EXPR
, void_type_node
, t
,
10196 build_and_jump (&l0
), build_and_jump (&l1
));
10197 gimplify_and_add (t
, pre_p
);
10199 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
10201 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10204 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
10207 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10208 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
10209 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
10210 gimplify_and_add (t
, pre_p
);
10212 t
= build_and_jump (&l2
);
10213 gimplify_and_add (t
, pre_p
);
10215 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
10217 gimple_seq_add_seq (pre_p
, copyin_seq
);
10219 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
10223 /* Expand code for an OpenMP single directive. */
10226 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10230 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
10232 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
10234 push_gimplify_context ();
10236 block
= make_node (BLOCK
);
10237 bind
= gimple_build_bind (NULL
, NULL
, block
);
10238 gsi_replace (gsi_p
, bind
, true);
10241 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
10242 &bind_body
, &dlist
, ctx
, NULL
);
10243 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
10245 gimple_seq_add_stmt (&bind_body
, single_stmt
);
10247 if (ctx
->record_type
)
10248 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
10250 lower_omp_single_simple (single_stmt
, &bind_body
);
10252 gimple_omp_set_body (single_stmt
, NULL
);
10254 gimple_seq_add_seq (&bind_body
, dlist
);
10256 bind_body
= maybe_catch_exception (bind_body
);
10258 t
= gimple_build_omp_return
10259 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
10260 OMP_CLAUSE_NOWAIT
));
10261 gimple_seq_add_stmt (&bind_body_tail
, t
);
10262 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
10263 if (ctx
->record_type
)
10265 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
10266 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10267 TREE_THIS_VOLATILE (clobber
) = 1;
10268 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
10269 clobber
), GSI_SAME_STMT
);
10271 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
10272 gimple_bind_set_body (bind
, bind_body
);
10274 pop_gimplify_context (bind
);
10276 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10277 BLOCK_VARS (block
) = ctx
->block_vars
;
10278 if (BLOCK_VARS (block
))
10279 TREE_USED (block
) = 1;
10283 /* Expand code for an OpenMP master directive. */
10286 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10288 tree block
, lab
= NULL
, x
, bfn_decl
;
10289 gimple stmt
= gsi_stmt (*gsi_p
);
10291 location_t loc
= gimple_location (stmt
);
10294 push_gimplify_context ();
10296 block
= make_node (BLOCK
);
10297 bind
= gimple_build_bind (NULL
, NULL
, block
);
10298 gsi_replace (gsi_p
, bind
, true);
10299 gimple_bind_add_stmt (bind
, stmt
);
10301 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
10302 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
10303 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
10304 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
10306 gimplify_and_add (x
, &tseq
);
10307 gimple_bind_add_seq (bind
, tseq
);
10309 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10310 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10311 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10312 gimple_omp_set_body (stmt
, NULL
);
10314 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
10316 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10318 pop_gimplify_context (bind
);
10320 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10321 BLOCK_VARS (block
) = ctx
->block_vars
;
10325 /* Expand code for an OpenMP taskgroup directive. */
10328 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10330 gimple stmt
= gsi_stmt (*gsi_p
);
10333 tree block
= make_node (BLOCK
);
10335 bind
= gimple_build_bind (NULL
, NULL
, block
);
10336 gsi_replace (gsi_p
, bind
, true);
10337 gimple_bind_add_stmt (bind
, stmt
);
10339 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
10341 gimple_bind_add_stmt (bind
, x
);
10343 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10344 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10345 gimple_omp_set_body (stmt
, NULL
);
10347 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10349 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10350 BLOCK_VARS (block
) = ctx
->block_vars
;
10354 /* Expand code for an OpenMP ordered directive. */
10357 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10360 gimple stmt
= gsi_stmt (*gsi_p
);
10364 push_gimplify_context ();
10366 block
= make_node (BLOCK
);
10367 bind
= gimple_build_bind (NULL
, NULL
, block
);
10368 gsi_replace (gsi_p
, bind
, true);
10369 gimple_bind_add_stmt (bind
, stmt
);
10371 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
10373 gimple_bind_add_stmt (bind
, x
);
10375 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10376 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10377 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10378 gimple_omp_set_body (stmt
, NULL
);
10380 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
10381 gimple_bind_add_stmt (bind
, x
);
10383 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10385 pop_gimplify_context (bind
);
10387 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10388 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10392 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10393 substitution of a couple of function calls. But in the NAMED case,
10394 requires that languages coordinate a symbol name. It is therefore
10395 best put here in common code. */
10397 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
10400 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10403 tree name
, lock
, unlock
;
10404 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
10406 location_t loc
= gimple_location (stmt
);
10409 name
= gimple_omp_critical_name (stmt
);
10414 if (!critical_name_mutexes
)
10415 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
10417 tree
*n
= critical_name_mutexes
->get (name
);
10422 decl
= create_tmp_var_raw (ptr_type_node
);
10424 new_str
= ACONCAT ((".gomp_critical_user_",
10425 IDENTIFIER_POINTER (name
), NULL
));
10426 DECL_NAME (decl
) = get_identifier (new_str
);
10427 TREE_PUBLIC (decl
) = 1;
10428 TREE_STATIC (decl
) = 1;
10429 DECL_COMMON (decl
) = 1;
10430 DECL_ARTIFICIAL (decl
) = 1;
10431 DECL_IGNORED_P (decl
) = 1;
10433 varpool_node::finalize_decl (decl
);
10435 critical_name_mutexes
->put (name
, decl
);
10440 /* If '#pragma omp critical' is inside offloaded region or
10441 inside function marked as offloadable, the symbol must be
10442 marked as offloadable too. */
10444 if (cgraph_node::get (current_function_decl
)->offloadable
)
10445 varpool_node::get_create (decl
)->offloadable
= 1;
10447 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
10448 if (is_gimple_omp_offloaded (octx
->stmt
))
10450 varpool_node::get_create (decl
)->offloadable
= 1;
10454 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
10455 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
10457 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
10458 unlock
= build_call_expr_loc (loc
, unlock
, 1,
10459 build_fold_addr_expr_loc (loc
, decl
));
10463 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
10464 lock
= build_call_expr_loc (loc
, lock
, 0);
10466 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
10467 unlock
= build_call_expr_loc (loc
, unlock
, 0);
10470 push_gimplify_context ();
10472 block
= make_node (BLOCK
);
10473 bind
= gimple_build_bind (NULL
, NULL
, block
);
10474 gsi_replace (gsi_p
, bind
, true);
10475 gimple_bind_add_stmt (bind
, stmt
);
10477 tbody
= gimple_bind_body (bind
);
10478 gimplify_and_add (lock
, &tbody
);
10479 gimple_bind_set_body (bind
, tbody
);
10481 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10482 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10483 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10484 gimple_omp_set_body (stmt
, NULL
);
10486 tbody
= gimple_bind_body (bind
);
10487 gimplify_and_add (unlock
, &tbody
);
10488 gimple_bind_set_body (bind
, tbody
);
10490 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10492 pop_gimplify_context (bind
);
10493 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10494 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10498 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10499 for a lastprivate clause. Given a loop control predicate of (V
10500 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10501 is appended to *DLIST, iterator initialization is appended to
10505 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
10506 gimple_seq
*dlist
, struct omp_context
*ctx
)
10508 tree clauses
, cond
, vinit
;
10509 enum tree_code cond_code
;
10512 cond_code
= fd
->loop
.cond_code
;
10513 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
10515 /* When possible, use a strict equality expression. This can let VRP
10516 type optimizations deduce the value and remove a copy. */
10517 if (tree_fits_shwi_p (fd
->loop
.step
))
10519 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
10520 if (step
== 1 || step
== -1)
10521 cond_code
= EQ_EXPR
;
10524 tree n2
= fd
->loop
.n2
;
10525 if (fd
->collapse
> 1
10526 && TREE_CODE (n2
) != INTEGER_CST
10527 && gimple_omp_for_combined_into_p (fd
->for_stmt
)
10528 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
10530 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
10531 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
)
10533 struct omp_for_data outer_fd
;
10534 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
10535 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
10538 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
10540 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
10542 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
10543 if (!gimple_seq_empty_p (stmts
))
10545 gimple_seq_add_seq (&stmts
, *dlist
);
10548 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10549 vinit
= fd
->loop
.n1
;
10550 if (cond_code
== EQ_EXPR
10551 && tree_fits_shwi_p (fd
->loop
.n2
)
10552 && ! integer_zerop (fd
->loop
.n2
))
10553 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
10555 vinit
= unshare_expr (vinit
);
10557 /* Initialize the iterator variable, so that threads that don't execute
10558 any iterations don't execute the lastprivate clauses by accident. */
10559 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
10564 /* Lower code for an OMP loop directive. */
10567 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10569 tree
*rhs_p
, block
;
10570 struct omp_for_data fd
, *fdp
= NULL
;
10571 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
10573 gimple_seq omp_for_body
, body
, dlist
;
10576 push_gimplify_context ();
10578 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
10580 block
= make_node (BLOCK
);
10581 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10582 /* Replace at gsi right away, so that 'stmt' is no member
10583 of a sequence anymore as we're going to add to a different
10585 gsi_replace (gsi_p
, new_stmt
, true);
10587 /* Move declaration of temporaries in the loop body before we make
10589 omp_for_body
= gimple_omp_body (stmt
);
10590 if (!gimple_seq_empty_p (omp_for_body
)
10591 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
10594 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
10595 tree vars
= gimple_bind_vars (inner_bind
);
10596 gimple_bind_append_vars (new_stmt
, vars
);
10597 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10598 keep them on the inner_bind and it's block. */
10599 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
10600 if (gimple_bind_block (inner_bind
))
10601 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
10604 if (gimple_omp_for_combined_into_p (stmt
))
10606 extract_omp_for_data (stmt
, &fd
, NULL
);
10609 /* We need two temporaries with fd.loop.v type (istart/iend)
10610 and then (fd.collapse - 1) temporaries with the same
10611 type for count2 ... countN-1 vars if not constant. */
10613 tree type
= fd
.iter_type
;
10614 if (fd
.collapse
> 1
10615 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
10616 count
+= fd
.collapse
- 1;
10617 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
10618 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
10619 tree clauses
= *pc
;
10622 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
10623 OMP_CLAUSE__LOOPTEMP_
);
10624 for (i
= 0; i
< count
; i
++)
10629 gcc_assert (outerc
);
10630 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
10631 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
10632 OMP_CLAUSE__LOOPTEMP_
);
10636 temp
= create_tmp_var (type
);
10637 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
10639 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
10640 OMP_CLAUSE_DECL (*pc
) = temp
;
10641 pc
= &OMP_CLAUSE_CHAIN (*pc
);
10646 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10649 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
10651 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
10653 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10655 /* Lower the header expressions. At this point, we can assume that
10656 the header is of the form:
10658 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10660 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10661 using the .omp_data_s mapping, if needed. */
10662 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
10664 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
10665 if (!is_gimple_min_invariant (*rhs_p
))
10666 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10668 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
10669 if (!is_gimple_min_invariant (*rhs_p
))
10670 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10672 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
10673 if (!is_gimple_min_invariant (*rhs_p
))
10674 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10677 /* Once lowered, extract the bounds and clauses. */
10678 extract_omp_for_data (stmt
, &fd
, NULL
);
10680 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
10682 gimple_seq_add_stmt (&body
, stmt
);
10683 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
10685 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
10688 /* After the loop, add exit clauses. */
10689 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
10691 if (ctx
->cancellable
)
10692 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
10694 gimple_seq_add_seq (&body
, dlist
);
10696 body
= maybe_catch_exception (body
);
10698 /* Region exit marker goes at the end of the loop body. */
10699 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
10700 maybe_add_implicit_barrier_cancel (ctx
, &body
);
10701 pop_gimplify_context (new_stmt
);
10703 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10704 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
10705 if (BLOCK_VARS (block
))
10706 TREE_USED (block
) = 1;
10708 gimple_bind_set_body (new_stmt
, body
);
10709 gimple_omp_set_body (stmt
, NULL
);
10710 gimple_omp_for_set_pre_body (stmt
, NULL
);
10713 /* Callback for walk_stmts. Check if the current statement only contains
10714 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10717 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
10718 bool *handled_ops_p
,
10719 struct walk_stmt_info
*wi
)
10721 int *info
= (int *) wi
->info
;
10722 gimple stmt
= gsi_stmt (*gsi_p
);
10724 *handled_ops_p
= true;
10725 switch (gimple_code (stmt
))
10729 case GIMPLE_OMP_FOR
:
10730 case GIMPLE_OMP_SECTIONS
:
10731 *info
= *info
== 0 ? 1 : -1;
10740 struct omp_taskcopy_context
10742 /* This field must be at the beginning, as we do "inheritance": Some
10743 callback functions for tree-inline.c (e.g., omp_copy_decl)
10744 receive a copy_body_data pointer that is up-casted to an
10745 omp_context pointer. */
10751 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
10753 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
10755 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
10756 return create_tmp_var (TREE_TYPE (var
));
10762 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
10764 tree name
, new_fields
= NULL
, type
, f
;
10766 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
10767 name
= DECL_NAME (TYPE_NAME (orig_type
));
10768 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
10769 TYPE_DECL
, name
, type
);
10770 TYPE_NAME (type
) = name
;
10772 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
10774 tree new_f
= copy_node (f
);
10775 DECL_CONTEXT (new_f
) = type
;
10776 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
10777 TREE_CHAIN (new_f
) = new_fields
;
10778 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10779 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10780 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
10782 new_fields
= new_f
;
10783 tcctx
->cb
.decl_map
->put (f
, new_f
);
10785 TYPE_FIELDS (type
) = nreverse (new_fields
);
10786 layout_type (type
);
10790 /* Create task copyfn. */
10793 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
10795 struct function
*child_cfun
;
10796 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
10797 tree record_type
, srecord_type
, bind
, list
;
10798 bool record_needs_remap
= false, srecord_needs_remap
= false;
10800 struct omp_taskcopy_context tcctx
;
10801 location_t loc
= gimple_location (task_stmt
);
10803 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
10804 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
10805 gcc_assert (child_cfun
->cfg
== NULL
);
10806 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
10808 /* Reset DECL_CONTEXT on function arguments. */
10809 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
10810 DECL_CONTEXT (t
) = child_fn
;
10812 /* Populate the function. */
10813 push_gimplify_context ();
10814 push_cfun (child_cfun
);
10816 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
10817 TREE_SIDE_EFFECTS (bind
) = 1;
10819 DECL_SAVED_TREE (child_fn
) = bind
;
10820 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
10822 /* Remap src and dst argument types if needed. */
10823 record_type
= ctx
->record_type
;
10824 srecord_type
= ctx
->srecord_type
;
10825 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
10826 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10828 record_needs_remap
= true;
10831 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
10832 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10834 srecord_needs_remap
= true;
10838 if (record_needs_remap
|| srecord_needs_remap
)
10840 memset (&tcctx
, '\0', sizeof (tcctx
));
10841 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
10842 tcctx
.cb
.dst_fn
= child_fn
;
10843 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
10844 gcc_checking_assert (tcctx
.cb
.src_node
);
10845 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
10846 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
10847 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
10848 tcctx
.cb
.eh_lp_nr
= 0;
10849 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
10850 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
10853 if (record_needs_remap
)
10854 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
10855 if (srecord_needs_remap
)
10856 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
10859 tcctx
.cb
.decl_map
= NULL
;
10861 arg
= DECL_ARGUMENTS (child_fn
);
10862 TREE_TYPE (arg
) = build_pointer_type (record_type
);
10863 sarg
= DECL_CHAIN (arg
);
10864 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
10866 /* First pass: initialize temporaries used in record_type and srecord_type
10867 sizes and field offsets. */
10868 if (tcctx
.cb
.decl_map
)
10869 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10870 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10874 decl
= OMP_CLAUSE_DECL (c
);
10875 p
= tcctx
.cb
.decl_map
->get (decl
);
10878 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10879 sf
= (tree
) n
->value
;
10880 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10881 src
= build_simple_mem_ref_loc (loc
, sarg
);
10882 src
= omp_build_component_ref (src
, sf
);
10883 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
10884 append_to_statement_list (t
, &list
);
10887 /* Second pass: copy shared var pointers and copy construct non-VLA
10888 firstprivate vars. */
10889 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10890 switch (OMP_CLAUSE_CODE (c
))
10892 case OMP_CLAUSE_SHARED
:
10893 decl
= OMP_CLAUSE_DECL (c
);
10894 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10897 f
= (tree
) n
->value
;
10898 if (tcctx
.cb
.decl_map
)
10899 f
= *tcctx
.cb
.decl_map
->get (f
);
10900 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10901 sf
= (tree
) n
->value
;
10902 if (tcctx
.cb
.decl_map
)
10903 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10904 src
= build_simple_mem_ref_loc (loc
, sarg
);
10905 src
= omp_build_component_ref (src
, sf
);
10906 dst
= build_simple_mem_ref_loc (loc
, arg
);
10907 dst
= omp_build_component_ref (dst
, f
);
10908 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10909 append_to_statement_list (t
, &list
);
10911 case OMP_CLAUSE_FIRSTPRIVATE
:
10912 decl
= OMP_CLAUSE_DECL (c
);
10913 if (is_variable_sized (decl
))
10915 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10918 f
= (tree
) n
->value
;
10919 if (tcctx
.cb
.decl_map
)
10920 f
= *tcctx
.cb
.decl_map
->get (f
);
10921 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10924 sf
= (tree
) n
->value
;
10925 if (tcctx
.cb
.decl_map
)
10926 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10927 src
= build_simple_mem_ref_loc (loc
, sarg
);
10928 src
= omp_build_component_ref (src
, sf
);
10929 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
10930 src
= build_simple_mem_ref_loc (loc
, src
);
10934 dst
= build_simple_mem_ref_loc (loc
, arg
);
10935 dst
= omp_build_component_ref (dst
, f
);
10936 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10937 append_to_statement_list (t
, &list
);
10939 case OMP_CLAUSE_PRIVATE
:
10940 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
10942 decl
= OMP_CLAUSE_DECL (c
);
10943 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10944 f
= (tree
) n
->value
;
10945 if (tcctx
.cb
.decl_map
)
10946 f
= *tcctx
.cb
.decl_map
->get (f
);
10947 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10950 sf
= (tree
) n
->value
;
10951 if (tcctx
.cb
.decl_map
)
10952 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10953 src
= build_simple_mem_ref_loc (loc
, sarg
);
10954 src
= omp_build_component_ref (src
, sf
);
10955 if (use_pointer_for_field (decl
, NULL
))
10956 src
= build_simple_mem_ref_loc (loc
, src
);
10960 dst
= build_simple_mem_ref_loc (loc
, arg
);
10961 dst
= omp_build_component_ref (dst
, f
);
10962 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10963 append_to_statement_list (t
, &list
);
10969 /* Last pass: handle VLA firstprivates. */
10970 if (tcctx
.cb
.decl_map
)
10971 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10972 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10976 decl
= OMP_CLAUSE_DECL (c
);
10977 if (!is_variable_sized (decl
))
10979 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10982 f
= (tree
) n
->value
;
10983 f
= *tcctx
.cb
.decl_map
->get (f
);
10984 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
10985 ind
= DECL_VALUE_EXPR (decl
);
10986 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
10987 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
10988 n
= splay_tree_lookup (ctx
->sfield_map
,
10989 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10990 sf
= (tree
) n
->value
;
10991 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10992 src
= build_simple_mem_ref_loc (loc
, sarg
);
10993 src
= omp_build_component_ref (src
, sf
);
10994 src
= build_simple_mem_ref_loc (loc
, src
);
10995 dst
= build_simple_mem_ref_loc (loc
, arg
);
10996 dst
= omp_build_component_ref (dst
, f
);
10997 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10998 append_to_statement_list (t
, &list
);
10999 n
= splay_tree_lookup (ctx
->field_map
,
11000 (splay_tree_key
) TREE_OPERAND (ind
, 0));
11001 df
= (tree
) n
->value
;
11002 df
= *tcctx
.cb
.decl_map
->get (df
);
11003 ptr
= build_simple_mem_ref_loc (loc
, arg
);
11004 ptr
= omp_build_component_ref (ptr
, df
);
11005 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
11006 build_fold_addr_expr_loc (loc
, dst
));
11007 append_to_statement_list (t
, &list
);
11010 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
11011 append_to_statement_list (t
, &list
);
11013 if (tcctx
.cb
.decl_map
)
11014 delete tcctx
.cb
.decl_map
;
11015 pop_gimplify_context (NULL
);
11016 BIND_EXPR_BODY (bind
) = list
;
11021 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
11025 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
11027 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
11028 OMP_CLAUSE_DEPEND
);
11029 gcc_assert (clauses
);
11030 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11031 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
11032 switch (OMP_CLAUSE_DEPEND_KIND (c
))
11034 case OMP_CLAUSE_DEPEND_IN
:
11037 case OMP_CLAUSE_DEPEND_OUT
:
11038 case OMP_CLAUSE_DEPEND_INOUT
:
11042 gcc_unreachable ();
11044 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
11045 tree array
= create_tmp_var (type
);
11046 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
11048 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
11049 gimple_seq_add_stmt (iseq
, g
);
11050 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
11052 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
11053 gimple_seq_add_stmt (iseq
, g
);
11054 for (i
= 0; i
< 2; i
++)
11056 if ((i
? n_in
: n_out
) == 0)
11058 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11059 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
11060 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
11062 tree t
= OMP_CLAUSE_DECL (c
);
11063 t
= fold_convert (ptr_type_node
, t
);
11064 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
11065 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
11066 NULL_TREE
, NULL_TREE
);
11067 g
= gimple_build_assign (r
, t
);
11068 gimple_seq_add_stmt (iseq
, g
);
11071 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
11072 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
11073 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
11074 OMP_CLAUSE_CHAIN (c
) = *p
;
11076 tree clobber
= build_constructor (type
, NULL
);
11077 TREE_THIS_VOLATILE (clobber
) = 1;
11078 g
= gimple_build_assign (array
, clobber
);
11079 gimple_seq_add_stmt (oseq
, g
);
11082 /* Lower the OpenMP parallel or task directive in the current statement
11083 in GSI_P. CTX holds context information for the directive. */
11086 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11090 gimple stmt
= gsi_stmt (*gsi_p
);
11091 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
11092 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
11093 location_t loc
= gimple_location (stmt
);
11095 clauses
= gimple_omp_taskreg_clauses (stmt
);
11097 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
11098 par_body
= gimple_bind_body (par_bind
);
11099 child_fn
= ctx
->cb
.dst_fn
;
11100 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11101 && !gimple_omp_parallel_combined_p (stmt
))
11103 struct walk_stmt_info wi
;
11106 memset (&wi
, 0, sizeof (wi
));
11108 wi
.val_only
= true;
11109 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
11111 gimple_omp_parallel_set_combined_p (stmt
, true);
11113 gimple_seq dep_ilist
= NULL
;
11114 gimple_seq dep_olist
= NULL
;
11115 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11116 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
11118 push_gimplify_context ();
11119 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11120 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
11123 if (ctx
->srecord_type
)
11124 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
11126 push_gimplify_context ();
11131 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
11132 lower_omp (&par_body
, ctx
);
11133 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
11134 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
11136 /* Declare all the variables created by mapping and the variables
11137 declared in the scope of the parallel body. */
11138 record_vars_into (ctx
->block_vars
, child_fn
);
11139 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
11141 if (ctx
->record_type
)
11144 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
11145 : ctx
->record_type
, ".omp_data_o");
11146 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11147 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11148 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
11153 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
11154 lower_send_shared_vars (&ilist
, &olist
, ctx
);
11156 if (ctx
->record_type
)
11158 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
11159 TREE_THIS_VOLATILE (clobber
) = 1;
11160 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11164 /* Once all the expansions are done, sequence all the different
11165 fragments inside gimple_omp_body. */
11169 if (ctx
->record_type
)
11171 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11172 /* fixup_child_record_type might have changed receiver_decl's type. */
11173 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11174 gimple_seq_add_stmt (&new_body
,
11175 gimple_build_assign (ctx
->receiver_decl
, t
));
11178 gimple_seq_add_seq (&new_body
, par_ilist
);
11179 gimple_seq_add_seq (&new_body
, par_body
);
11180 gimple_seq_add_seq (&new_body
, par_rlist
);
11181 if (ctx
->cancellable
)
11182 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
11183 gimple_seq_add_seq (&new_body
, par_olist
);
11184 new_body
= maybe_catch_exception (new_body
);
11185 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
11186 gimple_seq_add_stmt (&new_body
,
11187 gimple_build_omp_continue (integer_zero_node
,
11188 integer_zero_node
));
11189 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11190 gimple_omp_set_body (stmt
, new_body
);
11192 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
11193 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
11194 gimple_bind_add_seq (bind
, ilist
);
11195 gimple_bind_add_stmt (bind
, stmt
);
11196 gimple_bind_add_seq (bind
, olist
);
11198 pop_gimplify_context (NULL
);
11202 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11203 gimple_bind_add_stmt (dep_bind
, bind
);
11204 gimple_bind_add_seq (dep_bind
, dep_olist
);
11205 pop_gimplify_context (dep_bind
);
11209 /* Lower the GIMPLE_OMP_TARGET in the current statement
11210 in GSI_P. CTX holds context information for the directive. */
11213 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11216 tree child_fn
, t
, c
;
11217 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
11218 gbind
*tgt_bind
, *bind
;
11219 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
11220 location_t loc
= gimple_location (stmt
);
11221 bool offloaded
, data_region
;
11222 unsigned int map_cnt
= 0;
11224 offloaded
= is_gimple_omp_offloaded (stmt
);
11225 switch (gimple_omp_target_kind (stmt
))
11227 case GF_OMP_TARGET_KIND_REGION
:
11228 case GF_OMP_TARGET_KIND_UPDATE
:
11229 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11230 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11231 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11232 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11233 data_region
= false;
11235 case GF_OMP_TARGET_KIND_DATA
:
11236 case GF_OMP_TARGET_KIND_OACC_DATA
:
11237 data_region
= true;
11240 gcc_unreachable ();
11243 clauses
= gimple_omp_target_clauses (stmt
);
11249 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
11250 tgt_body
= gimple_bind_body (tgt_bind
);
11252 else if (data_region
)
11253 tgt_body
= gimple_omp_body (stmt
);
11254 child_fn
= ctx
->cb
.dst_fn
;
11256 push_gimplify_context ();
11261 && is_gimple_omp_oacc (stmt
))
11262 oacc_process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
11264 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11265 switch (OMP_CLAUSE_CODE (c
))
11271 case OMP_CLAUSE_MAP
:
11272 #ifdef ENABLE_CHECKING
11273 /* First check what we're prepared to handle in the following. */
11274 switch (OMP_CLAUSE_MAP_KIND (c
))
11276 case GOMP_MAP_ALLOC
:
11278 case GOMP_MAP_FROM
:
11279 case GOMP_MAP_TOFROM
:
11280 case GOMP_MAP_POINTER
:
11281 case GOMP_MAP_TO_PSET
:
11283 case GOMP_MAP_FORCE_ALLOC
:
11284 case GOMP_MAP_FORCE_TO
:
11285 case GOMP_MAP_FORCE_FROM
:
11286 case GOMP_MAP_FORCE_TOFROM
:
11287 case GOMP_MAP_FORCE_PRESENT
:
11288 case GOMP_MAP_FORCE_DEALLOC
:
11289 case GOMP_MAP_FORCE_DEVICEPTR
:
11290 gcc_assert (is_gimple_omp_oacc (stmt
));
11293 gcc_unreachable ();
11297 case OMP_CLAUSE_TO
:
11298 case OMP_CLAUSE_FROM
:
11299 var
= OMP_CLAUSE_DECL (c
);
11302 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11303 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11308 if (DECL_SIZE (var
)
11309 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
11311 tree var2
= DECL_VALUE_EXPR (var
);
11312 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
11313 var2
= TREE_OPERAND (var2
, 0);
11314 gcc_assert (DECL_P (var2
));
11318 if (!maybe_lookup_field (var
, ctx
))
11323 x
= build_receiver_ref (var
, true, ctx
);
11324 tree new_var
= lookup_decl (var
, ctx
);
11325 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11326 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11327 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11328 x
= build_simple_mem_ref (x
);
11329 SET_DECL_VALUE_EXPR (new_var
, x
);
11330 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11337 target_nesting_level
++;
11338 lower_omp (&tgt_body
, ctx
);
11339 target_nesting_level
--;
11341 else if (data_region
)
11342 lower_omp (&tgt_body
, ctx
);
11346 /* Declare all the variables created by mapping and the variables
11347 declared in the scope of the target body. */
11348 record_vars_into (ctx
->block_vars
, child_fn
);
11349 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
11354 if (ctx
->record_type
)
11357 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
11358 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11359 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11360 t
= make_tree_vec (3);
11361 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
11362 TREE_VEC_ELT (t
, 1)
11363 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
11364 ".omp_data_sizes");
11365 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
11366 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
11367 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
11370 if (is_gimple_omp_oacc (stmt
))
11372 tkind_type
= short_unsigned_type_node
;
11377 tkind_type
= unsigned_char_type_node
;
11380 TREE_VEC_ELT (t
, 2)
11381 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
11382 ".omp_data_kinds");
11383 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
11384 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
11385 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
11386 gimple_omp_target_set_data_arg (stmt
, t
);
11388 vec
<constructor_elt
, va_gc
> *vsize
;
11389 vec
<constructor_elt
, va_gc
> *vkind
;
11390 vec_alloc (vsize
, map_cnt
);
11391 vec_alloc (vkind
, map_cnt
);
11392 unsigned int map_idx
= 0;
11394 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11395 switch (OMP_CLAUSE_CODE (c
))
11401 case OMP_CLAUSE_MAP
:
11402 case OMP_CLAUSE_TO
:
11403 case OMP_CLAUSE_FROM
:
11405 ovar
= OMP_CLAUSE_DECL (c
);
11406 if (!DECL_P (ovar
))
11408 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11409 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11411 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
11412 == get_base_address (ovar
));
11413 nc
= OMP_CLAUSE_CHAIN (c
);
11414 ovar
= OMP_CLAUSE_DECL (nc
);
11418 tree x
= build_sender_ref (ovar
, ctx
);
11420 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
11421 gimplify_assign (x
, v
, &ilist
);
11427 if (DECL_SIZE (ovar
)
11428 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
11430 tree ovar2
= DECL_VALUE_EXPR (ovar
);
11431 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
11432 ovar2
= TREE_OPERAND (ovar2
, 0);
11433 gcc_assert (DECL_P (ovar2
));
11436 if (!maybe_lookup_field (ovar
, ctx
))
11440 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
11441 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
11442 talign
= DECL_ALIGN_UNIT (ovar
);
11445 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11446 tree x
= build_sender_ref (ovar
, ctx
);
11447 if (maybe_lookup_oacc_reduction (var
, ctx
))
11449 gcc_checking_assert (offloaded
11450 && is_gimple_omp_oacc (stmt
));
11451 gimplify_assign (x
, var
, &ilist
);
11453 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11454 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11455 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11456 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
11458 gcc_assert (offloaded
);
11460 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
11461 mark_addressable (avar
);
11462 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
11463 talign
= DECL_ALIGN_UNIT (avar
);
11464 avar
= build_fold_addr_expr (avar
);
11465 gimplify_assign (x
, avar
, &ilist
);
11467 else if (is_gimple_reg (var
))
11469 gcc_assert (offloaded
);
11470 tree avar
= create_tmp_var (TREE_TYPE (var
));
11471 mark_addressable (avar
);
11472 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
11473 if (GOMP_MAP_COPY_TO_P (map_kind
)
11474 || map_kind
== GOMP_MAP_POINTER
11475 || map_kind
== GOMP_MAP_TO_PSET
11476 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11477 gimplify_assign (avar
, var
, &ilist
);
11478 avar
= build_fold_addr_expr (avar
);
11479 gimplify_assign (x
, avar
, &ilist
);
11480 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
11481 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11482 && !TYPE_READONLY (TREE_TYPE (var
)))
11484 x
= build_sender_ref (ovar
, ctx
);
11485 x
= build_simple_mem_ref (x
);
11486 gimplify_assign (var
, x
, &olist
);
11491 var
= build_fold_addr_expr (var
);
11492 gimplify_assign (x
, var
, &ilist
);
11495 tree s
= OMP_CLAUSE_SIZE (c
);
11496 if (s
== NULL_TREE
)
11497 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11498 s
= fold_convert (size_type_node
, s
);
11499 tree purpose
= size_int (map_idx
++);
11500 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11501 if (TREE_CODE (s
) != INTEGER_CST
)
11502 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11504 unsigned HOST_WIDE_INT tkind
;
11505 switch (OMP_CLAUSE_CODE (c
))
11507 case OMP_CLAUSE_MAP
:
11508 tkind
= OMP_CLAUSE_MAP_KIND (c
);
11510 case OMP_CLAUSE_TO
:
11511 tkind
= GOMP_MAP_TO
;
11513 case OMP_CLAUSE_FROM
:
11514 tkind
= GOMP_MAP_FROM
;
11517 gcc_unreachable ();
11519 gcc_checking_assert (tkind
11520 < (HOST_WIDE_INT_C (1U) << talign_shift
));
11521 talign
= ceil_log2 (talign
);
11522 tkind
|= talign
<< talign_shift
;
11523 gcc_checking_assert (tkind
11524 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11525 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
11526 build_int_cstu (tkind_type
, tkind
));
11531 gcc_assert (map_idx
== map_cnt
);
11533 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
11534 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
11535 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
11536 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
11537 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
11539 gimple_seq initlist
= NULL
;
11540 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
11541 TREE_VEC_ELT (t
, 1)),
11542 &initlist
, true, NULL_TREE
);
11543 gimple_seq_add_seq (&ilist
, initlist
);
11545 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
11547 TREE_THIS_VOLATILE (clobber
) = 1;
11548 gimple_seq_add_stmt (&olist
,
11549 gimple_build_assign (TREE_VEC_ELT (t
, 1),
11553 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
11554 TREE_THIS_VOLATILE (clobber
) = 1;
11555 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11559 /* Once all the expansions are done, sequence all the different
11560 fragments inside gimple_omp_body. */
11565 && ctx
->record_type
)
11567 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11568 /* fixup_child_record_type might have changed receiver_decl's type. */
11569 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11570 gimple_seq_add_stmt (&new_body
,
11571 gimple_build_assign (ctx
->receiver_decl
, t
));
11576 gimple_seq_add_seq (&new_body
, tgt_body
);
11577 new_body
= maybe_catch_exception (new_body
);
11579 else if (data_region
)
11580 new_body
= tgt_body
;
11581 if (offloaded
|| data_region
)
11583 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11584 gimple_omp_set_body (stmt
, new_body
);
11587 bind
= gimple_build_bind (NULL
, NULL
,
11588 tgt_bind
? gimple_bind_block (tgt_bind
)
11590 gsi_replace (gsi_p
, bind
, true);
11591 gimple_bind_add_seq (bind
, irlist
);
11592 gimple_bind_add_seq (bind
, ilist
);
11593 gimple_bind_add_stmt (bind
, stmt
);
11594 gimple_bind_add_seq (bind
, olist
);
11595 gimple_bind_add_seq (bind
, orlist
);
11597 pop_gimplify_context (NULL
);
11600 /* Expand code for an OpenMP teams directive. */
11603 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11605 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
11606 push_gimplify_context ();
11608 tree block
= make_node (BLOCK
);
11609 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
11610 gsi_replace (gsi_p
, bind
, true);
11611 gimple_seq bind_body
= NULL
;
11612 gimple_seq dlist
= NULL
;
11613 gimple_seq olist
= NULL
;
11615 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11616 OMP_CLAUSE_NUM_TEAMS
);
11617 if (num_teams
== NULL_TREE
)
11618 num_teams
= build_int_cst (unsigned_type_node
, 0);
11621 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
11622 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
11623 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
11625 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11626 OMP_CLAUSE_THREAD_LIMIT
);
11627 if (thread_limit
== NULL_TREE
)
11628 thread_limit
= build_int_cst (unsigned_type_node
, 0);
11631 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
11632 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
11633 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
11637 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
11638 &bind_body
, &dlist
, ctx
, NULL
);
11639 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
11640 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
11641 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
11643 location_t loc
= gimple_location (teams_stmt
);
11644 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
11645 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
11646 gimple_set_location (call
, loc
);
11647 gimple_seq_add_stmt (&bind_body
, call
);
11649 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
11650 gimple_omp_set_body (teams_stmt
, NULL
);
11651 gimple_seq_add_seq (&bind_body
, olist
);
11652 gimple_seq_add_seq (&bind_body
, dlist
);
11653 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
11654 gimple_bind_set_body (bind
, bind_body
);
11656 pop_gimplify_context (bind
);
11658 gimple_bind_append_vars (bind
, ctx
->block_vars
);
11659 BLOCK_VARS (block
) = ctx
->block_vars
;
11660 if (BLOCK_VARS (block
))
11661 TREE_USED (block
) = 1;
11665 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11666 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11667 of OMP context, but with task_shared_vars set. */
11670 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
11675 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11676 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
11679 if (task_shared_vars
11681 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
11684 /* If a global variable has been privatized, TREE_CONSTANT on
11685 ADDR_EXPR might be wrong. */
11686 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
11687 recompute_tree_invariant_for_addr_expr (t
);
11689 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
11694 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11696 gimple stmt
= gsi_stmt (*gsi_p
);
11697 struct walk_stmt_info wi
;
11700 if (gimple_has_location (stmt
))
11701 input_location
= gimple_location (stmt
);
11703 if (task_shared_vars
)
11704 memset (&wi
, '\0', sizeof (wi
));
11706 /* If we have issued syntax errors, avoid doing any heavy lifting.
11707 Just replace the OMP directives with a NOP to avoid
11708 confusing RTL expansion. */
11709 if (seen_error () && is_gimple_omp (stmt
))
11711 gsi_replace (gsi_p
, gimple_build_nop (), true);
11715 switch (gimple_code (stmt
))
11719 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
11720 if ((ctx
|| task_shared_vars
)
11721 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
11722 lower_omp_regimplify_p
,
11723 ctx
? NULL
: &wi
, NULL
)
11724 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
11725 lower_omp_regimplify_p
,
11726 ctx
? NULL
: &wi
, NULL
)))
11727 gimple_regimplify_operands (cond_stmt
, gsi_p
);
11731 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
11733 case GIMPLE_EH_FILTER
:
11734 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
11737 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
11738 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
11740 case GIMPLE_TRANSACTION
:
11741 lower_omp (gimple_transaction_body_ptr (
11742 as_a
<gtransaction
*> (stmt
)),
11746 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
11748 case GIMPLE_OMP_PARALLEL
:
11749 case GIMPLE_OMP_TASK
:
11750 ctx
= maybe_lookup_ctx (stmt
);
11752 if (ctx
->cancellable
)
11753 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11754 lower_omp_taskreg (gsi_p
, ctx
);
11756 case GIMPLE_OMP_FOR
:
11757 ctx
= maybe_lookup_ctx (stmt
);
11759 if (ctx
->cancellable
)
11760 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11761 lower_omp_for (gsi_p
, ctx
);
11763 case GIMPLE_OMP_SECTIONS
:
11764 ctx
= maybe_lookup_ctx (stmt
);
11766 if (ctx
->cancellable
)
11767 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11768 lower_omp_sections (gsi_p
, ctx
);
11770 case GIMPLE_OMP_SINGLE
:
11771 ctx
= maybe_lookup_ctx (stmt
);
11773 lower_omp_single (gsi_p
, ctx
);
11775 case GIMPLE_OMP_MASTER
:
11776 ctx
= maybe_lookup_ctx (stmt
);
11778 lower_omp_master (gsi_p
, ctx
);
11780 case GIMPLE_OMP_TASKGROUP
:
11781 ctx
= maybe_lookup_ctx (stmt
);
11783 lower_omp_taskgroup (gsi_p
, ctx
);
11785 case GIMPLE_OMP_ORDERED
:
11786 ctx
= maybe_lookup_ctx (stmt
);
11788 lower_omp_ordered (gsi_p
, ctx
);
11790 case GIMPLE_OMP_CRITICAL
:
11791 ctx
= maybe_lookup_ctx (stmt
);
11793 lower_omp_critical (gsi_p
, ctx
);
11795 case GIMPLE_OMP_ATOMIC_LOAD
:
11796 if ((ctx
|| task_shared_vars
)
11797 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
11798 as_a
<gomp_atomic_load
*> (stmt
)),
11799 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
11800 gimple_regimplify_operands (stmt
, gsi_p
);
11802 case GIMPLE_OMP_TARGET
:
11803 ctx
= maybe_lookup_ctx (stmt
);
11805 lower_omp_target (gsi_p
, ctx
);
11807 case GIMPLE_OMP_TEAMS
:
11808 ctx
= maybe_lookup_ctx (stmt
);
11810 lower_omp_teams (gsi_p
, ctx
);
11814 call_stmt
= as_a
<gcall
*> (stmt
);
11815 fndecl
= gimple_call_fndecl (call_stmt
);
11817 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
11818 switch (DECL_FUNCTION_CODE (fndecl
))
11820 case BUILT_IN_GOMP_BARRIER
:
11824 case BUILT_IN_GOMP_CANCEL
:
11825 case BUILT_IN_GOMP_CANCELLATION_POINT
:
11828 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
11829 cctx
= cctx
->outer
;
11830 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
11831 if (!cctx
->cancellable
)
11833 if (DECL_FUNCTION_CODE (fndecl
)
11834 == BUILT_IN_GOMP_CANCELLATION_POINT
)
11836 stmt
= gimple_build_nop ();
11837 gsi_replace (gsi_p
, stmt
, false);
11841 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
11843 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
11844 gimple_call_set_fndecl (call_stmt
, fndecl
);
11845 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
11848 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
11849 gimple_call_set_lhs (call_stmt
, lhs
);
11850 tree fallthru_label
;
11851 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
11853 g
= gimple_build_label (fallthru_label
);
11854 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11855 g
= gimple_build_cond (NE_EXPR
, lhs
,
11856 fold_convert (TREE_TYPE (lhs
),
11857 boolean_false_node
),
11858 cctx
->cancel_label
, fallthru_label
);
11859 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11866 if ((ctx
|| task_shared_vars
)
11867 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
11870 /* Just remove clobbers, this should happen only if we have
11871 "privatized" local addressable variables in SIMD regions,
11872 the clobber isn't needed in that case and gimplifying address
11873 of the ARRAY_REF into a pointer and creating MEM_REF based
11874 clobber would create worse code than we get with the clobber
11876 if (gimple_clobber_p (stmt
))
11878 gsi_replace (gsi_p
, gimple_build_nop (), true);
11881 gimple_regimplify_operands (stmt
, gsi_p
);
11888 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
11890 location_t saved_location
= input_location
;
11891 gimple_stmt_iterator gsi
;
11892 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11893 lower_omp_1 (&gsi
, ctx
);
11894 /* During gimplification, we haven't folded statments inside offloading
11895 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
11896 if (target_nesting_level
|| taskreg_nesting_level
)
11897 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11899 input_location
= saved_location
;
11902 /* Main entry point. */
11904 static unsigned int
11905 execute_lower_omp (void)
11911 /* This pass always runs, to provide PROP_gimple_lomp.
11912 But often, there is nothing to do. */
11913 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
11914 && flag_openmp_simd
== 0)
11917 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
11918 delete_omp_context
);
11920 body
= gimple_body (current_function_decl
);
11921 scan_omp (&body
, NULL
);
11922 gcc_assert (taskreg_nesting_level
== 0);
11923 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
11924 finish_taskreg_scan (ctx
);
11925 taskreg_contexts
.release ();
11927 if (all_contexts
->root
)
11929 if (task_shared_vars
)
11930 push_gimplify_context ();
11931 lower_omp (&body
, NULL
);
11932 if (task_shared_vars
)
11933 pop_gimplify_context (NULL
);
11938 splay_tree_delete (all_contexts
);
11939 all_contexts
= NULL
;
11941 BITMAP_FREE (task_shared_vars
);
11947 const pass_data pass_data_lower_omp
=
11949 GIMPLE_PASS
, /* type */
11950 "omplower", /* name */
11951 OPTGROUP_NONE
, /* optinfo_flags */
11952 TV_NONE
, /* tv_id */
11953 PROP_gimple_any
, /* properties_required */
11954 PROP_gimple_lomp
, /* properties_provided */
11955 0, /* properties_destroyed */
11956 0, /* todo_flags_start */
11957 0, /* todo_flags_finish */
11960 class pass_lower_omp
: public gimple_opt_pass
11963 pass_lower_omp (gcc::context
*ctxt
)
11964 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
11967 /* opt_pass methods: */
11968 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
11970 }; // class pass_lower_omp
11972 } // anon namespace
11975 make_pass_lower_omp (gcc::context
*ctxt
)
11977 return new pass_lower_omp (ctxt
);
11980 /* The following is a utility to diagnose structured block violations.
11981 It is not part of the "omplower" pass, as that's invoked too late. It
11982 should be invoked by the respective front ends after gimplification. */
11984 static splay_tree all_labels
;
11986 /* Check for mismatched contexts and generate an error if needed. Return
11987 true if an error is detected. */
11990 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
11991 gimple branch_ctx
, gimple label_ctx
)
11993 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
11994 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
11996 if (label_ctx
== branch_ctx
)
11999 const char* kind
= NULL
;
12004 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
12005 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
12007 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
12008 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
12009 kind
= "Cilk Plus";
12013 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
12014 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
12016 gcc_checking_assert (kind
== NULL
);
12022 gcc_checking_assert (flag_openmp
);
12027 Previously we kept track of the label's entire context in diagnose_sb_[12]
12028 so we could traverse it and issue a correct "exit" or "enter" error
12029 message upon a structured block violation.
12031 We built the context by building a list with tree_cons'ing, but there is
12032 no easy counterpart in gimple tuples. It seems like far too much work
12033 for issuing exit/enter error messages. If someone really misses the
12034 distinct error message... patches welcome.
12038 /* Try to avoid confusing the user by producing and error message
12039 with correct "exit" or "enter" verbiage. We prefer "exit"
12040 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12041 if (branch_ctx
== NULL
)
12047 if (TREE_VALUE (label_ctx
) == branch_ctx
)
12052 label_ctx
= TREE_CHAIN (label_ctx
);
12057 error ("invalid exit from %s structured block", kind
);
12059 error ("invalid entry to %s structured block", kind
);
12062 /* If it's obvious we have an invalid entry, be specific about the error. */
12063 if (branch_ctx
== NULL
)
12064 error ("invalid entry to %s structured block", kind
);
12067 /* Otherwise, be vague and lazy, but efficient. */
12068 error ("invalid branch to/from %s structured block", kind
);
12071 gsi_replace (gsi_p
, gimple_build_nop (), false);
12075 /* Pass 1: Create a minimal tree of structured blocks, and record
12076 where each label is found. */
12079 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12080 struct walk_stmt_info
*wi
)
12082 gimple context
= (gimple
) wi
->info
;
12083 gimple inner_context
;
12084 gimple stmt
= gsi_stmt (*gsi_p
);
12086 *handled_ops_p
= true;
12088 switch (gimple_code (stmt
))
12092 case GIMPLE_OMP_PARALLEL
:
12093 case GIMPLE_OMP_TASK
:
12094 case GIMPLE_OMP_SECTIONS
:
12095 case GIMPLE_OMP_SINGLE
:
12096 case GIMPLE_OMP_SECTION
:
12097 case GIMPLE_OMP_MASTER
:
12098 case GIMPLE_OMP_ORDERED
:
12099 case GIMPLE_OMP_CRITICAL
:
12100 case GIMPLE_OMP_TARGET
:
12101 case GIMPLE_OMP_TEAMS
:
12102 case GIMPLE_OMP_TASKGROUP
:
12103 /* The minimal context here is just the current OMP construct. */
12104 inner_context
= stmt
;
12105 wi
->info
= inner_context
;
12106 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12107 wi
->info
= context
;
12110 case GIMPLE_OMP_FOR
:
12111 inner_context
= stmt
;
12112 wi
->info
= inner_context
;
12113 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12115 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
12116 diagnose_sb_1
, NULL
, wi
);
12117 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12118 wi
->info
= context
;
12122 splay_tree_insert (all_labels
,
12123 (splay_tree_key
) gimple_label_label (
12124 as_a
<glabel
*> (stmt
)),
12125 (splay_tree_value
) context
);
12135 /* Pass 2: Check each branch and see if its context differs from that of
12136 the destination label's context. */
12139 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12140 struct walk_stmt_info
*wi
)
12142 gimple context
= (gimple
) wi
->info
;
12144 gimple stmt
= gsi_stmt (*gsi_p
);
12146 *handled_ops_p
= true;
12148 switch (gimple_code (stmt
))
12152 case GIMPLE_OMP_PARALLEL
:
12153 case GIMPLE_OMP_TASK
:
12154 case GIMPLE_OMP_SECTIONS
:
12155 case GIMPLE_OMP_SINGLE
:
12156 case GIMPLE_OMP_SECTION
:
12157 case GIMPLE_OMP_MASTER
:
12158 case GIMPLE_OMP_ORDERED
:
12159 case GIMPLE_OMP_CRITICAL
:
12160 case GIMPLE_OMP_TARGET
:
12161 case GIMPLE_OMP_TEAMS
:
12162 case GIMPLE_OMP_TASKGROUP
:
12164 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12165 wi
->info
= context
;
12168 case GIMPLE_OMP_FOR
:
12170 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12172 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
12173 diagnose_sb_2
, NULL
, wi
);
12174 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12175 wi
->info
= context
;
12180 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
12181 tree lab
= gimple_cond_true_label (cond_stmt
);
12184 n
= splay_tree_lookup (all_labels
,
12185 (splay_tree_key
) lab
);
12186 diagnose_sb_0 (gsi_p
, context
,
12187 n
? (gimple
) n
->value
: NULL
);
12189 lab
= gimple_cond_false_label (cond_stmt
);
12192 n
= splay_tree_lookup (all_labels
,
12193 (splay_tree_key
) lab
);
12194 diagnose_sb_0 (gsi_p
, context
,
12195 n
? (gimple
) n
->value
: NULL
);
12202 tree lab
= gimple_goto_dest (stmt
);
12203 if (TREE_CODE (lab
) != LABEL_DECL
)
12206 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12207 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
12211 case GIMPLE_SWITCH
:
12213 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
12215 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
12217 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
12218 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12219 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
12225 case GIMPLE_RETURN
:
12226 diagnose_sb_0 (gsi_p
, context
, NULL
);
12236 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12239 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
12242 gimple last
= last_stmt (bb
);
12243 enum gimple_code code
= gimple_code (last
);
12244 struct omp_region
*cur_region
= *region
;
12245 bool fallthru
= false;
12249 case GIMPLE_OMP_PARALLEL
:
12250 case GIMPLE_OMP_TASK
:
12251 case GIMPLE_OMP_FOR
:
12252 case GIMPLE_OMP_SINGLE
:
12253 case GIMPLE_OMP_TEAMS
:
12254 case GIMPLE_OMP_MASTER
:
12255 case GIMPLE_OMP_TASKGROUP
:
12256 case GIMPLE_OMP_ORDERED
:
12257 case GIMPLE_OMP_CRITICAL
:
12258 case GIMPLE_OMP_SECTION
:
12259 cur_region
= new_omp_region (bb
, code
, cur_region
);
12263 case GIMPLE_OMP_TARGET
:
12264 cur_region
= new_omp_region (bb
, code
, cur_region
);
12266 switch (gimple_omp_target_kind (last
))
12268 case GF_OMP_TARGET_KIND_REGION
:
12269 case GF_OMP_TARGET_KIND_DATA
:
12270 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12271 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12272 case GF_OMP_TARGET_KIND_OACC_DATA
:
12274 case GF_OMP_TARGET_KIND_UPDATE
:
12275 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12276 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12277 cur_region
= cur_region
->outer
;
12280 gcc_unreachable ();
12284 case GIMPLE_OMP_SECTIONS
:
12285 cur_region
= new_omp_region (bb
, code
, cur_region
);
12289 case GIMPLE_OMP_SECTIONS_SWITCH
:
12293 case GIMPLE_OMP_ATOMIC_LOAD
:
12294 case GIMPLE_OMP_ATOMIC_STORE
:
12298 case GIMPLE_OMP_RETURN
:
12299 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12300 somewhere other than the next block. This will be
12302 cur_region
->exit
= bb
;
12303 if (cur_region
->type
== GIMPLE_OMP_TASK
)
12304 /* Add an edge corresponding to not scheduling the task
12306 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
12307 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
12308 cur_region
= cur_region
->outer
;
12311 case GIMPLE_OMP_CONTINUE
:
12312 cur_region
->cont
= bb
;
12313 switch (cur_region
->type
)
12315 case GIMPLE_OMP_FOR
:
12316 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12317 succs edges as abnormal to prevent splitting
12319 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
12320 /* Make the loopback edge. */
12321 make_edge (bb
, single_succ (cur_region
->entry
),
12324 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12325 corresponds to the case that the body of the loop
12326 is not executed at all. */
12327 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
12328 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
12332 case GIMPLE_OMP_SECTIONS
:
12333 /* Wire up the edges into and out of the nested sections. */
12335 basic_block switch_bb
= single_succ (cur_region
->entry
);
12337 struct omp_region
*i
;
12338 for (i
= cur_region
->inner
; i
; i
= i
->next
)
12340 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
12341 make_edge (switch_bb
, i
->entry
, 0);
12342 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
12345 /* Make the loopback edge to the block with
12346 GIMPLE_OMP_SECTIONS_SWITCH. */
12347 make_edge (bb
, switch_bb
, 0);
12349 /* Make the edge from the switch to exit. */
12350 make_edge (switch_bb
, bb
->next_bb
, 0);
12355 case GIMPLE_OMP_TASK
:
12360 gcc_unreachable ();
12365 gcc_unreachable ();
12368 if (*region
!= cur_region
)
12370 *region
= cur_region
;
12372 *region_idx
= cur_region
->entry
->index
;
12380 static unsigned int
12381 diagnose_omp_structured_block_errors (void)
12383 struct walk_stmt_info wi
;
12384 gimple_seq body
= gimple_body (current_function_decl
);
12386 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
12388 memset (&wi
, 0, sizeof (wi
));
12389 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
12391 memset (&wi
, 0, sizeof (wi
));
12392 wi
.want_locations
= true;
12393 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
12395 gimple_set_body (current_function_decl
, body
);
12397 splay_tree_delete (all_labels
);
12405 const pass_data pass_data_diagnose_omp_blocks
=
12407 GIMPLE_PASS
, /* type */
12408 "*diagnose_omp_blocks", /* name */
12409 OPTGROUP_NONE
, /* optinfo_flags */
12410 TV_NONE
, /* tv_id */
12411 PROP_gimple_any
, /* properties_required */
12412 0, /* properties_provided */
12413 0, /* properties_destroyed */
12414 0, /* todo_flags_start */
12415 0, /* todo_flags_finish */
12418 class pass_diagnose_omp_blocks
: public gimple_opt_pass
12421 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12422 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
12425 /* opt_pass methods: */
12426 virtual bool gate (function
*)
12428 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
12430 virtual unsigned int execute (function
*)
12432 return diagnose_omp_structured_block_errors ();
12435 }; // class pass_diagnose_omp_blocks
12437 } // anon namespace
12440 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12442 return new pass_diagnose_omp_blocks (ctxt
);
12445 /* SIMD clone supporting code. */
12447 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12448 of arguments to reserve space for. */
12450 static struct cgraph_simd_clone
*
12451 simd_clone_struct_alloc (int nargs
)
12453 struct cgraph_simd_clone
*clone_info
;
12454 size_t len
= (sizeof (struct cgraph_simd_clone
)
12455 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
12456 clone_info
= (struct cgraph_simd_clone
*)
12457 ggc_internal_cleared_alloc (len
);
12461 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12464 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
12465 struct cgraph_simd_clone
*from
)
12467 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
12468 + ((from
->nargs
- from
->inbranch
)
12469 * sizeof (struct cgraph_simd_clone_arg
))));
12472 /* Return vector of parameter types of function FNDECL. This uses
12473 TYPE_ARG_TYPES if available, otherwise falls back to types of
12474 DECL_ARGUMENTS types. */
12477 simd_clone_vector_of_formal_parm_types (tree fndecl
)
12479 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
12480 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
12481 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
12484 FOR_EACH_VEC_ELT (args
, i
, arg
)
12485 args
[i
] = TREE_TYPE (args
[i
]);
12489 /* Given a simd function in NODE, extract the simd specific
12490 information from the OMP clauses passed in CLAUSES, and return
12491 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12492 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12493 otherwise set to FALSE. */
12495 static struct cgraph_simd_clone
*
12496 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
12497 bool *inbranch_specified
)
12499 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12502 *inbranch_specified
= false;
12504 n
= args
.length ();
12505 if (n
> 0 && args
.last () == void_type_node
)
12508 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12509 be cloned have a distinctive artificial label in addition to "omp
12513 && lookup_attribute ("cilk simd function",
12514 DECL_ATTRIBUTES (node
->decl
)));
12516 /* Allocate one more than needed just in case this is an in-branch
12517 clone which will require a mask argument. */
12518 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
12519 clone_info
->nargs
= n
;
12520 clone_info
->cilk_elemental
= cilk_clone
;
12527 clauses
= TREE_VALUE (clauses
);
12528 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
12531 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
12533 switch (OMP_CLAUSE_CODE (t
))
12535 case OMP_CLAUSE_INBRANCH
:
12536 clone_info
->inbranch
= 1;
12537 *inbranch_specified
= true;
12539 case OMP_CLAUSE_NOTINBRANCH
:
12540 clone_info
->inbranch
= 0;
12541 *inbranch_specified
= true;
12543 case OMP_CLAUSE_SIMDLEN
:
12544 clone_info
->simdlen
12545 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
12547 case OMP_CLAUSE_LINEAR
:
12549 tree decl
= OMP_CLAUSE_DECL (t
);
12550 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
12551 int argno
= TREE_INT_CST_LOW (decl
);
12552 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
12554 clone_info
->args
[argno
].arg_type
12555 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
12556 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12557 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
12558 && clone_info
->args
[argno
].linear_step
< n
);
12562 if (POINTER_TYPE_P (args
[argno
]))
12563 step
= fold_convert (ssizetype
, step
);
12564 if (!tree_fits_shwi_p (step
))
12566 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12567 "ignoring large linear step");
12571 else if (integer_zerop (step
))
12573 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12574 "ignoring zero linear step");
12580 clone_info
->args
[argno
].arg_type
12581 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
12582 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12587 case OMP_CLAUSE_UNIFORM
:
12589 tree decl
= OMP_CLAUSE_DECL (t
);
12590 int argno
= tree_to_uhwi (decl
);
12591 clone_info
->args
[argno
].arg_type
12592 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
12595 case OMP_CLAUSE_ALIGNED
:
12597 tree decl
= OMP_CLAUSE_DECL (t
);
12598 int argno
= tree_to_uhwi (decl
);
12599 clone_info
->args
[argno
].alignment
12600 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
12611 /* Given a SIMD clone in NODE, calculate the characteristic data
12612 type and return the coresponding type. The characteristic data
12613 type is computed as described in the Intel Vector ABI. */
12616 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
12617 struct cgraph_simd_clone
*clone_info
)
12619 tree type
= integer_type_node
;
12620 tree fndecl
= node
->decl
;
12622 /* a) For non-void function, the characteristic data type is the
12624 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
12625 type
= TREE_TYPE (TREE_TYPE (fndecl
));
12627 /* b) If the function has any non-uniform, non-linear parameters,
12628 then the characteristic data type is the type of the first
12632 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
12633 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
12634 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
12642 /* c) If the characteristic data type determined by a) or b) above
12643 is struct, union, or class type which is pass-by-value (except
12644 for the type that maps to the built-in complex data type), the
12645 characteristic data type is int. */
12646 if (RECORD_OR_UNION_TYPE_P (type
)
12647 && !aggregate_value_p (type
, NULL
)
12648 && TREE_CODE (type
) != COMPLEX_TYPE
)
12649 return integer_type_node
;
12651 /* d) If none of the above three classes is applicable, the
12652 characteristic data type is int. */
12656 /* e) For Intel Xeon Phi native and offload compilation, if the
12657 resulting characteristic data type is 8-bit or 16-bit integer
12658 data type, the characteristic data type is int. */
12659 /* Well, we don't handle Xeon Phi yet. */
12663 simd_clone_mangle (struct cgraph_node
*node
,
12664 struct cgraph_simd_clone
*clone_info
)
12666 char vecsize_mangle
= clone_info
->vecsize_mangle
;
12667 char mask
= clone_info
->inbranch
? 'M' : 'N';
12668 unsigned int simdlen
= clone_info
->simdlen
;
12672 gcc_assert (vecsize_mangle
&& simdlen
);
12674 pp_string (&pp
, "_ZGV");
12675 pp_character (&pp
, vecsize_mangle
);
12676 pp_character (&pp
, mask
);
12677 pp_decimal_int (&pp
, simdlen
);
12679 for (n
= 0; n
< clone_info
->nargs
; ++n
)
12681 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
12683 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
12684 pp_character (&pp
, 'u');
12685 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12687 gcc_assert (arg
.linear_step
!= 0);
12688 pp_character (&pp
, 'l');
12689 if (arg
.linear_step
> 1)
12690 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12691 else if (arg
.linear_step
< 0)
12693 pp_character (&pp
, 'n');
12694 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
12698 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
12700 pp_character (&pp
, 's');
12701 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12704 pp_character (&pp
, 'v');
12707 pp_character (&pp
, 'a');
12708 pp_decimal_int (&pp
, arg
.alignment
);
12712 pp_underscore (&pp
);
12713 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
12716 pp_string (&pp
, str
);
12717 str
= pp_formatted_text (&pp
);
12719 /* If there already is a SIMD clone with the same mangled name, don't
12720 add another one. This can happen e.g. for
12721 #pragma omp declare simd
12722 #pragma omp declare simd simdlen(8)
12723 int foo (int, int);
12724 if the simdlen is assumed to be 8 for the first one, etc. */
12725 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
12726 clone
= clone
->simdclone
->next_clone
)
12727 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
12731 return get_identifier (str
);
12734 /* Create a simd clone of OLD_NODE and return it. */
12736 static struct cgraph_node
*
12737 simd_clone_create (struct cgraph_node
*old_node
)
12739 struct cgraph_node
*new_node
;
12740 if (old_node
->definition
)
12742 if (!old_node
->has_gimple_body_p ())
12744 old_node
->get_body ();
12745 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
12751 tree old_decl
= old_node
->decl
;
12752 tree new_decl
= copy_node (old_node
->decl
);
12753 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
12754 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
12755 SET_DECL_RTL (new_decl
, NULL
);
12756 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
12757 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
12758 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
12759 symtab
->call_cgraph_insertion_hooks (new_node
);
12761 if (new_node
== NULL
)
12764 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
12766 /* The function cgraph_function_versioning () will force the new
12767 symbol local. Undo this, and inherit external visability from
12769 new_node
->local
.local
= old_node
->local
.local
;
12770 new_node
->externally_visible
= old_node
->externally_visible
;
12775 /* Adjust the return type of the given function to its appropriate
12776 vector counterpart. Returns a simd array to be used throughout the
12777 function as a return value. */
12780 simd_clone_adjust_return_type (struct cgraph_node
*node
)
12782 tree fndecl
= node
->decl
;
12783 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
12784 unsigned int veclen
;
12787 /* Adjust the function return type. */
12788 if (orig_rettype
== void_type_node
)
12790 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
12791 t
= TREE_TYPE (TREE_TYPE (fndecl
));
12792 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
12793 veclen
= node
->simdclone
->vecsize_int
;
12795 veclen
= node
->simdclone
->vecsize_float
;
12796 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
12797 if (veclen
> node
->simdclone
->simdlen
)
12798 veclen
= node
->simdclone
->simdlen
;
12799 if (POINTER_TYPE_P (t
))
12800 t
= pointer_sized_int_node
;
12801 if (veclen
== node
->simdclone
->simdlen
)
12802 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
12805 t
= build_vector_type (t
, veclen
);
12806 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
12808 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
12809 if (!node
->definition
)
12812 t
= DECL_RESULT (fndecl
);
12813 /* Adjust the DECL_RESULT. */
12814 gcc_assert (TREE_TYPE (t
) != void_type_node
);
12815 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
12818 tree atype
= build_array_type_nelts (orig_rettype
,
12819 node
->simdclone
->simdlen
);
12820 if (veclen
!= node
->simdclone
->simdlen
)
12821 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
12823 /* Set up a SIMD array to use as the return value. */
12824 tree retval
= create_tmp_var_raw (atype
, "retval");
12825 gimple_add_tmp_var (retval
);
12829 /* Each vector argument has a corresponding array to be used locally
12830 as part of the eventual loop. Create such temporary array and
12833 PREFIX is the prefix to be used for the temporary.
12835 TYPE is the inner element type.
12837 SIMDLEN is the number of elements. */
12840 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
12842 tree atype
= build_array_type_nelts (type
, simdlen
);
12843 tree avar
= create_tmp_var_raw (atype
, prefix
);
12844 gimple_add_tmp_var (avar
);
12848 /* Modify the function argument types to their corresponding vector
12849 counterparts if appropriate. Also, create one array for each simd
12850 argument to be used locally when using the function arguments as
12853 NODE is the function whose arguments are to be adjusted.
12855 Returns an adjustment vector that will be filled describing how the
12856 argument types will be adjusted. */
12858 static ipa_parm_adjustment_vec
12859 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
12862 ipa_parm_adjustment_vec adjustments
;
12864 if (node
->definition
)
12865 args
= ipa_get_vector_of_formal_parms (node
->decl
);
12867 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12868 adjustments
.create (args
.length ());
12869 unsigned i
, j
, veclen
;
12870 struct ipa_parm_adjustment adj
;
12871 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
12873 memset (&adj
, 0, sizeof (adj
));
12874 tree parm
= args
[i
];
12875 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
12876 adj
.base_index
= i
;
12879 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
12880 node
->simdclone
->args
[i
].orig_type
= parm_type
;
12882 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
12884 /* No adjustment necessary for scalar arguments. */
12885 adj
.op
= IPA_PARM_OP_COPY
;
12889 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
12890 veclen
= node
->simdclone
->vecsize_int
;
12892 veclen
= node
->simdclone
->vecsize_float
;
12893 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
12894 if (veclen
> node
->simdclone
->simdlen
)
12895 veclen
= node
->simdclone
->simdlen
;
12896 adj
.arg_prefix
= "simd";
12897 if (POINTER_TYPE_P (parm_type
))
12898 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12900 adj
.type
= build_vector_type (parm_type
, veclen
);
12901 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
12902 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12904 adjustments
.safe_push (adj
);
12907 memset (&adj
, 0, sizeof (adj
));
12908 adj
.op
= IPA_PARM_OP_NEW
;
12909 adj
.arg_prefix
= "simd";
12910 adj
.base_index
= i
;
12911 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
12915 if (node
->definition
)
12916 node
->simdclone
->args
[i
].simd_array
12917 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
12918 parm_type
, node
->simdclone
->simdlen
);
12920 adjustments
.safe_push (adj
);
12923 if (node
->simdclone
->inbranch
)
12926 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
12929 memset (&adj
, 0, sizeof (adj
));
12930 adj
.op
= IPA_PARM_OP_NEW
;
12931 adj
.arg_prefix
= "mask";
12933 adj
.base_index
= i
;
12934 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
12935 veclen
= node
->simdclone
->vecsize_int
;
12937 veclen
= node
->simdclone
->vecsize_float
;
12938 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
12939 if (veclen
> node
->simdclone
->simdlen
)
12940 veclen
= node
->simdclone
->simdlen
;
12941 if (POINTER_TYPE_P (base_type
))
12942 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12944 adj
.type
= build_vector_type (base_type
, veclen
);
12945 adjustments
.safe_push (adj
);
12947 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12948 adjustments
.safe_push (adj
);
12950 /* We have previously allocated one extra entry for the mask. Use
12952 struct cgraph_simd_clone
*sc
= node
->simdclone
;
12954 if (node
->definition
)
12956 sc
->args
[i
].orig_arg
12957 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
12958 sc
->args
[i
].simd_array
12959 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
12961 sc
->args
[i
].orig_type
= base_type
;
12962 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
12965 if (node
->definition
)
12966 ipa_modify_formal_parameters (node
->decl
, adjustments
);
12969 tree new_arg_types
= NULL_TREE
, new_reversed
;
12970 bool last_parm_void
= false;
12971 if (args
.length () > 0 && args
.last () == void_type_node
)
12972 last_parm_void
= true;
12974 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
12975 j
= adjustments
.length ();
12976 for (i
= 0; i
< j
; i
++)
12978 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
12980 if (adj
->op
== IPA_PARM_OP_COPY
)
12981 ptype
= args
[adj
->base_index
];
12984 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
12986 new_reversed
= nreverse (new_arg_types
);
12987 if (last_parm_void
)
12990 TREE_CHAIN (new_arg_types
) = void_list_node
;
12992 new_reversed
= void_list_node
;
12995 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
12996 TYPE_ARG_TYPES (new_type
) = new_reversed
;
12997 TREE_TYPE (node
->decl
) = new_type
;
12999 adjustments
.release ();
13002 return adjustments
;
13005 /* Initialize and copy the function arguments in NODE to their
13006 corresponding local simd arrays. Returns a fresh gimple_seq with
13007 the instruction sequence generated. */
13010 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
13011 ipa_parm_adjustment_vec adjustments
)
13013 gimple_seq seq
= NULL
;
13014 unsigned i
= 0, j
= 0, k
;
13016 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
13018 arg
= DECL_CHAIN (arg
), i
++, j
++)
13020 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
13023 node
->simdclone
->args
[i
].vector_arg
= arg
;
13025 tree array
= node
->simdclone
->args
[i
].simd_array
;
13026 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
13028 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
13029 tree ptr
= build_fold_addr_expr (array
);
13030 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
13031 build_int_cst (ptype
, 0));
13032 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
13033 gimplify_and_add (t
, &seq
);
13037 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
13038 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
13039 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
13041 tree ptr
= build_fold_addr_expr (array
);
13045 arg
= DECL_CHAIN (arg
);
13049 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
13050 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
13051 build_int_cst (ptype
, k
* elemsize
));
13052 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
13053 gimplify_and_add (t
, &seq
);
13060 /* Callback info for ipa_simd_modify_stmt_ops below. */
13062 struct modify_stmt_info
{
13063 ipa_parm_adjustment_vec adjustments
;
13065 /* True if the parent statement was modified by
13066 ipa_simd_modify_stmt_ops. */
13070 /* Callback for walk_gimple_op.
13072 Adjust operands from a given statement as specified in the
13073 adjustments vector in the callback data. */
13076 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
13078 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13079 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
13080 tree
*orig_tp
= tp
;
13081 if (TREE_CODE (*tp
) == ADDR_EXPR
)
13082 tp
= &TREE_OPERAND (*tp
, 0);
13083 struct ipa_parm_adjustment
*cand
= NULL
;
13084 if (TREE_CODE (*tp
) == PARM_DECL
)
13085 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
13089 *walk_subtrees
= 0;
13092 tree repl
= NULL_TREE
;
13094 repl
= unshare_expr (cand
->new_decl
);
13099 *walk_subtrees
= 0;
13100 bool modified
= info
->modified
;
13101 info
->modified
= false;
13102 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
13103 if (!info
->modified
)
13105 info
->modified
= modified
;
13108 info
->modified
= modified
;
13117 repl
= build_fold_addr_expr (repl
);
13119 if (is_gimple_debug (info
->stmt
))
13121 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
13122 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
13123 DECL_ARTIFICIAL (vexpr
) = 1;
13124 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
13125 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
13130 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
13131 repl
= gimple_assign_lhs (stmt
);
13133 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
13134 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13137 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
13139 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
13145 info
->modified
= true;
13149 /* Traverse the function body and perform all modifications as
13150 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13151 modified such that the replacement/reduction value will now be an
13152 offset into the corresponding simd_array.
13154 This function will replace all function argument uses with their
13155 corresponding simd array elements, and ajust the return values
13159 ipa_simd_modify_function_body (struct cgraph_node
*node
,
13160 ipa_parm_adjustment_vec adjustments
,
13161 tree retval_array
, tree iter
)
13164 unsigned int i
, j
, l
;
13166 /* Re-use the adjustments array, but this time use it to replace
13167 every function argument use to an offset into the corresponding
13169 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
13171 if (!node
->simdclone
->args
[i
].vector_arg
)
13174 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
13175 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
13176 adjustments
[j
].new_decl
13177 = build4 (ARRAY_REF
,
13179 node
->simdclone
->args
[i
].simd_array
,
13181 NULL_TREE
, NULL_TREE
);
13182 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
13183 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
13184 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
13187 l
= adjustments
.length ();
13188 for (i
= 1; i
< num_ssa_names
; i
++)
13190 tree name
= ssa_name (i
);
13192 && SSA_NAME_VAR (name
)
13193 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
13195 for (j
= 0; j
< l
; j
++)
13196 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
13197 && adjustments
[j
].new_decl
)
13200 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
13203 = copy_var_decl (adjustments
[j
].base
,
13204 DECL_NAME (adjustments
[j
].base
),
13205 TREE_TYPE (adjustments
[j
].base
));
13206 adjustments
[j
].new_ssa_base
= base_var
;
13209 base_var
= adjustments
[j
].new_ssa_base
;
13210 if (SSA_NAME_IS_DEFAULT_DEF (name
))
13212 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13213 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
13214 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
13215 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
13216 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13217 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
13218 gimple stmt
= gimple_build_assign (name
, new_decl
);
13219 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13222 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13227 struct modify_stmt_info info
;
13228 info
.adjustments
= adjustments
;
13230 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
13232 gimple_stmt_iterator gsi
;
13234 gsi
= gsi_start_bb (bb
);
13235 while (!gsi_end_p (gsi
))
13237 gimple stmt
= gsi_stmt (gsi
);
13239 struct walk_stmt_info wi
;
13241 memset (&wi
, 0, sizeof (wi
));
13242 info
.modified
= false;
13244 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
13246 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
13248 tree retval
= gimple_return_retval (return_stmt
);
13251 gsi_remove (&gsi
, true);
13255 /* Replace `return foo' with `retval_array[iter] = foo'. */
13256 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
13257 retval_array
, iter
, NULL
, NULL
);
13258 stmt
= gimple_build_assign (ref
, retval
);
13259 gsi_replace (&gsi
, stmt
, true);
13260 info
.modified
= true;
13265 update_stmt (stmt
);
13266 if (maybe_clean_eh_stmt (stmt
))
13267 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
13274 /* Adjust the argument types in NODE to their appropriate vector
13278 simd_clone_adjust (struct cgraph_node
*node
)
13280 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
13282 targetm
.simd_clone
.adjust (node
);
13284 tree retval
= simd_clone_adjust_return_type (node
);
13285 ipa_parm_adjustment_vec adjustments
13286 = simd_clone_adjust_argument_types (node
);
13288 push_gimplify_context ();
13290 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
13292 /* Adjust all uses of vector arguments accordingly. Adjust all
13293 return values accordingly. */
13294 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
13295 tree iter1
= make_ssa_name (iter
);
13296 tree iter2
= make_ssa_name (iter
);
13297 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
13299 /* Initialize the iteration variable. */
13300 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13301 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
13302 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
13303 /* Insert the SIMD array and iv initialization at function
13305 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
13307 pop_gimplify_context (NULL
);
13309 /* Create a new BB right before the original exit BB, to hold the
13310 iteration increment and the condition/branch. */
13311 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
13312 basic_block incr_bb
= create_empty_bb (orig_exit
);
13313 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
13314 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13315 flag. Set it now to be a FALLTHRU_EDGE. */
13316 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
13317 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
13318 for (unsigned i
= 0;
13319 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
13321 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
13322 redirect_edge_succ (e
, incr_bb
);
13324 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
13325 e
->probability
= REG_BR_PROB_BASE
;
13326 gsi
= gsi_last_bb (incr_bb
);
13327 gimple g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
13328 build_int_cst (unsigned_type_node
, 1));
13329 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13331 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13332 struct loop
*loop
= alloc_loop ();
13333 cfun
->has_force_vectorize_loops
= true;
13334 loop
->safelen
= node
->simdclone
->simdlen
;
13335 loop
->force_vectorize
= true;
13336 loop
->header
= body_bb
;
13338 /* Branch around the body if the mask applies. */
13339 if (node
->simdclone
->inbranch
)
13341 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
13343 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
13344 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
13345 tree aref
= build4 (ARRAY_REF
,
13346 TREE_TYPE (TREE_TYPE (mask_array
)),
13349 g
= gimple_build_assign (mask
, aref
);
13350 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13351 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
13352 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
13354 aref
= build1 (VIEW_CONVERT_EXPR
,
13355 build_nonstandard_integer_type (bitsize
, 0), mask
);
13356 mask
= make_ssa_name (TREE_TYPE (aref
));
13357 g
= gimple_build_assign (mask
, aref
);
13358 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13361 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
13363 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13364 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
13365 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
13368 /* Generate the condition. */
13369 g
= gimple_build_cond (LT_EXPR
,
13371 build_int_cst (unsigned_type_node
,
13372 node
->simdclone
->simdlen
),
13374 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13375 e
= split_block (incr_bb
, gsi_stmt (gsi
));
13376 basic_block latch_bb
= e
->dest
;
13377 basic_block new_exit_bb
;
13378 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
13379 loop
->latch
= latch_bb
;
13381 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
13383 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
13384 /* The successor of incr_bb is already pointing to latch_bb; just
13386 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13387 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
13389 gphi
*phi
= create_phi_node (iter1
, body_bb
);
13390 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
13391 edge latch_edge
= single_succ_edge (latch_bb
);
13392 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
13394 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13396 /* Generate the new return. */
13397 gsi
= gsi_last_bb (new_exit_bb
);
13399 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
13400 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
13401 retval
= TREE_OPERAND (retval
, 0);
13404 retval
= build1 (VIEW_CONVERT_EXPR
,
13405 TREE_TYPE (TREE_TYPE (node
->decl
)),
13407 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
13408 false, GSI_CONTINUE_LINKING
);
13410 g
= gimple_build_return (retval
);
13411 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13413 /* Handle aligned clauses by replacing default defs of the aligned
13414 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13415 lhs. Handle linear by adding PHIs. */
13416 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
13417 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13418 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
13419 || !is_gimple_reg_type
13420 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
13422 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13423 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
13424 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
13427 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
13428 gimple_add_tmp_var (iter1
);
13430 gsi
= gsi_after_labels (entry_bb
);
13431 g
= gimple_build_assign (iter1
, orig_arg
);
13432 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13433 gsi
= gsi_after_labels (body_bb
);
13434 g
= gimple_build_assign (orig_arg
, iter1
);
13435 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13437 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13438 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
13439 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13441 && TREE_ADDRESSABLE
13442 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
13444 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13445 tree def
= ssa_default_def (cfun
, orig_arg
);
13446 if (def
&& !has_zero_uses (def
))
13448 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
13449 gimple_add_tmp_var (iter1
);
13450 gsi
= gsi_after_labels (entry_bb
);
13451 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
13452 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13453 gsi
= gsi_after_labels (body_bb
);
13454 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
13455 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13458 else if (node
->simdclone
->args
[i
].alignment
13459 && node
->simdclone
->args
[i
].arg_type
13460 == SIMD_CLONE_ARG_TYPE_UNIFORM
13461 && (node
->simdclone
->args
[i
].alignment
13462 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
13463 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13466 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
13467 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13468 tree def
= ssa_default_def (cfun
, orig_arg
);
13469 if (def
&& !has_zero_uses (def
))
13471 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
13472 gimple_seq seq
= NULL
;
13473 bool need_cvt
= false;
13475 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
13477 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
13480 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
13481 gimple_call_set_lhs (g
, t
);
13482 gimple_seq_add_stmt_without_update (&seq
, g
);
13485 t
= make_ssa_name (orig_arg
);
13486 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
13487 gimple_seq_add_stmt_without_update (&seq
, g
);
13489 gsi_insert_seq_on_edge_immediate
13490 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
13492 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13493 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
13495 node
->create_edge (cgraph_node::get_create (fn
),
13496 call
, entry_bb
->count
, freq
);
13498 imm_use_iterator iter
;
13499 use_operand_p use_p
;
13501 tree repl
= gimple_get_lhs (g
);
13502 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13503 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
13506 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13507 SET_USE (use_p
, repl
);
13510 else if (node
->simdclone
->args
[i
].arg_type
13511 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
13513 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13514 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13515 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
13516 tree def
= NULL_TREE
;
13517 if (TREE_ADDRESSABLE (orig_arg
))
13519 def
= make_ssa_name (TREE_TYPE (orig_arg
));
13520 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
13521 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
13522 gsi
= gsi_after_labels (entry_bb
);
13523 g
= gimple_build_assign (def
, orig_arg
);
13524 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13528 def
= ssa_default_def (cfun
, orig_arg
);
13529 if (!def
|| has_zero_uses (def
))
13533 iter1
= make_ssa_name (orig_arg
);
13534 iter2
= make_ssa_name (orig_arg
);
13539 phi
= create_phi_node (iter1
, body_bb
);
13540 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
13541 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13542 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13543 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
13544 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13545 ? TREE_TYPE (orig_arg
) : sizetype
;
13547 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
13548 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
13549 gsi
= gsi_last_bb (incr_bb
);
13550 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13552 imm_use_iterator iter
;
13553 use_operand_p use_p
;
13555 if (TREE_ADDRESSABLE (orig_arg
))
13557 gsi
= gsi_after_labels (body_bb
);
13558 g
= gimple_build_assign (orig_arg
, iter1
);
13559 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13562 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13563 if (use_stmt
== phi
)
13566 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13567 SET_USE (use_p
, iter1
);
13571 calculate_dominance_info (CDI_DOMINATORS
);
13572 add_loop (loop
, loop
->header
->loop_father
);
13573 update_ssa (TODO_update_ssa
);
13578 /* If the function in NODE is tagged as an elemental SIMD function,
13579 create the appropriate SIMD clones. */
13582 expand_simd_clones (struct cgraph_node
*node
)
13584 tree attr
= lookup_attribute ("omp declare simd",
13585 DECL_ATTRIBUTES (node
->decl
));
13586 if (attr
== NULL_TREE
13587 || node
->global
.inlined_to
13588 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
13592 #pragma omp declare simd
13594 in C, there we don't know the argument types at all. */
13595 if (!node
->definition
13596 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
13601 /* Start with parsing the "omp declare simd" attribute(s). */
13602 bool inbranch_clause_specified
;
13603 struct cgraph_simd_clone
*clone_info
13604 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
13605 &inbranch_clause_specified
);
13606 if (clone_info
== NULL
)
13609 int orig_simdlen
= clone_info
->simdlen
;
13610 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
13611 /* The target can return 0 (no simd clones should be created),
13612 1 (just one ISA of simd clones should be created) or higher
13613 count of ISA variants. In that case, clone_info is initialized
13614 for the first ISA variant. */
13616 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
13621 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13622 also create one inbranch and one !inbranch clone of it. */
13623 for (int i
= 0; i
< count
* 2; i
++)
13625 struct cgraph_simd_clone
*clone
= clone_info
;
13626 if (inbranch_clause_specified
&& (i
& 1) != 0)
13631 clone
= simd_clone_struct_alloc (clone_info
->nargs
13633 simd_clone_struct_copy (clone
, clone_info
);
13634 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13635 and simd_clone_adjust_argument_types did to the first
13637 clone
->nargs
-= clone_info
->inbranch
;
13638 clone
->simdlen
= orig_simdlen
;
13639 /* And call the target hook again to get the right ISA. */
13640 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
13644 clone
->inbranch
= 1;
13647 /* simd_clone_mangle might fail if such a clone has been created
13649 tree id
= simd_clone_mangle (node
, clone
);
13650 if (id
== NULL_TREE
)
13653 /* Only when we are sure we want to create the clone actually
13654 clone the function (or definitions) or create another
13655 extern FUNCTION_DECL (for prototypes without definitions). */
13656 struct cgraph_node
*n
= simd_clone_create (node
);
13660 n
->simdclone
= clone
;
13661 clone
->origin
= node
;
13662 clone
->next_clone
= NULL
;
13663 if (node
->simd_clones
== NULL
)
13665 clone
->prev_clone
= n
;
13666 node
->simd_clones
= n
;
13670 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
13671 clone
->prev_clone
->simdclone
->next_clone
= n
;
13672 node
->simd_clones
->simdclone
->prev_clone
= n
;
13674 symtab
->change_decl_assembler_name (n
->decl
, id
);
13675 /* And finally adjust the return type, parameters and for
13676 definitions also function body. */
13677 if (node
->definition
)
13678 simd_clone_adjust (n
);
13681 simd_clone_adjust_return_type (n
);
13682 simd_clone_adjust_argument_types (n
);
13686 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
13689 /* Entry point for IPA simd clone creation pass. */
13691 static unsigned int
13692 ipa_omp_simd_clone (void)
13694 struct cgraph_node
*node
;
13695 FOR_EACH_FUNCTION (node
)
13696 expand_simd_clones (node
);
13702 const pass_data pass_data_omp_simd_clone
=
13704 SIMPLE_IPA_PASS
, /* type */
13705 "simdclone", /* name */
13706 OPTGROUP_NONE
, /* optinfo_flags */
13707 TV_NONE
, /* tv_id */
13708 ( PROP_ssa
| PROP_cfg
), /* properties_required */
13709 0, /* properties_provided */
13710 0, /* properties_destroyed */
13711 0, /* todo_flags_start */
13712 0, /* todo_flags_finish */
13715 class pass_omp_simd_clone
: public simple_ipa_opt_pass
13718 pass_omp_simd_clone(gcc::context
*ctxt
)
13719 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
13722 /* opt_pass methods: */
13723 virtual bool gate (function
*);
13724 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
13728 pass_omp_simd_clone::gate (function
*)
13730 return ((flag_openmp
|| flag_openmp_simd
13732 || (in_lto_p
&& !flag_wpa
))
13733 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
13736 } // anon namespace
13738 simple_ipa_opt_pass
*
13739 make_pass_omp_simd_clone (gcc::context
*ctxt
)
13741 return new pass_omp_simd_clone (ctxt
);
13744 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
13745 adds their addresses and sizes to constructor-vector V_CTOR. */
13747 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
13748 vec
<constructor_elt
, va_gc
> *v_ctor
)
13750 unsigned len
= vec_safe_length (v_decls
);
13751 for (unsigned i
= 0; i
< len
; i
++)
13753 tree it
= (*v_decls
)[i
];
13754 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
13756 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
13758 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
13759 fold_convert (const_ptr_type_node
,
13760 DECL_SIZE_UNIT (it
)));
13764 /* Create new symbols containing (address, size) pairs for global variables,
13765 marked with "omp declare target" attribute, as well as addresses for the
13766 functions, which are outlined offloading regions. */
13768 omp_finish_file (void)
13770 unsigned num_funcs
= vec_safe_length (offload_funcs
);
13771 unsigned num_vars
= vec_safe_length (offload_vars
);
13773 if (num_funcs
== 0 && num_vars
== 0)
13776 if (targetm_common
.have_named_sections
)
13778 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
13779 vec_alloc (v_f
, num_funcs
);
13780 vec_alloc (v_v
, num_vars
* 2);
13782 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
13783 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
13785 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13787 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13789 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13790 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13791 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
13792 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
13793 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
13794 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
13795 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13796 get_identifier (".offload_func_table"),
13798 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13799 get_identifier (".offload_var_table"),
13801 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
13802 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
13803 otherwise a joint table in a binary will contain padding between
13804 tables from multiple object files. */
13805 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
13806 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
13807 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
13808 DECL_INITIAL (funcs_decl
) = ctor_f
;
13809 DECL_INITIAL (vars_decl
) = ctor_v
;
13810 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
13811 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
13813 varpool_node::finalize_decl (vars_decl
);
13814 varpool_node::finalize_decl (funcs_decl
);
13818 for (unsigned i
= 0; i
< num_funcs
; i
++)
13820 tree it
= (*offload_funcs
)[i
];
13821 targetm
.record_offload_symbol (it
);
13823 for (unsigned i
= 0; i
< num_vars
; i
++)
13825 tree it
= (*offload_vars
)[i
];
13826 targetm
.record_offload_symbol (it
);
13831 #include "gt-omp-low.h"