1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
33 #include "fold-const.h"
34 #include "stringpool.h"
35 #include "stor-layout.h"
38 #include "hard-reg-set.h"
40 #include "dominance.h"
43 #include "basic-block.h"
44 #include "tree-ssa-alias.h"
45 #include "internal-fn.h"
46 #include "gimple-fold.h"
47 #include "gimple-expr.h"
51 #include "gimple-iterator.h"
52 #include "gimplify-me.h"
53 #include "gimple-walk.h"
54 #include "tree-iterator.h"
55 #include "tree-inline.h"
56 #include "langhooks.h"
57 #include "diagnostic-core.h"
58 #include "gimple-ssa.h"
59 #include "plugin-api.h"
63 #include "tree-phinodes.h"
64 #include "ssa-iterators.h"
65 #include "tree-ssanames.h"
66 #include "tree-into-ssa.h"
68 #include "insn-config.h"
79 #include "tree-pass.h"
81 #include "splay-tree.h"
82 #include "insn-codes.h"
86 #include "common/common-target.h"
88 #include "gimple-low.h"
89 #include "tree-cfgcleanup.h"
90 #include "pretty-print.h"
91 #include "alloc-pool.h"
92 #include "symbol-summary.h"
94 #include "tree-nested.h"
98 #include "lto-section-names.h"
99 #include "gomp-constants.h"
102 /* Lowering of OMP parallel and workshare constructs proceeds in two
103 phases. The first phase scans the function looking for OMP statements
104 and then for variables that must be replaced to satisfy data sharing
105 clauses. The second phase expands code for the constructs, as well as
106 re-gimplifying things when variables have been replaced with complex
109 Final code generation is done by pass_expand_omp. The flowgraph is
110 scanned for regions which are then moved to a new
111 function, to be invoked by the thread library, or offloaded. */
113 /* OMP region information. Every parallel and workshare
114 directive is enclosed between two markers, the OMP_* directive
115 and a corresponding OMP_RETURN statement. */
119 /* The enclosing region. */
120 struct omp_region
*outer
;
122 /* First child region. */
123 struct omp_region
*inner
;
125 /* Next peer region. */
126 struct omp_region
*next
;
128 /* Block containing the omp directive as its last stmt. */
131 /* Block containing the OMP_RETURN as its last stmt. */
134 /* Block containing the OMP_CONTINUE as its last stmt. */
137 /* If this is a combined parallel+workshare region, this is a list
138 of additional arguments needed by the combined parallel+workshare
140 vec
<tree
, va_gc
> *ws_args
;
142 /* The code for the omp directive of this region. */
143 enum gimple_code type
;
145 /* Schedule kind, only used for OMP_FOR type regions. */
146 enum omp_clause_schedule_kind sched_kind
;
148 /* True if this is a combined parallel+workshare region. */
149 bool is_combined_parallel
;
152 /* Levels of parallelism as defined by OpenACC. Increasing numbers
153 correspond to deeper loop nesting levels. */
155 #define MASK_WORKER 2
156 #define MASK_VECTOR 4
158 /* Context structure. Used to store information about each parallel
159 directive in the code. */
161 typedef struct omp_context
163 /* This field must be at the beginning, as we do "inheritance": Some
164 callback functions for tree-inline.c (e.g., omp_copy_decl)
165 receive a copy_body_data pointer that is up-casted to an
166 omp_context pointer. */
169 /* The tree of contexts corresponding to the encountered constructs. */
170 struct omp_context
*outer
;
173 /* Map variables to fields in a structure that allows communication
174 between sending and receiving threads. */
175 splay_tree field_map
;
180 /* These are used just by task contexts, if task firstprivate fn is
181 needed. srecord_type is used to communicate from the thread
182 that encountered the task construct to task firstprivate fn,
183 record_type is allocated by GOMP_task, initialized by task firstprivate
184 fn and passed to the task body fn. */
185 splay_tree sfield_map
;
188 /* A chain of variables to add to the top-level block surrounding the
189 construct. In the case of a parallel, this is in the child function. */
192 /* A map of reduction pointer variables. For accelerators, each
193 reduction variable is replaced with an array. Each thread, in turn,
194 is assigned to a slot on that array. */
195 splay_tree reduction_map
;
197 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
198 barriers should jump to during omplower pass. */
201 /* What to do with variables with implicitly determined sharing
203 enum omp_clause_default_kind default_kind
;
205 /* Nesting depth of this context. Used to beautify error messages re
206 invalid gotos. The outermost ctx is depth 1, with depth 0 being
207 reserved for the main body of the function. */
210 /* True if this parallel directive is nested within another. */
213 /* True if this construct can be cancelled. */
216 /* For OpenACC loops, a mask of gang, worker and vector used at
217 levels below this one. */
219 /* For OpenACC loops, a mask of gang, worker and vector used at
220 this level and above. For parallel and kernels clauses, a mask
221 indicating which of num_gangs/num_workers/num_vectors was used. */
225 /* A structure holding the elements of:
226 for (V = N1; V cond N2; V += STEP) [...] */
228 struct omp_for_data_loop
230 tree v
, n1
, n2
, step
;
231 enum tree_code cond_code
;
234 /* A structure describing the main elements of a parallel loop. */
238 struct omp_for_data_loop loop
;
243 bool have_nowait
, have_ordered
;
244 enum omp_clause_schedule_kind sched_kind
;
245 struct omp_for_data_loop
*loops
;
249 static splay_tree all_contexts
;
250 static int taskreg_nesting_level
;
251 static int target_nesting_level
;
252 static struct omp_region
*root_omp_region
;
253 static bitmap task_shared_vars
;
254 static vec
<omp_context
*> taskreg_contexts
;
256 static void scan_omp (gimple_seq
*, omp_context
*);
257 static tree
scan_omp_1_op (tree
*, int *, void *);
259 #define WALK_SUBSTMTS \
263 case GIMPLE_EH_FILTER: \
264 case GIMPLE_TRANSACTION: \
265 /* The sub-statements for these should be walked. */ \
266 *handled_ops_p = false; \
269 /* Helper function to get the name of the array containing the partial
270 reductions for OpenACC reductions. */
272 oacc_get_reduction_array_id (tree node
)
274 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
275 int len
= strlen ("OACC") + strlen (id
);
276 char *temp_name
= XALLOCAVEC (char, len
+ 1);
277 snprintf (temp_name
, len
+ 1, "OACC%s", id
);
278 return IDENTIFIER_POINTER (get_identifier (temp_name
));
281 /* Determine the number of threads OpenACC threads used to determine the
282 size of the array of partial reductions. Currently, this is num_gangs
283 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
284 because it is independed of the device used. */
287 oacc_max_threads (omp_context
*ctx
)
289 tree nthreads
, vector_length
, gangs
, clauses
;
291 gangs
= fold_convert (sizetype
, integer_one_node
);
292 vector_length
= gangs
;
294 /* The reduction clause may be nested inside a loop directive.
295 Scan for the innermost vector_length clause. */
296 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
298 if (gimple_code (oc
->stmt
) != GIMPLE_OMP_TARGET
299 || (gimple_omp_target_kind (oc
->stmt
)
300 != GF_OMP_TARGET_KIND_OACC_PARALLEL
))
303 clauses
= gimple_omp_target_clauses (oc
->stmt
);
305 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
307 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
309 OMP_CLAUSE_VECTOR_LENGTH_EXPR
312 vector_length
= fold_convert (sizetype
, integer_one_node
);
314 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
316 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
317 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
319 gangs
= fold_convert (sizetype
, integer_one_node
);
324 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
329 /* Holds offload tables with decls. */
330 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
332 /* Convenience function for calling scan_omp_1_op on tree operands. */
335 scan_omp_op (tree
*tp
, omp_context
*ctx
)
337 struct walk_stmt_info wi
;
339 memset (&wi
, 0, sizeof (wi
));
341 wi
.want_locations
= true;
343 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
346 static void lower_omp (gimple_seq
*, omp_context
*);
347 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
348 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
350 /* Find an OMP clause of type KIND within CLAUSES. */
353 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
355 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
356 if (OMP_CLAUSE_CODE (clauses
) == kind
)
362 /* Return true if CTX is for an omp parallel. */
365 is_parallel_ctx (omp_context
*ctx
)
367 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
371 /* Return true if CTX is for an omp task. */
374 is_task_ctx (omp_context
*ctx
)
376 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
380 /* Return true if CTX is for an omp parallel or omp task. */
383 is_taskreg_ctx (omp_context
*ctx
)
385 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
386 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
390 /* Return true if REGION is a combined parallel+workshare region. */
393 is_combined_parallel (struct omp_region
*region
)
395 return region
->is_combined_parallel
;
399 /* Extract the header elements of parallel loop FOR_STMT and store
403 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
404 struct omp_for_data_loop
*loops
)
406 tree t
, var
, *collapse_iter
, *collapse_count
;
407 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
408 struct omp_for_data_loop
*loop
;
410 struct omp_for_data_loop dummy_loop
;
411 location_t loc
= gimple_location (for_stmt
);
412 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
413 bool distribute
= gimple_omp_for_kind (for_stmt
)
414 == GF_OMP_FOR_KIND_DISTRIBUTE
;
416 fd
->for_stmt
= for_stmt
;
418 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
419 if (fd
->collapse
> 1)
422 fd
->loops
= &fd
->loop
;
424 fd
->have_nowait
= distribute
|| simd
;
425 fd
->have_ordered
= false;
426 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
427 fd
->chunk_size
= NULL_TREE
;
428 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
429 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
430 collapse_iter
= NULL
;
431 collapse_count
= NULL
;
433 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
434 switch (OMP_CLAUSE_CODE (t
))
436 case OMP_CLAUSE_NOWAIT
:
437 fd
->have_nowait
= true;
439 case OMP_CLAUSE_ORDERED
:
440 fd
->have_ordered
= true;
442 case OMP_CLAUSE_SCHEDULE
:
443 gcc_assert (!distribute
);
444 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
445 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
447 case OMP_CLAUSE_DIST_SCHEDULE
:
448 gcc_assert (distribute
);
449 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
451 case OMP_CLAUSE_COLLAPSE
:
452 if (fd
->collapse
> 1)
454 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
455 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
462 /* FIXME: for now map schedule(auto) to schedule(static).
463 There should be analysis to determine whether all iterations
464 are approximately the same amount of work (then schedule(static)
465 is best) or if it varies (then schedule(dynamic,N) is better). */
466 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
468 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
469 gcc_assert (fd
->chunk_size
== NULL
);
471 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
472 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
473 gcc_assert (fd
->chunk_size
== NULL
);
474 else if (fd
->chunk_size
== NULL
)
476 /* We only need to compute a default chunk size for ordered
477 static loops and dynamic loops. */
478 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
480 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
481 ? integer_zero_node
: integer_one_node
;
484 for (i
= 0; i
< fd
->collapse
; i
++)
486 if (fd
->collapse
== 1)
488 else if (loops
!= NULL
)
493 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
494 gcc_assert (SSA_VAR_P (loop
->v
));
495 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
496 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
497 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
498 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
500 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
501 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
502 switch (loop
->cond_code
)
508 gcc_assert (gimple_omp_for_kind (for_stmt
)
509 == GF_OMP_FOR_KIND_CILKSIMD
510 || (gimple_omp_for_kind (for_stmt
)
511 == GF_OMP_FOR_KIND_CILKFOR
));
514 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
515 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
517 loop
->n2
= fold_build2_loc (loc
,
518 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
519 build_int_cst (TREE_TYPE (loop
->n2
), 1));
520 loop
->cond_code
= LT_EXPR
;
523 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
524 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
526 loop
->n2
= fold_build2_loc (loc
,
527 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
528 build_int_cst (TREE_TYPE (loop
->n2
), 1));
529 loop
->cond_code
= GT_EXPR
;
535 t
= gimple_omp_for_incr (for_stmt
, i
);
536 gcc_assert (TREE_OPERAND (t
, 0) == var
);
537 switch (TREE_CODE (t
))
540 loop
->step
= TREE_OPERAND (t
, 1);
542 case POINTER_PLUS_EXPR
:
543 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
546 loop
->step
= TREE_OPERAND (t
, 1);
547 loop
->step
= fold_build1_loc (loc
,
548 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
556 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
557 && !fd
->have_ordered
))
559 if (fd
->collapse
== 1)
560 iter_type
= TREE_TYPE (loop
->v
);
562 || TYPE_PRECISION (iter_type
)
563 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
565 = build_nonstandard_integer_type
566 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
568 else if (iter_type
!= long_long_unsigned_type_node
)
570 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
571 iter_type
= long_long_unsigned_type_node
;
572 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
573 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
574 >= TYPE_PRECISION (iter_type
))
578 if (loop
->cond_code
== LT_EXPR
)
579 n
= fold_build2_loc (loc
,
580 PLUS_EXPR
, TREE_TYPE (loop
->v
),
581 loop
->n2
, loop
->step
);
584 if (TREE_CODE (n
) != INTEGER_CST
585 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
586 iter_type
= long_long_unsigned_type_node
;
588 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
589 > TYPE_PRECISION (iter_type
))
593 if (loop
->cond_code
== LT_EXPR
)
596 n2
= fold_build2_loc (loc
,
597 PLUS_EXPR
, TREE_TYPE (loop
->v
),
598 loop
->n2
, loop
->step
);
602 n1
= fold_build2_loc (loc
,
603 MINUS_EXPR
, TREE_TYPE (loop
->v
),
604 loop
->n2
, loop
->step
);
607 if (TREE_CODE (n1
) != INTEGER_CST
608 || TREE_CODE (n2
) != INTEGER_CST
609 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
610 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
611 iter_type
= long_long_unsigned_type_node
;
615 if (collapse_count
&& *collapse_count
== NULL
)
617 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
618 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
619 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
620 if (t
&& integer_zerop (t
))
621 count
= build_zero_cst (long_long_unsigned_type_node
);
622 else if ((i
== 0 || count
!= NULL_TREE
)
623 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
624 && TREE_CONSTANT (loop
->n1
)
625 && TREE_CONSTANT (loop
->n2
)
626 && TREE_CODE (loop
->step
) == INTEGER_CST
)
628 tree itype
= TREE_TYPE (loop
->v
);
630 if (POINTER_TYPE_P (itype
))
631 itype
= signed_type_for (itype
);
632 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
633 t
= fold_build2_loc (loc
,
635 fold_convert_loc (loc
, itype
, loop
->step
), t
);
636 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
637 fold_convert_loc (loc
, itype
, loop
->n2
));
638 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
639 fold_convert_loc (loc
, itype
, loop
->n1
));
640 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
641 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
642 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
643 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
644 fold_convert_loc (loc
, itype
,
647 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
648 fold_convert_loc (loc
, itype
, loop
->step
));
649 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
650 if (count
!= NULL_TREE
)
651 count
= fold_build2_loc (loc
,
652 MULT_EXPR
, long_long_unsigned_type_node
,
656 if (TREE_CODE (count
) != INTEGER_CST
)
659 else if (count
&& !integer_zerop (count
))
666 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
667 || fd
->have_ordered
))
669 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
670 iter_type
= long_long_unsigned_type_node
;
672 iter_type
= long_integer_type_node
;
674 else if (collapse_iter
&& *collapse_iter
!= NULL
)
675 iter_type
= TREE_TYPE (*collapse_iter
);
676 fd
->iter_type
= iter_type
;
677 if (collapse_iter
&& *collapse_iter
== NULL
)
678 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
679 if (collapse_count
&& *collapse_count
== NULL
)
682 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
684 *collapse_count
= create_tmp_var (iter_type
, ".count");
687 if (fd
->collapse
> 1)
689 fd
->loop
.v
= *collapse_iter
;
690 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
691 fd
->loop
.n2
= *collapse_count
;
692 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
693 fd
->loop
.cond_code
= LT_EXPR
;
696 /* For OpenACC loops, force a chunk size of one, as this avoids the default
697 scheduling where several subsequent iterations are being executed by the
699 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
701 gcc_assert (fd
->chunk_size
== NULL_TREE
);
702 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
707 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
708 is the immediate dominator of PAR_ENTRY_BB, return true if there
709 are no data dependencies that would prevent expanding the parallel
710 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
712 When expanding a combined parallel+workshare region, the call to
713 the child function may need additional arguments in the case of
714 GIMPLE_OMP_FOR regions. In some cases, these arguments are
715 computed out of variables passed in from the parent to the child
716 via 'struct .omp_data_s'. For instance:
718 #pragma omp parallel for schedule (guided, i * 4)
723 # BLOCK 2 (PAR_ENTRY_BB)
725 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
727 # BLOCK 3 (WS_ENTRY_BB)
728 .omp_data_i = &.omp_data_o;
729 D.1667 = .omp_data_i->i;
731 #pragma omp for schedule (guided, D.1598)
733 When we outline the parallel region, the call to the child function
734 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
735 that value is computed *after* the call site. So, in principle we
736 cannot do the transformation.
738 To see whether the code in WS_ENTRY_BB blocks the combined
739 parallel+workshare call, we collect all the variables used in the
740 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
741 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
744 FIXME. If we had the SSA form built at this point, we could merely
745 hoist the code in block 3 into block 2 and be done with it. But at
746 this point we don't have dataflow information and though we could
747 hack something up here, it is really not worth the aggravation. */
750 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
752 struct omp_for_data fd
;
753 gimple ws_stmt
= last_stmt (ws_entry_bb
);
755 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
758 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
760 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
762 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
764 if (fd
.iter_type
!= long_integer_type_node
)
767 /* FIXME. We give up too easily here. If any of these arguments
768 are not constants, they will likely involve variables that have
769 been mapped into fields of .omp_data_s for sharing with the child
770 function. With appropriate data flow, it would be possible to
772 if (!is_gimple_min_invariant (fd
.loop
.n1
)
773 || !is_gimple_min_invariant (fd
.loop
.n2
)
774 || !is_gimple_min_invariant (fd
.loop
.step
)
775 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
782 /* Collect additional arguments needed to emit a combined
783 parallel+workshare call. WS_STMT is the workshare directive being
786 static vec
<tree
, va_gc
> *
787 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
790 location_t loc
= gimple_location (ws_stmt
);
791 vec
<tree
, va_gc
> *ws_args
;
793 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
795 struct omp_for_data fd
;
798 extract_omp_for_data (for_stmt
, &fd
, NULL
);
802 if (gimple_omp_for_combined_into_p (for_stmt
))
805 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
806 OMP_CLAUSE__LOOPTEMP_
);
808 n1
= OMP_CLAUSE_DECL (innerc
);
809 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
810 OMP_CLAUSE__LOOPTEMP_
);
812 n2
= OMP_CLAUSE_DECL (innerc
);
815 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
817 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
818 ws_args
->quick_push (t
);
820 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
821 ws_args
->quick_push (t
);
823 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
824 ws_args
->quick_push (t
);
828 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
829 ws_args
->quick_push (t
);
834 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
836 /* Number of sections is equal to the number of edges from the
837 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
838 the exit of the sections region. */
839 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
840 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
841 vec_alloc (ws_args
, 1);
842 ws_args
->quick_push (t
);
850 /* Discover whether REGION is a combined parallel+workshare region. */
853 determine_parallel_type (struct omp_region
*region
)
855 basic_block par_entry_bb
, par_exit_bb
;
856 basic_block ws_entry_bb
, ws_exit_bb
;
858 if (region
== NULL
|| region
->inner
== NULL
859 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
860 || region
->inner
->cont
== NULL
)
863 /* We only support parallel+for and parallel+sections. */
864 if (region
->type
!= GIMPLE_OMP_PARALLEL
865 || (region
->inner
->type
!= GIMPLE_OMP_FOR
866 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
869 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
870 WS_EXIT_BB -> PAR_EXIT_BB. */
871 par_entry_bb
= region
->entry
;
872 par_exit_bb
= region
->exit
;
873 ws_entry_bb
= region
->inner
->entry
;
874 ws_exit_bb
= region
->inner
->exit
;
876 if (single_succ (par_entry_bb
) == ws_entry_bb
877 && single_succ (ws_exit_bb
) == par_exit_bb
878 && workshare_safe_to_combine_p (ws_entry_bb
)
879 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
880 || (last_and_only_stmt (ws_entry_bb
)
881 && last_and_only_stmt (par_exit_bb
))))
883 gimple par_stmt
= last_stmt (par_entry_bb
);
884 gimple ws_stmt
= last_stmt (ws_entry_bb
);
886 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
888 /* If this is a combined parallel loop, we need to determine
889 whether or not to use the combined library calls. There
890 are two cases where we do not apply the transformation:
891 static loops and any kind of ordered loop. In the first
892 case, we already open code the loop so there is no need
893 to do anything else. In the latter case, the combined
894 parallel loop call would still need extra synchronization
895 to implement ordered semantics, so there would not be any
896 gain in using the combined call. */
897 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
898 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
900 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
901 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
903 region
->is_combined_parallel
= false;
904 region
->inner
->is_combined_parallel
= false;
909 region
->is_combined_parallel
= true;
910 region
->inner
->is_combined_parallel
= true;
911 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
916 /* Return true if EXPR is variable sized. */
919 is_variable_sized (const_tree expr
)
921 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
924 /* Return true if DECL is a reference type. */
927 is_reference (tree decl
)
929 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
932 /* Return the type of a decl. If the decl is reference type,
933 return its base type. */
935 get_base_type (tree decl
)
937 tree type
= TREE_TYPE (decl
);
938 if (is_reference (decl
))
939 type
= TREE_TYPE (type
);
943 /* Lookup variables. The "maybe" form
944 allows for the variable form to not have been entered, otherwise we
945 assert that the variable must have been entered. */
948 lookup_decl (tree var
, omp_context
*ctx
)
950 tree
*n
= ctx
->cb
.decl_map
->get (var
);
955 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
957 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
958 return n
? *n
: NULL_TREE
;
962 lookup_field (tree var
, omp_context
*ctx
)
965 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
966 return (tree
) n
->value
;
970 lookup_sfield (tree var
, omp_context
*ctx
)
973 n
= splay_tree_lookup (ctx
->sfield_map
974 ? ctx
->sfield_map
: ctx
->field_map
,
975 (splay_tree_key
) var
);
976 return (tree
) n
->value
;
980 maybe_lookup_field (tree var
, omp_context
*ctx
)
983 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
984 return n
? (tree
) n
->value
: NULL_TREE
;
988 lookup_oacc_reduction (const char *id
, omp_context
*ctx
)
991 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
992 return (tree
) n
->value
;
996 maybe_lookup_oacc_reduction (tree var
, omp_context
*ctx
)
998 splay_tree_node n
= NULL
;
999 if (ctx
->reduction_map
)
1000 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
1001 return n
? (tree
) n
->value
: NULL_TREE
;
1004 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1005 the parallel context if DECL is to be shared. */
1008 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1010 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1013 /* We can only use copy-in/copy-out semantics for shared variables
1014 when we know the value is not accessible from an outer scope. */
1017 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1019 /* ??? Trivially accessible from anywhere. But why would we even
1020 be passing an address in this case? Should we simply assert
1021 this to be false, or should we have a cleanup pass that removes
1022 these from the list of mappings? */
1023 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1026 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1027 without analyzing the expression whether or not its location
1028 is accessible to anyone else. In the case of nested parallel
1029 regions it certainly may be. */
1030 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1033 /* Do not use copy-in/copy-out for variables that have their
1035 if (TREE_ADDRESSABLE (decl
))
1038 /* lower_send_shared_vars only uses copy-in, but not copy-out
1040 if (TREE_READONLY (decl
)
1041 || ((TREE_CODE (decl
) == RESULT_DECL
1042 || TREE_CODE (decl
) == PARM_DECL
)
1043 && DECL_BY_REFERENCE (decl
)))
1046 /* Disallow copy-in/out in nested parallel if
1047 decl is shared in outer parallel, otherwise
1048 each thread could store the shared variable
1049 in its own copy-in location, making the
1050 variable no longer really shared. */
1051 if (shared_ctx
->is_nested
)
1055 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1056 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1063 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1064 c
; c
= OMP_CLAUSE_CHAIN (c
))
1065 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1066 && OMP_CLAUSE_DECL (c
) == decl
)
1070 goto maybe_mark_addressable_and_ret
;
1074 /* For tasks avoid using copy-in/out. As tasks can be
1075 deferred or executed in different thread, when GOMP_task
1076 returns, the task hasn't necessarily terminated. */
1077 if (is_task_ctx (shared_ctx
))
1080 maybe_mark_addressable_and_ret
:
1081 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1082 if (is_gimple_reg (outer
))
1084 /* Taking address of OUTER in lower_send_shared_vars
1085 might need regimplification of everything that uses the
1087 if (!task_shared_vars
)
1088 task_shared_vars
= BITMAP_ALLOC (NULL
);
1089 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1090 TREE_ADDRESSABLE (outer
) = 1;
1099 /* Construct a new automatic decl similar to VAR. */
1102 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1104 tree copy
= copy_var_decl (var
, name
, type
);
1106 DECL_CONTEXT (copy
) = current_function_decl
;
1107 DECL_CHAIN (copy
) = ctx
->block_vars
;
1108 ctx
->block_vars
= copy
;
1114 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1116 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1119 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1122 omp_build_component_ref (tree obj
, tree field
)
1124 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1125 if (TREE_THIS_VOLATILE (field
))
1126 TREE_THIS_VOLATILE (ret
) |= 1;
1127 if (TREE_READONLY (field
))
1128 TREE_READONLY (ret
) |= 1;
1132 /* Build tree nodes to access the field for VAR on the receiver side. */
1135 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1137 tree x
, field
= lookup_field (var
, ctx
);
1139 /* If the receiver record type was remapped in the child function,
1140 remap the field into the new record type. */
1141 x
= maybe_lookup_field (field
, ctx
);
1145 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1146 x
= omp_build_component_ref (x
, field
);
1148 x
= build_simple_mem_ref (x
);
1153 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1154 of a parallel, this is a component reference; for workshare constructs
1155 this is some variable. */
1158 build_outer_var_ref (tree var
, omp_context
*ctx
)
1162 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1164 else if (is_variable_sized (var
))
1166 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1167 x
= build_outer_var_ref (x
, ctx
);
1168 x
= build_simple_mem_ref (x
);
1170 else if (is_taskreg_ctx (ctx
))
1172 bool by_ref
= use_pointer_for_field (var
, NULL
);
1173 x
= build_receiver_ref (var
, by_ref
, ctx
);
1175 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1176 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1178 /* #pragma omp simd isn't a worksharing construct, and can reference even
1179 private vars in its linear etc. clauses. */
1181 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1182 x
= lookup_decl (var
, ctx
->outer
);
1183 else if (ctx
->outer
)
1184 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1188 else if (ctx
->outer
)
1189 x
= lookup_decl (var
, ctx
->outer
);
1190 else if (is_reference (var
))
1191 /* This can happen with orphaned constructs. If var is reference, it is
1192 possible it is shared and as such valid. */
1197 if (is_reference (var
))
1198 x
= build_simple_mem_ref (x
);
1203 /* Build tree nodes to access the field for VAR on the sender side. */
1206 build_sender_ref (tree var
, omp_context
*ctx
)
1208 tree field
= lookup_sfield (var
, ctx
);
1209 return omp_build_component_ref (ctx
->sender_decl
, field
);
1212 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1215 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1217 tree field
, type
, sfield
= NULL_TREE
;
1219 gcc_assert ((mask
& 1) == 0
1220 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1221 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1222 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1223 gcc_assert ((mask
& 3) == 3
1224 || !is_gimple_omp_oacc (ctx
->stmt
));
1226 type
= TREE_TYPE (var
);
1229 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1230 type
= build_pointer_type (build_pointer_type (type
));
1233 type
= build_pointer_type (type
);
1234 else if ((mask
& 3) == 1 && is_reference (var
))
1235 type
= TREE_TYPE (type
);
1237 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1238 FIELD_DECL
, DECL_NAME (var
), type
);
1240 /* Remember what variable this field was created for. This does have a
1241 side effect of making dwarf2out ignore this member, so for helpful
1242 debugging we clear it later in delete_omp_context. */
1243 DECL_ABSTRACT_ORIGIN (field
) = var
;
1244 if (type
== TREE_TYPE (var
))
1246 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1247 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1248 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1251 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1253 if ((mask
& 3) == 3)
1255 insert_field_into_struct (ctx
->record_type
, field
);
1256 if (ctx
->srecord_type
)
1258 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1259 FIELD_DECL
, DECL_NAME (var
), type
);
1260 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1261 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1262 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1263 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1264 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1269 if (ctx
->srecord_type
== NULL_TREE
)
1273 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1274 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1275 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1277 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1278 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1279 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1280 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1281 splay_tree_insert (ctx
->sfield_map
,
1282 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1283 (splay_tree_value
) sfield
);
1287 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1288 : ctx
->srecord_type
, field
);
1292 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1293 (splay_tree_value
) field
);
1294 if ((mask
& 2) && ctx
->sfield_map
)
1295 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1296 (splay_tree_value
) sfield
);
1300 install_var_local (tree var
, omp_context
*ctx
)
1302 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1303 insert_decl_map (&ctx
->cb
, var
, new_var
);
1307 /* Adjust the replacement for DECL in CTX for the new context. This means
1308 copying the DECL_VALUE_EXPR, and fixing up the type. */
1311 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1313 tree new_decl
, size
;
1315 new_decl
= lookup_decl (decl
, ctx
);
1317 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1319 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1320 && DECL_HAS_VALUE_EXPR_P (decl
))
1322 tree ve
= DECL_VALUE_EXPR (decl
);
1323 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1324 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1325 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1328 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1330 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1331 if (size
== error_mark_node
)
1332 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1333 DECL_SIZE (new_decl
) = size
;
1335 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1336 if (size
== error_mark_node
)
1337 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1338 DECL_SIZE_UNIT (new_decl
) = size
;
1342 /* The callback for remap_decl. Search all containing contexts for a
1343 mapping of the variable; this avoids having to duplicate the splay
1344 tree ahead of time. We know a mapping doesn't already exist in the
1345 given context. Create new mappings to implement default semantics. */
1348 omp_copy_decl (tree var
, copy_body_data
*cb
)
1350 omp_context
*ctx
= (omp_context
*) cb
;
1353 if (TREE_CODE (var
) == LABEL_DECL
)
1355 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1356 DECL_CONTEXT (new_var
) = current_function_decl
;
1357 insert_decl_map (&ctx
->cb
, var
, new_var
);
1361 while (!is_taskreg_ctx (ctx
))
1366 new_var
= maybe_lookup_decl (var
, ctx
);
1371 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1374 return error_mark_node
;
1378 /* Debugging dumps for parallel regions. */
1379 void dump_omp_region (FILE *, struct omp_region
*, int);
1380 void debug_omp_region (struct omp_region
*);
1381 void debug_all_omp_regions (void);
1383 /* Dump the parallel region tree rooted at REGION. */
1386 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1388 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1389 gimple_code_name
[region
->type
]);
1392 dump_omp_region (file
, region
->inner
, indent
+ 4);
1396 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1397 region
->cont
->index
);
1401 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1402 region
->exit
->index
);
1404 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1407 dump_omp_region (file
, region
->next
, indent
);
1411 debug_omp_region (struct omp_region
*region
)
1413 dump_omp_region (stderr
, region
, 0);
1417 debug_all_omp_regions (void)
1419 dump_omp_region (stderr
, root_omp_region
, 0);
1423 /* Create a new parallel region starting at STMT inside region PARENT. */
1425 static struct omp_region
*
1426 new_omp_region (basic_block bb
, enum gimple_code type
,
1427 struct omp_region
*parent
)
1429 struct omp_region
*region
= XCNEW (struct omp_region
);
1431 region
->outer
= parent
;
1433 region
->type
= type
;
1437 /* This is a nested region. Add it to the list of inner
1438 regions in PARENT. */
1439 region
->next
= parent
->inner
;
1440 parent
->inner
= region
;
1444 /* This is a toplevel region. Add it to the list of toplevel
1445 regions in ROOT_OMP_REGION. */
1446 region
->next
= root_omp_region
;
1447 root_omp_region
= region
;
1453 /* Release the memory associated with the region tree rooted at REGION. */
1456 free_omp_region_1 (struct omp_region
*region
)
1458 struct omp_region
*i
, *n
;
1460 for (i
= region
->inner
; i
; i
= n
)
1463 free_omp_region_1 (i
);
1469 /* Release the memory for the entire omp region tree. */
1472 free_omp_regions (void)
1474 struct omp_region
*r
, *n
;
1475 for (r
= root_omp_region
; r
; r
= n
)
1478 free_omp_region_1 (r
);
1480 root_omp_region
= NULL
;
1484 /* Create a new context, with OUTER_CTX being the surrounding context. */
1486 static omp_context
*
1487 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1489 omp_context
*ctx
= XCNEW (omp_context
);
1491 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1492 (splay_tree_value
) ctx
);
1497 ctx
->outer
= outer_ctx
;
1498 ctx
->cb
= outer_ctx
->cb
;
1499 ctx
->cb
.block
= NULL
;
1500 ctx
->depth
= outer_ctx
->depth
+ 1;
1501 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1505 ctx
->cb
.src_fn
= current_function_decl
;
1506 ctx
->cb
.dst_fn
= current_function_decl
;
1507 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1508 gcc_checking_assert (ctx
->cb
.src_node
);
1509 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1510 ctx
->cb
.src_cfun
= cfun
;
1511 ctx
->cb
.copy_decl
= omp_copy_decl
;
1512 ctx
->cb
.eh_lp_nr
= 0;
1513 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1517 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1522 static gimple_seq
maybe_catch_exception (gimple_seq
);
1524 /* Finalize task copyfn. */
1527 finalize_task_copyfn (gomp_task
*task_stmt
)
1529 struct function
*child_cfun
;
1531 gimple_seq seq
= NULL
, new_seq
;
1534 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1535 if (child_fn
== NULL_TREE
)
1538 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1539 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1541 push_cfun (child_cfun
);
1542 bind
= gimplify_body (child_fn
, false);
1543 gimple_seq_add_stmt (&seq
, bind
);
1544 new_seq
= maybe_catch_exception (seq
);
1547 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1549 gimple_seq_add_stmt (&seq
, bind
);
1551 gimple_set_body (child_fn
, seq
);
1554 /* Inform the callgraph about the new function. */
1555 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1556 node
->parallelized_function
= 1;
1557 cgraph_node::add_new_function (child_fn
, false);
1560 /* Destroy a omp_context data structures. Called through the splay tree
1561 value delete callback. */
1564 delete_omp_context (splay_tree_value value
)
1566 omp_context
*ctx
= (omp_context
*) value
;
1568 delete ctx
->cb
.decl_map
;
1571 splay_tree_delete (ctx
->field_map
);
1572 if (ctx
->sfield_map
)
1573 splay_tree_delete (ctx
->sfield_map
);
1574 /* Reduction map is copied to nested contexts, so only delete it in the
1576 if (ctx
->reduction_map
1577 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
1578 && is_gimple_omp_offloaded (ctx
->stmt
)
1579 && is_gimple_omp_oacc (ctx
->stmt
))
1580 splay_tree_delete (ctx
->reduction_map
);
1582 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1583 it produces corrupt debug information. */
1584 if (ctx
->record_type
)
1587 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1588 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1590 if (ctx
->srecord_type
)
1593 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1594 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1597 if (is_task_ctx (ctx
))
1598 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1603 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1607 fixup_child_record_type (omp_context
*ctx
)
1609 tree f
, type
= ctx
->record_type
;
1611 /* ??? It isn't sufficient to just call remap_type here, because
1612 variably_modified_type_p doesn't work the way we expect for
1613 record types. Testing each field for whether it needs remapping
1614 and creating a new record by hand works, however. */
1615 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1616 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1620 tree name
, new_fields
= NULL
;
1622 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1623 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1624 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1625 TYPE_DECL
, name
, type
);
1626 TYPE_NAME (type
) = name
;
1628 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1630 tree new_f
= copy_node (f
);
1631 DECL_CONTEXT (new_f
) = type
;
1632 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1633 DECL_CHAIN (new_f
) = new_fields
;
1634 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1635 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1637 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1641 /* Arrange to be able to look up the receiver field
1642 given the sender field. */
1643 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1644 (splay_tree_value
) new_f
);
1646 TYPE_FIELDS (type
) = nreverse (new_fields
);
1650 TREE_TYPE (ctx
->receiver_decl
)
1651 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1654 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1655 specified by CLAUSES. */
1658 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1661 bool scan_array_reductions
= false;
1663 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1667 switch (OMP_CLAUSE_CODE (c
))
1669 case OMP_CLAUSE_PRIVATE
:
1670 decl
= OMP_CLAUSE_DECL (c
);
1671 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1673 else if (!is_variable_sized (decl
))
1674 install_var_local (decl
, ctx
);
1677 case OMP_CLAUSE_SHARED
:
1678 decl
= OMP_CLAUSE_DECL (c
);
1679 /* Ignore shared directives in teams construct. */
1680 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1682 /* Global variables don't need to be copied,
1683 the receiver side will use them directly. */
1684 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1685 if (is_global_var (odecl
))
1687 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1690 gcc_assert (is_taskreg_ctx (ctx
));
1691 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1692 || !is_variable_sized (decl
));
1693 /* Global variables don't need to be copied,
1694 the receiver side will use them directly. */
1695 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1697 by_ref
= use_pointer_for_field (decl
, ctx
);
1698 if (! TREE_READONLY (decl
)
1699 || TREE_ADDRESSABLE (decl
)
1701 || is_reference (decl
))
1703 install_var_field (decl
, by_ref
, 3, ctx
);
1704 install_var_local (decl
, ctx
);
1707 /* We don't need to copy const scalar vars back. */
1708 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1711 case OMP_CLAUSE_LASTPRIVATE
:
1712 /* Let the corresponding firstprivate clause create
1714 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1718 case OMP_CLAUSE_FIRSTPRIVATE
:
1719 if (is_gimple_omp_oacc (ctx
->stmt
))
1721 sorry ("clause not supported yet");
1725 case OMP_CLAUSE_REDUCTION
:
1726 case OMP_CLAUSE_LINEAR
:
1727 decl
= OMP_CLAUSE_DECL (c
);
1729 if (is_variable_sized (decl
))
1731 if (is_task_ctx (ctx
))
1732 install_var_field (decl
, false, 1, ctx
);
1735 else if (is_taskreg_ctx (ctx
))
1738 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1739 by_ref
= use_pointer_for_field (decl
, NULL
);
1741 if (is_task_ctx (ctx
)
1742 && (global
|| by_ref
|| is_reference (decl
)))
1744 install_var_field (decl
, false, 1, ctx
);
1746 install_var_field (decl
, by_ref
, 2, ctx
);
1749 install_var_field (decl
, by_ref
, 3, ctx
);
1751 install_var_local (decl
, ctx
);
1752 if (is_gimple_omp_oacc (ctx
->stmt
)
1753 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
1755 /* Create a decl for the reduction array. */
1756 tree var
= OMP_CLAUSE_DECL (c
);
1757 tree type
= get_base_type (var
);
1758 tree ptype
= build_pointer_type (type
);
1759 tree array
= create_tmp_var (ptype
,
1760 oacc_get_reduction_array_id (var
));
1761 omp_context
*c
= (ctx
->field_map
? ctx
: ctx
->outer
);
1762 install_var_field (array
, true, 3, c
);
1763 install_var_local (array
, c
);
1765 /* Insert it into the current context. */
1766 splay_tree_insert (ctx
->reduction_map
, (splay_tree_key
)
1767 oacc_get_reduction_array_id (var
),
1768 (splay_tree_value
) array
);
1769 splay_tree_insert (ctx
->reduction_map
,
1770 (splay_tree_key
) array
,
1771 (splay_tree_value
) array
);
1775 case OMP_CLAUSE__LOOPTEMP_
:
1776 gcc_assert (is_parallel_ctx (ctx
));
1777 decl
= OMP_CLAUSE_DECL (c
);
1778 install_var_field (decl
, false, 3, ctx
);
1779 install_var_local (decl
, ctx
);
1782 case OMP_CLAUSE_COPYPRIVATE
:
1783 case OMP_CLAUSE_COPYIN
:
1784 decl
= OMP_CLAUSE_DECL (c
);
1785 by_ref
= use_pointer_for_field (decl
, NULL
);
1786 install_var_field (decl
, by_ref
, 3, ctx
);
1789 case OMP_CLAUSE_DEFAULT
:
1790 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1793 case OMP_CLAUSE_FINAL
:
1795 case OMP_CLAUSE_NUM_THREADS
:
1796 case OMP_CLAUSE_NUM_TEAMS
:
1797 case OMP_CLAUSE_THREAD_LIMIT
:
1798 case OMP_CLAUSE_DEVICE
:
1799 case OMP_CLAUSE_SCHEDULE
:
1800 case OMP_CLAUSE_DIST_SCHEDULE
:
1801 case OMP_CLAUSE_DEPEND
:
1802 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1803 case OMP_CLAUSE_NUM_GANGS
:
1804 case OMP_CLAUSE_NUM_WORKERS
:
1805 case OMP_CLAUSE_VECTOR_LENGTH
:
1807 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1811 case OMP_CLAUSE_FROM
:
1812 case OMP_CLAUSE_MAP
:
1814 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1815 decl
= OMP_CLAUSE_DECL (c
);
1816 /* Global variables with "omp declare target" attribute
1817 don't need to be copied, the receiver side will use them
1819 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1821 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1822 && varpool_node::get_create (decl
)->offloadable
)
1824 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1825 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
1827 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1828 not offloaded; there is nothing to map for those. */
1829 if (!is_gimple_omp_offloaded (ctx
->stmt
)
1830 && !POINTER_TYPE_P (TREE_TYPE (decl
))
1831 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
1836 if (DECL_SIZE (decl
)
1837 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1839 tree decl2
= DECL_VALUE_EXPR (decl
);
1840 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1841 decl2
= TREE_OPERAND (decl2
, 0);
1842 gcc_assert (DECL_P (decl2
));
1843 install_var_field (decl2
, true, 3, ctx
);
1844 install_var_local (decl2
, ctx
);
1845 install_var_local (decl
, ctx
);
1849 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1850 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1851 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1852 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1853 install_var_field (decl
, true, 7, ctx
);
1855 install_var_field (decl
, true, 3, ctx
);
1856 if (is_gimple_omp_offloaded (ctx
->stmt
))
1857 install_var_local (decl
, ctx
);
1862 tree base
= get_base_address (decl
);
1863 tree nc
= OMP_CLAUSE_CHAIN (c
);
1866 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1867 && OMP_CLAUSE_DECL (nc
) == base
1868 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
1869 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1871 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1872 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1878 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1879 decl
= OMP_CLAUSE_DECL (c
);
1881 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1882 (splay_tree_key
) decl
));
1884 = build_decl (OMP_CLAUSE_LOCATION (c
),
1885 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1886 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1887 insert_field_into_struct (ctx
->record_type
, field
);
1888 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1889 (splay_tree_value
) field
);
1894 case OMP_CLAUSE_NOWAIT
:
1895 case OMP_CLAUSE_ORDERED
:
1896 case OMP_CLAUSE_COLLAPSE
:
1897 case OMP_CLAUSE_UNTIED
:
1898 case OMP_CLAUSE_MERGEABLE
:
1899 case OMP_CLAUSE_PROC_BIND
:
1900 case OMP_CLAUSE_SAFELEN
:
1901 case OMP_CLAUSE_ASYNC
:
1902 case OMP_CLAUSE_WAIT
:
1903 case OMP_CLAUSE_GANG
:
1904 case OMP_CLAUSE_WORKER
:
1905 case OMP_CLAUSE_VECTOR
:
1908 case OMP_CLAUSE_ALIGNED
:
1909 decl
= OMP_CLAUSE_DECL (c
);
1910 if (is_global_var (decl
)
1911 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1912 install_var_local (decl
, ctx
);
1915 case OMP_CLAUSE_DEVICE_RESIDENT
:
1916 case OMP_CLAUSE_USE_DEVICE
:
1917 case OMP_CLAUSE__CACHE_
:
1918 case OMP_CLAUSE_INDEPENDENT
:
1919 case OMP_CLAUSE_AUTO
:
1920 case OMP_CLAUSE_SEQ
:
1921 sorry ("Clause not supported yet");
1929 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1931 switch (OMP_CLAUSE_CODE (c
))
1933 case OMP_CLAUSE_LASTPRIVATE
:
1934 /* Let the corresponding firstprivate clause create
1936 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1937 scan_array_reductions
= true;
1938 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1942 case OMP_CLAUSE_FIRSTPRIVATE
:
1943 if (is_gimple_omp_oacc (ctx
->stmt
))
1945 sorry ("clause not supported yet");
1949 case OMP_CLAUSE_PRIVATE
:
1950 case OMP_CLAUSE_REDUCTION
:
1951 case OMP_CLAUSE_LINEAR
:
1952 decl
= OMP_CLAUSE_DECL (c
);
1953 if (is_variable_sized (decl
))
1954 install_var_local (decl
, ctx
);
1955 fixup_remapped_decl (decl
, ctx
,
1956 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1957 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1958 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1959 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1960 scan_array_reductions
= true;
1961 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1962 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1963 scan_array_reductions
= true;
1966 case OMP_CLAUSE_SHARED
:
1967 /* Ignore shared directives in teams construct. */
1968 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1970 decl
= OMP_CLAUSE_DECL (c
);
1971 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1972 fixup_remapped_decl (decl
, ctx
, false);
1975 case OMP_CLAUSE_MAP
:
1976 if (!is_gimple_omp_offloaded (ctx
->stmt
))
1978 decl
= OMP_CLAUSE_DECL (c
);
1980 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1981 && varpool_node::get_create (decl
)->offloadable
)
1985 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1986 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1987 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1989 tree new_decl
= lookup_decl (decl
, ctx
);
1990 TREE_TYPE (new_decl
)
1991 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1993 else if (DECL_SIZE (decl
)
1994 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1996 tree decl2
= DECL_VALUE_EXPR (decl
);
1997 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1998 decl2
= TREE_OPERAND (decl2
, 0);
1999 gcc_assert (DECL_P (decl2
));
2000 fixup_remapped_decl (decl2
, ctx
, false);
2001 fixup_remapped_decl (decl
, ctx
, true);
2004 fixup_remapped_decl (decl
, ctx
, false);
2008 case OMP_CLAUSE_COPYPRIVATE
:
2009 case OMP_CLAUSE_COPYIN
:
2010 case OMP_CLAUSE_DEFAULT
:
2012 case OMP_CLAUSE_NUM_THREADS
:
2013 case OMP_CLAUSE_NUM_TEAMS
:
2014 case OMP_CLAUSE_THREAD_LIMIT
:
2015 case OMP_CLAUSE_DEVICE
:
2016 case OMP_CLAUSE_SCHEDULE
:
2017 case OMP_CLAUSE_DIST_SCHEDULE
:
2018 case OMP_CLAUSE_NOWAIT
:
2019 case OMP_CLAUSE_ORDERED
:
2020 case OMP_CLAUSE_COLLAPSE
:
2021 case OMP_CLAUSE_UNTIED
:
2022 case OMP_CLAUSE_FINAL
:
2023 case OMP_CLAUSE_MERGEABLE
:
2024 case OMP_CLAUSE_PROC_BIND
:
2025 case OMP_CLAUSE_SAFELEN
:
2026 case OMP_CLAUSE_ALIGNED
:
2027 case OMP_CLAUSE_DEPEND
:
2028 case OMP_CLAUSE__LOOPTEMP_
:
2030 case OMP_CLAUSE_FROM
:
2031 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2032 case OMP_CLAUSE_ASYNC
:
2033 case OMP_CLAUSE_WAIT
:
2034 case OMP_CLAUSE_NUM_GANGS
:
2035 case OMP_CLAUSE_NUM_WORKERS
:
2036 case OMP_CLAUSE_VECTOR_LENGTH
:
2037 case OMP_CLAUSE_GANG
:
2038 case OMP_CLAUSE_WORKER
:
2039 case OMP_CLAUSE_VECTOR
:
2042 case OMP_CLAUSE_DEVICE_RESIDENT
:
2043 case OMP_CLAUSE_USE_DEVICE
:
2044 case OMP_CLAUSE__CACHE_
:
2045 case OMP_CLAUSE_INDEPENDENT
:
2046 case OMP_CLAUSE_AUTO
:
2047 case OMP_CLAUSE_SEQ
:
2048 sorry ("Clause not supported yet");
2056 gcc_checking_assert (!scan_array_reductions
2057 || !is_gimple_omp_oacc (ctx
->stmt
));
2058 if (scan_array_reductions
)
2059 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2060 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2061 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2063 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2064 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2066 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2067 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2068 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2069 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2070 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2071 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2074 /* Create a new name for omp child function. Returns an identifier. If
2075 IS_CILK_FOR is true then the suffix for the child function is
2079 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2082 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2083 return clone_function_name (current_function_decl
,
2084 task_copy
? "_omp_cpyfn" : "_omp_fn");
2087 /* Returns the type of the induction variable for the child function for
2088 _Cilk_for and the types for _high and _low variables based on TYPE. */
2091 cilk_for_check_loop_diff_type (tree type
)
2093 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2095 if (TYPE_UNSIGNED (type
))
2096 return uint32_type_node
;
2098 return integer_type_node
;
2102 if (TYPE_UNSIGNED (type
))
2103 return uint64_type_node
;
2105 return long_long_integer_type_node
;
2109 /* Build a decl for the omp child function. It'll not contain a body
2110 yet, just the bare decl. */
2113 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2115 tree decl
, type
, name
, t
;
2118 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2119 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2120 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2121 tree cilk_var_type
= NULL_TREE
;
2123 name
= create_omp_child_function_name (task_copy
,
2124 cilk_for_count
!= NULL_TREE
);
2126 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2127 ptr_type_node
, NULL_TREE
);
2128 else if (cilk_for_count
)
2130 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2131 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2132 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2133 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2136 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2138 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2140 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2143 ctx
->cb
.dst_fn
= decl
;
2145 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2147 TREE_STATIC (decl
) = 1;
2148 TREE_USED (decl
) = 1;
2149 DECL_ARTIFICIAL (decl
) = 1;
2150 DECL_IGNORED_P (decl
) = 0;
2151 TREE_PUBLIC (decl
) = 0;
2152 DECL_UNINLINABLE (decl
) = 1;
2153 DECL_EXTERNAL (decl
) = 0;
2154 DECL_CONTEXT (decl
) = NULL_TREE
;
2155 DECL_INITIAL (decl
) = make_node (BLOCK
);
2156 if (cgraph_node::get (current_function_decl
)->offloadable
)
2157 cgraph_node::get_create (decl
)->offloadable
= 1;
2161 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2162 if (is_gimple_omp_offloaded (octx
->stmt
))
2164 cgraph_node::get_create (decl
)->offloadable
= 1;
2165 #ifdef ENABLE_OFFLOADING
2166 g
->have_offload
= true;
2172 if (cgraph_node::get_create (decl
)->offloadable
2173 && !lookup_attribute ("omp declare target",
2174 DECL_ATTRIBUTES (current_function_decl
)))
2175 DECL_ATTRIBUTES (decl
)
2176 = tree_cons (get_identifier ("omp target entrypoint"),
2177 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2179 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2180 RESULT_DECL
, NULL_TREE
, void_type_node
);
2181 DECL_ARTIFICIAL (t
) = 1;
2182 DECL_IGNORED_P (t
) = 1;
2183 DECL_CONTEXT (t
) = decl
;
2184 DECL_RESULT (decl
) = t
;
2186 /* _Cilk_for's child function requires two extra parameters called
2187 __low and __high that are set the by Cilk runtime when it calls this
2191 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2192 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2193 DECL_ARTIFICIAL (t
) = 1;
2194 DECL_NAMELESS (t
) = 1;
2195 DECL_ARG_TYPE (t
) = ptr_type_node
;
2196 DECL_CONTEXT (t
) = current_function_decl
;
2198 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2199 DECL_ARGUMENTS (decl
) = t
;
2201 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2202 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2203 DECL_ARTIFICIAL (t
) = 1;
2204 DECL_NAMELESS (t
) = 1;
2205 DECL_ARG_TYPE (t
) = ptr_type_node
;
2206 DECL_CONTEXT (t
) = current_function_decl
;
2208 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2209 DECL_ARGUMENTS (decl
) = t
;
2212 tree data_name
= get_identifier (".omp_data_i");
2213 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2215 DECL_ARTIFICIAL (t
) = 1;
2216 DECL_NAMELESS (t
) = 1;
2217 DECL_ARG_TYPE (t
) = ptr_type_node
;
2218 DECL_CONTEXT (t
) = current_function_decl
;
2221 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2222 DECL_ARGUMENTS (decl
) = t
;
2224 ctx
->receiver_decl
= t
;
2227 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2228 PARM_DECL
, get_identifier (".omp_data_o"),
2230 DECL_ARTIFICIAL (t
) = 1;
2231 DECL_NAMELESS (t
) = 1;
2232 DECL_ARG_TYPE (t
) = ptr_type_node
;
2233 DECL_CONTEXT (t
) = current_function_decl
;
2235 TREE_ADDRESSABLE (t
) = 1;
2236 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2237 DECL_ARGUMENTS (decl
) = t
;
2240 /* Allocate memory for the function structure. The call to
2241 allocate_struct_function clobbers CFUN, so we need to restore
2243 push_struct_function (decl
);
2244 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2248 /* Callback for walk_gimple_seq. Check if combined parallel
2249 contains gimple_omp_for_combined_into_p OMP_FOR. */
2252 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2253 bool *handled_ops_p
,
2254 struct walk_stmt_info
*wi
)
2256 gimple stmt
= gsi_stmt (*gsi_p
);
2258 *handled_ops_p
= true;
2259 switch (gimple_code (stmt
))
2263 case GIMPLE_OMP_FOR
:
2264 if (gimple_omp_for_combined_into_p (stmt
)
2265 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2268 return integer_zero_node
;
2277 /* Scan an OpenMP parallel directive. */
2280 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2284 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2286 /* Ignore parallel directives with empty bodies, unless there
2287 are copyin clauses. */
2289 && empty_body_p (gimple_omp_body (stmt
))
2290 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2291 OMP_CLAUSE_COPYIN
) == NULL
)
2293 gsi_replace (gsi
, gimple_build_nop (), false);
2297 if (gimple_omp_parallel_combined_p (stmt
))
2299 struct walk_stmt_info wi
;
2301 memset (&wi
, 0, sizeof (wi
));
2303 walk_gimple_seq (gimple_omp_body (stmt
),
2304 find_combined_for
, NULL
, &wi
);
2307 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
) wi
.info
);
2308 struct omp_for_data fd
;
2309 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2310 /* We need two temporaries with fd.loop.v type (istart/iend)
2311 and then (fd.collapse - 1) temporaries with the same
2312 type for count2 ... countN-1 vars if not constant. */
2313 size_t count
= 2, i
;
2314 tree type
= fd
.iter_type
;
2316 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2317 count
+= fd
.collapse
- 1;
2318 for (i
= 0; i
< count
; i
++)
2320 tree temp
= create_tmp_var (type
);
2321 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2322 OMP_CLAUSE__LOOPTEMP_
);
2323 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2324 OMP_CLAUSE_DECL (c
) = temp
;
2325 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2326 gimple_omp_parallel_set_clauses (stmt
, c
);
2331 ctx
= new_omp_context (stmt
, outer_ctx
);
2332 taskreg_contexts
.safe_push (ctx
);
2333 if (taskreg_nesting_level
> 1)
2334 ctx
->is_nested
= true;
2335 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2336 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2337 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2338 name
= create_tmp_var_name (".omp_data_s");
2339 name
= build_decl (gimple_location (stmt
),
2340 TYPE_DECL
, name
, ctx
->record_type
);
2341 DECL_ARTIFICIAL (name
) = 1;
2342 DECL_NAMELESS (name
) = 1;
2343 TYPE_NAME (ctx
->record_type
) = name
;
2344 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2345 create_omp_child_function (ctx
, false);
2346 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2348 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2349 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2351 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2352 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2355 /* Scan an OpenMP task directive. */
2358 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2362 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2364 /* Ignore task directives with empty bodies. */
2366 && empty_body_p (gimple_omp_body (stmt
)))
2368 gsi_replace (gsi
, gimple_build_nop (), false);
2372 ctx
= new_omp_context (stmt
, outer_ctx
);
2373 taskreg_contexts
.safe_push (ctx
);
2374 if (taskreg_nesting_level
> 1)
2375 ctx
->is_nested
= true;
2376 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2377 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2378 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2379 name
= create_tmp_var_name (".omp_data_s");
2380 name
= build_decl (gimple_location (stmt
),
2381 TYPE_DECL
, name
, ctx
->record_type
);
2382 DECL_ARTIFICIAL (name
) = 1;
2383 DECL_NAMELESS (name
) = 1;
2384 TYPE_NAME (ctx
->record_type
) = name
;
2385 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2386 create_omp_child_function (ctx
, false);
2387 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2389 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2391 if (ctx
->srecord_type
)
2393 name
= create_tmp_var_name (".omp_data_a");
2394 name
= build_decl (gimple_location (stmt
),
2395 TYPE_DECL
, name
, ctx
->srecord_type
);
2396 DECL_ARTIFICIAL (name
) = 1;
2397 DECL_NAMELESS (name
) = 1;
2398 TYPE_NAME (ctx
->srecord_type
) = name
;
2399 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2400 create_omp_child_function (ctx
, true);
2403 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2405 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2407 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2408 t
= build_int_cst (long_integer_type_node
, 0);
2409 gimple_omp_task_set_arg_size (stmt
, t
);
2410 t
= build_int_cst (long_integer_type_node
, 1);
2411 gimple_omp_task_set_arg_align (stmt
, t
);
2416 /* If any decls have been made addressable during scan_omp,
2417 adjust their fields if needed, and layout record types
2418 of parallel/task constructs. */
2421 finish_taskreg_scan (omp_context
*ctx
)
2423 if (ctx
->record_type
== NULL_TREE
)
2426 /* If any task_shared_vars were needed, verify all
2427 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2428 statements if use_pointer_for_field hasn't changed
2429 because of that. If it did, update field types now. */
2430 if (task_shared_vars
)
2434 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2435 c
; c
= OMP_CLAUSE_CHAIN (c
))
2436 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2438 tree decl
= OMP_CLAUSE_DECL (c
);
2440 /* Global variables don't need to be copied,
2441 the receiver side will use them directly. */
2442 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2444 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2445 || !use_pointer_for_field (decl
, ctx
))
2447 tree field
= lookup_field (decl
, ctx
);
2448 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2449 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2451 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2452 TREE_THIS_VOLATILE (field
) = 0;
2453 DECL_USER_ALIGN (field
) = 0;
2454 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2455 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2456 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2457 if (ctx
->srecord_type
)
2459 tree sfield
= lookup_sfield (decl
, ctx
);
2460 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2461 TREE_THIS_VOLATILE (sfield
) = 0;
2462 DECL_USER_ALIGN (sfield
) = 0;
2463 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2464 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2465 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2470 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2472 layout_type (ctx
->record_type
);
2473 fixup_child_record_type (ctx
);
2477 location_t loc
= gimple_location (ctx
->stmt
);
2478 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2479 /* Move VLA fields to the end. */
2480 p
= &TYPE_FIELDS (ctx
->record_type
);
2482 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2483 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2486 *p
= TREE_CHAIN (*p
);
2487 TREE_CHAIN (*q
) = NULL_TREE
;
2488 q
= &TREE_CHAIN (*q
);
2491 p
= &DECL_CHAIN (*p
);
2493 layout_type (ctx
->record_type
);
2494 fixup_child_record_type (ctx
);
2495 if (ctx
->srecord_type
)
2496 layout_type (ctx
->srecord_type
);
2497 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2498 TYPE_SIZE_UNIT (ctx
->record_type
));
2499 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2500 t
= build_int_cst (long_integer_type_node
,
2501 TYPE_ALIGN_UNIT (ctx
->record_type
));
2502 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2507 static omp_context
*
2508 enclosing_target_ctx (omp_context
*ctx
)
2511 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2513 gcc_assert (ctx
!= NULL
);
2518 oacc_loop_or_target_p (gimple stmt
)
2520 enum gimple_code outer_type
= gimple_code (stmt
);
2521 return ((outer_type
== GIMPLE_OMP_TARGET
2522 && ((gimple_omp_target_kind (stmt
)
2523 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2524 || (gimple_omp_target_kind (stmt
)
2525 == GF_OMP_TARGET_KIND_OACC_KERNELS
)))
2526 || (outer_type
== GIMPLE_OMP_FOR
2527 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
));
2530 /* Scan a GIMPLE_OMP_FOR. */
2533 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2535 enum gimple_code outer_type
= GIMPLE_ERROR_MARK
;
2538 tree clauses
= gimple_omp_for_clauses (stmt
);
2541 outer_type
= gimple_code (outer_ctx
->stmt
);
2543 ctx
= new_omp_context (stmt
, outer_ctx
);
2545 if (is_gimple_omp_oacc (stmt
))
2547 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2548 ctx
->gwv_this
= outer_ctx
->gwv_this
;
2549 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2552 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_GANG
)
2554 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WORKER
)
2556 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR
)
2560 ctx
->gwv_this
|= val
;
2563 /* Skip; not nested inside a region. */
2566 if (!oacc_loop_or_target_p (outer_ctx
->stmt
))
2568 /* Skip; not nested inside an OpenACC region. */
2571 if (outer_type
== GIMPLE_OMP_FOR
)
2572 outer_ctx
->gwv_below
|= val
;
2573 if (OMP_CLAUSE_OPERAND (c
, 0) != NULL_TREE
)
2575 omp_context
*enclosing
= enclosing_target_ctx (outer_ctx
);
2576 if (gimple_omp_target_kind (enclosing
->stmt
)
2577 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2578 error_at (gimple_location (stmt
),
2579 "no arguments allowed to gang, worker and vector clauses inside parallel");
2584 scan_sharing_clauses (clauses
, ctx
);
2586 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2587 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2589 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2590 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2591 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2592 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2594 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2596 if (is_gimple_omp_oacc (stmt
))
2598 if (ctx
->gwv_this
& ctx
->gwv_below
)
2599 error_at (gimple_location (stmt
),
2600 "gang, worker and vector may occur only once in a loop nest");
2601 else if (ctx
->gwv_below
!= 0
2602 && ctx
->gwv_this
> ctx
->gwv_below
)
2603 error_at (gimple_location (stmt
),
2604 "gang, worker and vector must occur in this order in a loop nest");
2605 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2606 outer_ctx
->gwv_below
|= ctx
->gwv_below
;
2610 /* Scan an OpenMP sections directive. */
2613 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
2617 ctx
= new_omp_context (stmt
, outer_ctx
);
2618 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2619 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2622 /* Scan an OpenMP single directive. */
2625 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
2630 ctx
= new_omp_context (stmt
, outer_ctx
);
2631 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2632 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2633 name
= create_tmp_var_name (".omp_copy_s");
2634 name
= build_decl (gimple_location (stmt
),
2635 TYPE_DECL
, name
, ctx
->record_type
);
2636 TYPE_NAME (ctx
->record_type
) = name
;
2638 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2639 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2641 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2642 ctx
->record_type
= NULL
;
2644 layout_type (ctx
->record_type
);
2647 /* Scan a GIMPLE_OMP_TARGET. */
2650 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
2654 bool offloaded
= is_gimple_omp_offloaded (stmt
);
2655 tree clauses
= gimple_omp_target_clauses (stmt
);
2657 ctx
= new_omp_context (stmt
, outer_ctx
);
2658 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2659 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2660 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2661 name
= create_tmp_var_name (".omp_data_t");
2662 name
= build_decl (gimple_location (stmt
),
2663 TYPE_DECL
, name
, ctx
->record_type
);
2664 DECL_ARTIFICIAL (name
) = 1;
2665 DECL_NAMELESS (name
) = 1;
2666 TYPE_NAME (ctx
->record_type
) = name
;
2667 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2670 if (is_gimple_omp_oacc (stmt
))
2671 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
2674 create_omp_child_function (ctx
, false);
2675 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2678 if (is_gimple_omp_oacc (stmt
))
2680 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2682 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_GANGS
)
2683 ctx
->gwv_this
|= MASK_GANG
;
2684 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_WORKERS
)
2685 ctx
->gwv_this
|= MASK_WORKER
;
2686 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR_LENGTH
)
2687 ctx
->gwv_this
|= MASK_VECTOR
;
2691 scan_sharing_clauses (clauses
, ctx
);
2692 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2694 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2695 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2698 TYPE_FIELDS (ctx
->record_type
)
2699 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2700 #ifdef ENABLE_CHECKING
2702 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2703 for (field
= TYPE_FIELDS (ctx
->record_type
);
2705 field
= DECL_CHAIN (field
))
2706 gcc_assert (DECL_ALIGN (field
) == align
);
2708 layout_type (ctx
->record_type
);
2710 fixup_child_record_type (ctx
);
2714 /* Scan an OpenMP teams directive. */
2717 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
2719 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2720 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2721 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2724 /* Check nesting restrictions. */
2726 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2728 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2729 inside an OpenACC CTX. */
2730 if (!(is_gimple_omp (stmt
)
2731 && is_gimple_omp_oacc (stmt
)))
2733 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
2734 if (is_gimple_omp (ctx_
->stmt
)
2735 && is_gimple_omp_oacc (ctx_
->stmt
))
2737 error_at (gimple_location (stmt
),
2738 "non-OpenACC construct inside of OpenACC region");
2745 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2746 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2748 error_at (gimple_location (stmt
),
2749 "OpenMP constructs may not be nested inside simd region");
2752 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2754 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2755 || (gimple_omp_for_kind (stmt
)
2756 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2757 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2759 error_at (gimple_location (stmt
),
2760 "only distribute or parallel constructs are allowed to "
2761 "be closely nested inside teams construct");
2766 switch (gimple_code (stmt
))
2768 case GIMPLE_OMP_FOR
:
2769 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2771 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2773 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2775 error_at (gimple_location (stmt
),
2776 "distribute construct must be closely nested inside "
2784 if (is_gimple_call (stmt
)
2785 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2786 == BUILT_IN_GOMP_CANCEL
2787 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2788 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2790 const char *bad
= NULL
;
2791 const char *kind
= NULL
;
2794 error_at (gimple_location (stmt
), "orphaned %qs construct",
2795 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2796 == BUILT_IN_GOMP_CANCEL
2797 ? "#pragma omp cancel"
2798 : "#pragma omp cancellation point");
2801 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2802 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2806 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2807 bad
= "#pragma omp parallel";
2808 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2809 == BUILT_IN_GOMP_CANCEL
2810 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2811 ctx
->cancellable
= true;
2815 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2816 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2817 bad
= "#pragma omp for";
2818 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2819 == BUILT_IN_GOMP_CANCEL
2820 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2822 ctx
->cancellable
= true;
2823 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2825 warning_at (gimple_location (stmt
), 0,
2826 "%<#pragma omp cancel for%> inside "
2827 "%<nowait%> for construct");
2828 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2829 OMP_CLAUSE_ORDERED
))
2830 warning_at (gimple_location (stmt
), 0,
2831 "%<#pragma omp cancel for%> inside "
2832 "%<ordered%> for construct");
2837 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2838 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2839 bad
= "#pragma omp sections";
2840 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2841 == BUILT_IN_GOMP_CANCEL
2842 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2844 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2846 ctx
->cancellable
= true;
2847 if (find_omp_clause (gimple_omp_sections_clauses
2850 warning_at (gimple_location (stmt
), 0,
2851 "%<#pragma omp cancel sections%> inside "
2852 "%<nowait%> sections construct");
2856 gcc_assert (ctx
->outer
2857 && gimple_code (ctx
->outer
->stmt
)
2858 == GIMPLE_OMP_SECTIONS
);
2859 ctx
->outer
->cancellable
= true;
2860 if (find_omp_clause (gimple_omp_sections_clauses
2863 warning_at (gimple_location (stmt
), 0,
2864 "%<#pragma omp cancel sections%> inside "
2865 "%<nowait%> sections construct");
2871 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2872 bad
= "#pragma omp task";
2874 ctx
->cancellable
= true;
2878 error_at (gimple_location (stmt
), "invalid arguments");
2883 error_at (gimple_location (stmt
),
2884 "%<%s %s%> construct not closely nested inside of %qs",
2885 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2886 == BUILT_IN_GOMP_CANCEL
2887 ? "#pragma omp cancel"
2888 : "#pragma omp cancellation point", kind
, bad
);
2893 case GIMPLE_OMP_SECTIONS
:
2894 case GIMPLE_OMP_SINGLE
:
2895 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2896 switch (gimple_code (ctx
->stmt
))
2898 case GIMPLE_OMP_FOR
:
2899 case GIMPLE_OMP_SECTIONS
:
2900 case GIMPLE_OMP_SINGLE
:
2901 case GIMPLE_OMP_ORDERED
:
2902 case GIMPLE_OMP_MASTER
:
2903 case GIMPLE_OMP_TASK
:
2904 case GIMPLE_OMP_CRITICAL
:
2905 if (is_gimple_call (stmt
))
2907 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2908 != BUILT_IN_GOMP_BARRIER
)
2910 error_at (gimple_location (stmt
),
2911 "barrier region may not be closely nested inside "
2912 "of work-sharing, critical, ordered, master or "
2913 "explicit task region");
2916 error_at (gimple_location (stmt
),
2917 "work-sharing region may not be closely nested inside "
2918 "of work-sharing, critical, ordered, master or explicit "
2921 case GIMPLE_OMP_PARALLEL
:
2927 case GIMPLE_OMP_MASTER
:
2928 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2929 switch (gimple_code (ctx
->stmt
))
2931 case GIMPLE_OMP_FOR
:
2932 case GIMPLE_OMP_SECTIONS
:
2933 case GIMPLE_OMP_SINGLE
:
2934 case GIMPLE_OMP_TASK
:
2935 error_at (gimple_location (stmt
),
2936 "master region may not be closely nested inside "
2937 "of work-sharing or explicit task region");
2939 case GIMPLE_OMP_PARALLEL
:
2945 case GIMPLE_OMP_ORDERED
:
2946 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2947 switch (gimple_code (ctx
->stmt
))
2949 case GIMPLE_OMP_CRITICAL
:
2950 case GIMPLE_OMP_TASK
:
2951 error_at (gimple_location (stmt
),
2952 "ordered region may not be closely nested inside "
2953 "of critical or explicit task region");
2955 case GIMPLE_OMP_FOR
:
2956 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2957 OMP_CLAUSE_ORDERED
) == NULL
)
2959 error_at (gimple_location (stmt
),
2960 "ordered region must be closely nested inside "
2961 "a loop region with an ordered clause");
2965 case GIMPLE_OMP_PARALLEL
:
2966 error_at (gimple_location (stmt
),
2967 "ordered region must be closely nested inside "
2968 "a loop region with an ordered clause");
2974 case GIMPLE_OMP_CRITICAL
:
2977 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
2978 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2979 if (gomp_critical
*other_crit
2980 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
2981 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
2983 error_at (gimple_location (stmt
),
2984 "critical region may not be nested inside a critical "
2985 "region with the same name");
2990 case GIMPLE_OMP_TEAMS
:
2992 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2993 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2995 error_at (gimple_location (stmt
),
2996 "teams construct not closely nested inside of target "
3001 case GIMPLE_OMP_TARGET
:
3002 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3004 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3006 if (is_gimple_omp (stmt
)
3007 && is_gimple_omp_oacc (stmt
)
3008 && is_gimple_omp (ctx
->stmt
))
3010 error_at (gimple_location (stmt
),
3011 "OpenACC construct inside of non-OpenACC region");
3017 const char *stmt_name
, *ctx_stmt_name
;
3018 switch (gimple_omp_target_kind (stmt
))
3020 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3021 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3022 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3023 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3024 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3025 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3026 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3027 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
: stmt_name
= "enter/exit data"; break;
3028 default: gcc_unreachable ();
3030 switch (gimple_omp_target_kind (ctx
->stmt
))
3032 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3033 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3034 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: ctx_stmt_name
= "parallel"; break;
3035 case GF_OMP_TARGET_KIND_OACC_KERNELS
: ctx_stmt_name
= "kernels"; break;
3036 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3037 default: gcc_unreachable ();
3040 /* OpenACC/OpenMP mismatch? */
3041 if (is_gimple_omp_oacc (stmt
)
3042 != is_gimple_omp_oacc (ctx
->stmt
))
3044 error_at (gimple_location (stmt
),
3045 "%s %s construct inside of %s %s region",
3046 (is_gimple_omp_oacc (stmt
)
3047 ? "OpenACC" : "OpenMP"), stmt_name
,
3048 (is_gimple_omp_oacc (ctx
->stmt
)
3049 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3052 if (is_gimple_omp_offloaded (ctx
->stmt
))
3054 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3055 if (is_gimple_omp_oacc (ctx
->stmt
))
3057 error_at (gimple_location (stmt
),
3058 "%s construct inside of %s region",
3059 stmt_name
, ctx_stmt_name
);
3064 gcc_checking_assert (!is_gimple_omp_oacc (stmt
));
3065 warning_at (gimple_location (stmt
), 0,
3066 "%s construct inside of %s region",
3067 stmt_name
, ctx_stmt_name
);
3079 /* Helper function scan_omp.
3081 Callback for walk_tree or operators in walk_gimple_stmt used to
3082 scan for OMP directives in TP. */
3085 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3087 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3088 omp_context
*ctx
= (omp_context
*) wi
->info
;
3091 switch (TREE_CODE (t
))
3098 *tp
= remap_decl (t
, &ctx
->cb
);
3102 if (ctx
&& TYPE_P (t
))
3103 *tp
= remap_type (t
, &ctx
->cb
);
3104 else if (!DECL_P (t
))
3109 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3110 if (tem
!= TREE_TYPE (t
))
3112 if (TREE_CODE (t
) == INTEGER_CST
)
3113 *tp
= wide_int_to_tree (tem
, t
);
3115 TREE_TYPE (t
) = tem
;
3125 /* Return true if FNDECL is a setjmp or a longjmp. */
3128 setjmp_or_longjmp_p (const_tree fndecl
)
3130 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3131 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3132 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3135 tree declname
= DECL_NAME (fndecl
);
3138 const char *name
= IDENTIFIER_POINTER (declname
);
3139 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3143 /* Helper function for scan_omp.
3145 Callback for walk_gimple_stmt used to scan for OMP directives in
3146 the current statement in GSI. */
3149 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3150 struct walk_stmt_info
*wi
)
3152 gimple stmt
= gsi_stmt (*gsi
);
3153 omp_context
*ctx
= (omp_context
*) wi
->info
;
3155 if (gimple_has_location (stmt
))
3156 input_location
= gimple_location (stmt
);
3158 /* Check the nesting restrictions. */
3159 bool remove
= false;
3160 if (is_gimple_omp (stmt
))
3161 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3162 else if (is_gimple_call (stmt
))
3164 tree fndecl
= gimple_call_fndecl (stmt
);
3167 if (setjmp_or_longjmp_p (fndecl
)
3169 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3170 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3173 error_at (gimple_location (stmt
),
3174 "setjmp/longjmp inside simd construct");
3176 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3177 switch (DECL_FUNCTION_CODE (fndecl
))
3179 case BUILT_IN_GOMP_BARRIER
:
3180 case BUILT_IN_GOMP_CANCEL
:
3181 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3182 case BUILT_IN_GOMP_TASKYIELD
:
3183 case BUILT_IN_GOMP_TASKWAIT
:
3184 case BUILT_IN_GOMP_TASKGROUP_START
:
3185 case BUILT_IN_GOMP_TASKGROUP_END
:
3186 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3195 stmt
= gimple_build_nop ();
3196 gsi_replace (gsi
, stmt
, false);
3199 *handled_ops_p
= true;
3201 switch (gimple_code (stmt
))
3203 case GIMPLE_OMP_PARALLEL
:
3204 taskreg_nesting_level
++;
3205 scan_omp_parallel (gsi
, ctx
);
3206 taskreg_nesting_level
--;
3209 case GIMPLE_OMP_TASK
:
3210 taskreg_nesting_level
++;
3211 scan_omp_task (gsi
, ctx
);
3212 taskreg_nesting_level
--;
3215 case GIMPLE_OMP_FOR
:
3216 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3219 case GIMPLE_OMP_SECTIONS
:
3220 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3223 case GIMPLE_OMP_SINGLE
:
3224 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3227 case GIMPLE_OMP_SECTION
:
3228 case GIMPLE_OMP_MASTER
:
3229 case GIMPLE_OMP_TASKGROUP
:
3230 case GIMPLE_OMP_ORDERED
:
3231 case GIMPLE_OMP_CRITICAL
:
3232 ctx
= new_omp_context (stmt
, ctx
);
3233 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3236 case GIMPLE_OMP_TARGET
:
3237 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3240 case GIMPLE_OMP_TEAMS
:
3241 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3248 *handled_ops_p
= false;
3250 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3252 var
= DECL_CHAIN (var
))
3253 insert_decl_map (&ctx
->cb
, var
, var
);
3257 *handled_ops_p
= false;
3265 /* Scan all the statements starting at the current statement. CTX
3266 contains context information about the OMP directives and
3267 clauses found during the scan. */
3270 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3272 location_t saved_location
;
3273 struct walk_stmt_info wi
;
3275 memset (&wi
, 0, sizeof (wi
));
3277 wi
.want_locations
= true;
3279 saved_location
= input_location
;
3280 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3281 input_location
= saved_location
;
3284 /* Re-gimplification and code generation routines. */
3286 /* Build a call to GOMP_barrier. */
3289 build_omp_barrier (tree lhs
)
3291 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3292 : BUILT_IN_GOMP_BARRIER
);
3293 gcall
*g
= gimple_build_call (fndecl
, 0);
3295 gimple_call_set_lhs (g
, lhs
);
3299 /* If a context was created for STMT when it was scanned, return it. */
3301 static omp_context
*
3302 maybe_lookup_ctx (gimple stmt
)
3305 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3306 return n
? (omp_context
*) n
->value
: NULL
;
3310 /* Find the mapping for DECL in CTX or the immediately enclosing
3311 context that has a mapping for DECL.
3313 If CTX is a nested parallel directive, we may have to use the decl
3314 mappings created in CTX's parent context. Suppose that we have the
3315 following parallel nesting (variable UIDs showed for clarity):
3318 #omp parallel shared(iD.1562) -> outer parallel
3319 iD.1562 = iD.1562 + 1;
3321 #omp parallel shared (iD.1562) -> inner parallel
3322 iD.1562 = iD.1562 - 1;
3324 Each parallel structure will create a distinct .omp_data_s structure
3325 for copying iD.1562 in/out of the directive:
3327 outer parallel .omp_data_s.1.i -> iD.1562
3328 inner parallel .omp_data_s.2.i -> iD.1562
3330 A shared variable mapping will produce a copy-out operation before
3331 the parallel directive and a copy-in operation after it. So, in
3332 this case we would have:
3335 .omp_data_o.1.i = iD.1562;
3336 #omp parallel shared(iD.1562) -> outer parallel
3337 .omp_data_i.1 = &.omp_data_o.1
3338 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3340 .omp_data_o.2.i = iD.1562; -> **
3341 #omp parallel shared(iD.1562) -> inner parallel
3342 .omp_data_i.2 = &.omp_data_o.2
3343 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3346 ** This is a problem. The symbol iD.1562 cannot be referenced
3347 inside the body of the outer parallel region. But since we are
3348 emitting this copy operation while expanding the inner parallel
3349 directive, we need to access the CTX structure of the outer
3350 parallel directive to get the correct mapping:
3352 .omp_data_o.2.i = .omp_data_i.1->i
3354 Since there may be other workshare or parallel directives enclosing
3355 the parallel directive, it may be necessary to walk up the context
3356 parent chain. This is not a problem in general because nested
3357 parallelism happens only rarely. */
3360 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3365 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3366 t
= maybe_lookup_decl (decl
, up
);
3368 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3370 return t
? t
: decl
;
3374 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3375 in outer contexts. */
3378 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3383 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3384 t
= maybe_lookup_decl (decl
, up
);
3386 return t
? t
: decl
;
3390 /* Construct the initialization value for reduction CLAUSE. */
3393 omp_reduction_init (tree clause
, tree type
)
3395 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3396 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3403 case TRUTH_ORIF_EXPR
:
3404 case TRUTH_XOR_EXPR
:
3406 return build_zero_cst (type
);
3409 case TRUTH_AND_EXPR
:
3410 case TRUTH_ANDIF_EXPR
:
3412 return fold_convert_loc (loc
, type
, integer_one_node
);
3415 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3418 if (SCALAR_FLOAT_TYPE_P (type
))
3420 REAL_VALUE_TYPE max
, min
;
3421 if (HONOR_INFINITIES (type
))
3424 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3427 real_maxval (&min
, 1, TYPE_MODE (type
));
3428 return build_real (type
, min
);
3432 gcc_assert (INTEGRAL_TYPE_P (type
));
3433 return TYPE_MIN_VALUE (type
);
3437 if (SCALAR_FLOAT_TYPE_P (type
))
3439 REAL_VALUE_TYPE max
;
3440 if (HONOR_INFINITIES (type
))
3443 real_maxval (&max
, 0, TYPE_MODE (type
));
3444 return build_real (type
, max
);
3448 gcc_assert (INTEGRAL_TYPE_P (type
));
3449 return TYPE_MAX_VALUE (type
);
3457 /* Return alignment to be assumed for var in CLAUSE, which should be
3458 OMP_CLAUSE_ALIGNED. */
3461 omp_clause_aligned_alignment (tree clause
)
3463 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3464 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3466 /* Otherwise return implementation defined alignment. */
3467 unsigned int al
= 1;
3468 machine_mode mode
, vmode
;
3469 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3471 vs
= 1 << floor_log2 (vs
);
3472 static enum mode_class classes
[]
3473 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3474 for (int i
= 0; i
< 4; i
+= 2)
3475 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3477 mode
= GET_MODE_WIDER_MODE (mode
))
3479 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3480 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3483 && GET_MODE_SIZE (vmode
) < vs
3484 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3485 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3487 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3488 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3490 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3491 / GET_MODE_SIZE (mode
));
3492 if (TYPE_MODE (type
) != vmode
)
3494 if (TYPE_ALIGN_UNIT (type
) > al
)
3495 al
= TYPE_ALIGN_UNIT (type
);
3497 return build_int_cst (integer_type_node
, al
);
3500 /* Return maximum possible vectorization factor for the target. */
3507 || !flag_tree_loop_optimize
3508 || (!flag_tree_loop_vectorize
3509 && (global_options_set
.x_flag_tree_loop_vectorize
3510 || global_options_set
.x_flag_tree_vectorize
)))
3513 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3516 vs
= 1 << floor_log2 (vs
);
3519 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3520 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3521 return GET_MODE_NUNITS (vqimode
);
3525 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3529 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3530 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3534 max_vf
= omp_max_vf ();
3537 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3538 OMP_CLAUSE_SAFELEN
);
3539 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3541 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3543 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3547 idx
= create_tmp_var (unsigned_type_node
);
3548 lane
= create_tmp_var (unsigned_type_node
);
3554 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3555 tree avar
= create_tmp_var_raw (atype
);
3556 if (TREE_ADDRESSABLE (new_var
))
3557 TREE_ADDRESSABLE (avar
) = 1;
3558 DECL_ATTRIBUTES (avar
)
3559 = tree_cons (get_identifier ("omp simd array"), NULL
,
3560 DECL_ATTRIBUTES (avar
));
3561 gimple_add_tmp_var (avar
);
3562 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3563 NULL_TREE
, NULL_TREE
);
3564 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3565 NULL_TREE
, NULL_TREE
);
3566 if (DECL_P (new_var
))
3568 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3569 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3574 /* Helper function of lower_rec_input_clauses. For a reference
3575 in simd reduction, add an underlying variable it will reference. */
3578 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3580 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3581 if (TREE_CONSTANT (z
))
3583 const char *name
= NULL
;
3584 if (DECL_NAME (new_vard
))
3585 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3587 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3588 gimple_add_tmp_var (z
);
3589 TREE_ADDRESSABLE (z
) = 1;
3590 z
= build_fold_addr_expr_loc (loc
, z
);
3591 gimplify_assign (new_vard
, z
, ilist
);
3595 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3596 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3597 private variables. Initialization statements go in ILIST, while calls
3598 to destructors go in DLIST. */
3601 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3602 omp_context
*ctx
, struct omp_for_data
*fd
)
3604 tree c
, dtor
, copyin_seq
, x
, ptr
;
3605 bool copyin_by_ref
= false;
3606 bool lastprivate_firstprivate
= false;
3607 bool reduction_omp_orig_ref
= false;
3609 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3610 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3612 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3613 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3614 gimple_seq llist
[2] = { NULL
, NULL
};
3618 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3619 with data sharing clauses referencing variable sized vars. That
3620 is unnecessarily hard to support and very unlikely to result in
3621 vectorized code anyway. */
3623 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3624 switch (OMP_CLAUSE_CODE (c
))
3626 case OMP_CLAUSE_LINEAR
:
3627 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3630 case OMP_CLAUSE_REDUCTION
:
3631 case OMP_CLAUSE_PRIVATE
:
3632 case OMP_CLAUSE_FIRSTPRIVATE
:
3633 case OMP_CLAUSE_LASTPRIVATE
:
3634 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3641 /* Do all the fixed sized types in the first pass, and the variable sized
3642 types in the second pass. This makes sure that the scalar arguments to
3643 the variable sized types are processed before we use them in the
3644 variable sized operations. */
3645 for (pass
= 0; pass
< 2; ++pass
)
3647 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3649 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3652 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3656 case OMP_CLAUSE_PRIVATE
:
3657 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3660 case OMP_CLAUSE_SHARED
:
3661 /* Ignore shared directives in teams construct. */
3662 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3664 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3666 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3669 case OMP_CLAUSE_FIRSTPRIVATE
:
3670 case OMP_CLAUSE_COPYIN
:
3671 case OMP_CLAUSE_LINEAR
:
3673 case OMP_CLAUSE_REDUCTION
:
3674 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3675 reduction_omp_orig_ref
= true;
3677 case OMP_CLAUSE__LOOPTEMP_
:
3678 /* Handle _looptemp_ clauses only on parallel. */
3682 case OMP_CLAUSE_LASTPRIVATE
:
3683 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3685 lastprivate_firstprivate
= true;
3689 /* Even without corresponding firstprivate, if
3690 decl is Fortran allocatable, it needs outer var
3693 && lang_hooks
.decls
.omp_private_outer_ref
3694 (OMP_CLAUSE_DECL (c
)))
3695 lastprivate_firstprivate
= true;
3697 case OMP_CLAUSE_ALIGNED
:
3700 var
= OMP_CLAUSE_DECL (c
);
3701 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3702 && !is_global_var (var
))
3704 new_var
= maybe_lookup_decl (var
, ctx
);
3705 if (new_var
== NULL_TREE
)
3706 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3707 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3708 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3709 omp_clause_aligned_alignment (c
));
3710 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3711 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3712 gimplify_and_add (x
, ilist
);
3714 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3715 && is_global_var (var
))
3717 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3718 new_var
= lookup_decl (var
, ctx
);
3719 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3720 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3721 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3722 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3723 omp_clause_aligned_alignment (c
));
3724 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3725 x
= create_tmp_var (ptype
);
3726 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3727 gimplify_and_add (t
, ilist
);
3728 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3729 SET_DECL_VALUE_EXPR (new_var
, t
);
3730 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3737 new_var
= var
= OMP_CLAUSE_DECL (c
);
3738 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3739 new_var
= lookup_decl (var
, ctx
);
3741 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3746 else if (is_variable_sized (var
))
3748 /* For variable sized types, we need to allocate the
3749 actual storage here. Call alloca and store the
3750 result in the pointer decl that we created elsewhere. */
3754 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3759 ptr
= DECL_VALUE_EXPR (new_var
);
3760 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3761 ptr
= TREE_OPERAND (ptr
, 0);
3762 gcc_assert (DECL_P (ptr
));
3763 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3765 /* void *tmp = __builtin_alloca */
3766 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3767 stmt
= gimple_build_call (atmp
, 1, x
);
3768 tmp
= create_tmp_var_raw (ptr_type_node
);
3769 gimple_add_tmp_var (tmp
);
3770 gimple_call_set_lhs (stmt
, tmp
);
3772 gimple_seq_add_stmt (ilist
, stmt
);
3774 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3775 gimplify_assign (ptr
, x
, ilist
);
3778 else if (is_reference (var
))
3780 /* For references that are being privatized for Fortran,
3781 allocate new backing storage for the new pointer
3782 variable. This allows us to avoid changing all the
3783 code that expects a pointer to something that expects
3784 a direct variable. */
3788 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3789 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3791 x
= build_receiver_ref (var
, false, ctx
);
3792 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3794 else if (TREE_CONSTANT (x
))
3796 /* For reduction in SIMD loop, defer adding the
3797 initialization of the reference, because if we decide
3798 to use SIMD array for it, the initilization could cause
3800 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3804 const char *name
= NULL
;
3805 if (DECL_NAME (var
))
3806 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3808 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3810 gimple_add_tmp_var (x
);
3811 TREE_ADDRESSABLE (x
) = 1;
3812 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3817 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3818 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3823 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3824 gimplify_assign (new_var
, x
, ilist
);
3827 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3829 else if (c_kind
== OMP_CLAUSE_REDUCTION
3830 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3838 switch (OMP_CLAUSE_CODE (c
))
3840 case OMP_CLAUSE_SHARED
:
3841 /* Ignore shared directives in teams construct. */
3842 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3844 /* Shared global vars are just accessed directly. */
3845 if (is_global_var (new_var
))
3847 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3848 needs to be delayed until after fixup_child_record_type so
3849 that we get the correct type during the dereference. */
3850 by_ref
= use_pointer_for_field (var
, ctx
);
3851 x
= build_receiver_ref (var
, by_ref
, ctx
);
3852 SET_DECL_VALUE_EXPR (new_var
, x
);
3853 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3855 /* ??? If VAR is not passed by reference, and the variable
3856 hasn't been initialized yet, then we'll get a warning for
3857 the store into the omp_data_s structure. Ideally, we'd be
3858 able to notice this and not store anything at all, but
3859 we're generating code too early. Suppress the warning. */
3861 TREE_NO_WARNING (var
) = 1;
3864 case OMP_CLAUSE_LASTPRIVATE
:
3865 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3869 case OMP_CLAUSE_PRIVATE
:
3870 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3871 x
= build_outer_var_ref (var
, ctx
);
3872 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3874 if (is_task_ctx (ctx
))
3875 x
= build_receiver_ref (var
, false, ctx
);
3877 x
= build_outer_var_ref (var
, ctx
);
3883 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3886 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3887 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3888 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3889 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3890 idx
, lane
, ivar
, lvar
))
3893 x
= lang_hooks
.decls
.omp_clause_default_ctor
3894 (c
, unshare_expr (ivar
), x
);
3896 gimplify_and_add (x
, &llist
[0]);
3899 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3902 gimple_seq tseq
= NULL
;
3905 gimplify_stmt (&dtor
, &tseq
);
3906 gimple_seq_add_seq (&llist
[1], tseq
);
3913 gimplify_and_add (nx
, ilist
);
3917 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3920 gimple_seq tseq
= NULL
;
3923 gimplify_stmt (&dtor
, &tseq
);
3924 gimple_seq_add_seq (dlist
, tseq
);
3928 case OMP_CLAUSE_LINEAR
:
3929 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3930 goto do_firstprivate
;
3931 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3934 x
= build_outer_var_ref (var
, ctx
);
3937 case OMP_CLAUSE_FIRSTPRIVATE
:
3938 if (is_task_ctx (ctx
))
3940 if (is_reference (var
) || is_variable_sized (var
))
3942 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3944 || use_pointer_for_field (var
, NULL
))
3946 x
= build_receiver_ref (var
, false, ctx
);
3947 SET_DECL_VALUE_EXPR (new_var
, x
);
3948 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3953 x
= build_outer_var_ref (var
, ctx
);
3956 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3957 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3959 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3960 tree stept
= TREE_TYPE (t
);
3961 tree ct
= find_omp_clause (clauses
,
3962 OMP_CLAUSE__LOOPTEMP_
);
3964 tree l
= OMP_CLAUSE_DECL (ct
);
3965 tree n1
= fd
->loop
.n1
;
3966 tree step
= fd
->loop
.step
;
3967 tree itype
= TREE_TYPE (l
);
3968 if (POINTER_TYPE_P (itype
))
3969 itype
= signed_type_for (itype
);
3970 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3971 if (TYPE_UNSIGNED (itype
)
3972 && fd
->loop
.cond_code
== GT_EXPR
)
3973 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3974 fold_build1 (NEGATE_EXPR
, itype
, l
),
3975 fold_build1 (NEGATE_EXPR
,
3978 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3979 t
= fold_build2 (MULT_EXPR
, stept
,
3980 fold_convert (stept
, l
), t
);
3982 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3984 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3986 gimplify_and_add (x
, ilist
);
3990 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3991 x
= fold_build2 (POINTER_PLUS_EXPR
,
3992 TREE_TYPE (x
), x
, t
);
3994 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3997 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3998 || TREE_ADDRESSABLE (new_var
))
3999 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4000 idx
, lane
, ivar
, lvar
))
4002 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
4004 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
4005 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
4006 gimplify_and_add (x
, ilist
);
4007 gimple_stmt_iterator gsi
4008 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4010 = gimple_build_assign (unshare_expr (lvar
), iv
);
4011 gsi_insert_before_without_update (&gsi
, g
,
4013 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4014 enum tree_code code
= PLUS_EXPR
;
4015 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4016 code
= POINTER_PLUS_EXPR
;
4017 g
= gimple_build_assign (iv
, code
, iv
, t
);
4018 gsi_insert_before_without_update (&gsi
, g
,
4022 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4023 (c
, unshare_expr (ivar
), x
);
4024 gimplify_and_add (x
, &llist
[0]);
4025 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4028 gimple_seq tseq
= NULL
;
4031 gimplify_stmt (&dtor
, &tseq
);
4032 gimple_seq_add_seq (&llist
[1], tseq
);
4037 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
4038 gimplify_and_add (x
, ilist
);
4041 case OMP_CLAUSE__LOOPTEMP_
:
4042 gcc_assert (is_parallel_ctx (ctx
));
4043 x
= build_outer_var_ref (var
, ctx
);
4044 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4045 gimplify_and_add (x
, ilist
);
4048 case OMP_CLAUSE_COPYIN
:
4049 by_ref
= use_pointer_for_field (var
, NULL
);
4050 x
= build_receiver_ref (var
, by_ref
, ctx
);
4051 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4052 append_to_statement_list (x
, ©in_seq
);
4053 copyin_by_ref
|= by_ref
;
4056 case OMP_CLAUSE_REDUCTION
:
4057 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4059 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4061 x
= build_outer_var_ref (var
, ctx
);
4063 if (is_reference (var
)
4064 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4066 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4067 SET_DECL_VALUE_EXPR (placeholder
, x
);
4068 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4069 tree new_vard
= new_var
;
4070 if (is_reference (var
))
4072 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4073 new_vard
= TREE_OPERAND (new_var
, 0);
4074 gcc_assert (DECL_P (new_vard
));
4077 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4078 idx
, lane
, ivar
, lvar
))
4080 if (new_vard
== new_var
)
4082 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4083 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4087 SET_DECL_VALUE_EXPR (new_vard
,
4088 build_fold_addr_expr (ivar
));
4089 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4091 x
= lang_hooks
.decls
.omp_clause_default_ctor
4092 (c
, unshare_expr (ivar
),
4093 build_outer_var_ref (var
, ctx
));
4095 gimplify_and_add (x
, &llist
[0]);
4096 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4098 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4099 lower_omp (&tseq
, ctx
);
4100 gimple_seq_add_seq (&llist
[0], tseq
);
4102 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4103 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4104 lower_omp (&tseq
, ctx
);
4105 gimple_seq_add_seq (&llist
[1], tseq
);
4106 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4107 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4108 if (new_vard
== new_var
)
4109 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4111 SET_DECL_VALUE_EXPR (new_vard
,
4112 build_fold_addr_expr (lvar
));
4113 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4118 gimplify_stmt (&dtor
, &tseq
);
4119 gimple_seq_add_seq (&llist
[1], tseq
);
4123 /* If this is a reference to constant size reduction var
4124 with placeholder, we haven't emitted the initializer
4125 for it because it is undesirable if SIMD arrays are used.
4126 But if they aren't used, we need to emit the deferred
4127 initialization now. */
4128 else if (is_reference (var
) && is_simd
)
4129 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4130 x
= lang_hooks
.decls
.omp_clause_default_ctor
4131 (c
, unshare_expr (new_var
),
4132 build_outer_var_ref (var
, ctx
));
4134 gimplify_and_add (x
, ilist
);
4135 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4137 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4138 lower_omp (&tseq
, ctx
);
4139 gimple_seq_add_seq (ilist
, tseq
);
4141 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4144 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4145 lower_omp (&tseq
, ctx
);
4146 gimple_seq_add_seq (dlist
, tseq
);
4147 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4149 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4154 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4155 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4156 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4158 /* reduction(-:var) sums up the partial results, so it
4159 acts identically to reduction(+:var). */
4160 if (code
== MINUS_EXPR
)
4163 tree new_vard
= new_var
;
4164 if (is_simd
&& is_reference (var
))
4166 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4167 new_vard
= TREE_OPERAND (new_var
, 0);
4168 gcc_assert (DECL_P (new_vard
));
4171 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4172 idx
, lane
, ivar
, lvar
))
4174 tree ref
= build_outer_var_ref (var
, ctx
);
4176 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
4178 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
4179 ref
= build_outer_var_ref (var
, ctx
);
4180 gimplify_assign (ref
, x
, &llist
[1]);
4182 if (new_vard
!= new_var
)
4184 SET_DECL_VALUE_EXPR (new_vard
,
4185 build_fold_addr_expr (lvar
));
4186 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4191 if (is_reference (var
) && is_simd
)
4192 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4193 gimplify_assign (new_var
, x
, ilist
);
4196 tree ref
= build_outer_var_ref (var
, ctx
);
4198 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4199 ref
= build_outer_var_ref (var
, ctx
);
4200 gimplify_assign (ref
, x
, dlist
);
4214 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
4215 /* Don't want uninit warnings on simduid, it is always uninitialized,
4216 but we use it not for the value, but for the DECL_UID only. */
4217 TREE_NO_WARNING (uid
) = 1;
4219 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
4220 gimple_call_set_lhs (g
, lane
);
4221 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4222 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
4223 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
4224 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
4225 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4226 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4227 g
= gimple_build_assign (lane
, INTEGER_CST
,
4228 build_int_cst (unsigned_type_node
, 0));
4229 gimple_seq_add_stmt (ilist
, g
);
4230 for (int i
= 0; i
< 2; i
++)
4233 tree vf
= create_tmp_var (unsigned_type_node
);
4234 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
4235 gimple_call_set_lhs (g
, vf
);
4236 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
4237 gimple_seq_add_stmt (seq
, g
);
4238 tree t
= build_int_cst (unsigned_type_node
, 0);
4239 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
4240 gimple_seq_add_stmt (seq
, g
);
4241 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4242 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
4243 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4244 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
4245 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
4246 gimple_seq_add_seq (seq
, llist
[i
]);
4247 t
= build_int_cst (unsigned_type_node
, 1);
4248 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
4249 gimple_seq_add_stmt (seq
, g
);
4250 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
4251 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
4252 gimple_seq_add_stmt (seq
, g
);
4253 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
4257 /* The copyin sequence is not to be executed by the main thread, since
4258 that would result in self-copies. Perhaps not visible to scalars,
4259 but it certainly is to C++ operator=. */
4262 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
4264 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
4265 build_int_cst (TREE_TYPE (x
), 0));
4266 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
4267 gimplify_and_add (x
, ilist
);
4270 /* If any copyin variable is passed by reference, we must ensure the
4271 master thread doesn't modify it before it is copied over in all
4272 threads. Similarly for variables in both firstprivate and
4273 lastprivate clauses we need to ensure the lastprivate copying
4274 happens after firstprivate copying in all threads. And similarly
4275 for UDRs if initializer expression refers to omp_orig. */
4276 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
4278 /* Don't add any barrier for #pragma omp simd or
4279 #pragma omp distribute. */
4280 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
4281 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
4282 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
4285 /* If max_vf is non-zero, then we can use only a vectorization factor
4286 up to the max_vf we chose. So stick it into the safelen clause. */
4289 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4290 OMP_CLAUSE_SAFELEN
);
4292 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
4293 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4296 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
4297 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
4299 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4300 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4306 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4307 both parallel and workshare constructs. PREDICATE may be NULL if it's
4311 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
4314 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
4315 bool par_clauses
= false;
4316 tree simduid
= NULL
, lastlane
= NULL
;
4318 /* Early exit if there are no lastprivate or linear clauses. */
4319 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
4320 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
4321 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
4322 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
4324 if (clauses
== NULL
)
4326 /* If this was a workshare clause, see if it had been combined
4327 with its parallel. In that case, look for the clauses on the
4328 parallel statement itself. */
4329 if (is_parallel_ctx (ctx
))
4333 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4336 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4337 OMP_CLAUSE_LASTPRIVATE
);
4338 if (clauses
== NULL
)
4346 tree label_true
, arm1
, arm2
;
4348 label
= create_artificial_label (UNKNOWN_LOCATION
);
4349 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
4350 arm1
= TREE_OPERAND (predicate
, 0);
4351 arm2
= TREE_OPERAND (predicate
, 1);
4352 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4353 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4354 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
4356 gimple_seq_add_stmt (stmt_list
, stmt
);
4357 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
4360 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4361 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4363 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
4365 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
4368 for (c
= clauses
; c
;)
4371 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4373 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4374 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4375 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
4377 var
= OMP_CLAUSE_DECL (c
);
4378 new_var
= lookup_decl (var
, ctx
);
4380 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
4382 tree val
= DECL_VALUE_EXPR (new_var
);
4383 if (TREE_CODE (val
) == ARRAY_REF
4384 && VAR_P (TREE_OPERAND (val
, 0))
4385 && lookup_attribute ("omp simd array",
4386 DECL_ATTRIBUTES (TREE_OPERAND (val
,
4389 if (lastlane
== NULL
)
4391 lastlane
= create_tmp_var (unsigned_type_node
);
4393 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4395 TREE_OPERAND (val
, 1));
4396 gimple_call_set_lhs (g
, lastlane
);
4397 gimple_seq_add_stmt (stmt_list
, g
);
4399 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4400 TREE_OPERAND (val
, 0), lastlane
,
4401 NULL_TREE
, NULL_TREE
);
4405 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4406 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4408 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4409 gimple_seq_add_seq (stmt_list
,
4410 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4411 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4413 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4414 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4416 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4417 gimple_seq_add_seq (stmt_list
,
4418 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4419 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4422 x
= build_outer_var_ref (var
, ctx
);
4423 if (is_reference (var
))
4424 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4425 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4426 gimplify_and_add (x
, stmt_list
);
4428 c
= OMP_CLAUSE_CHAIN (c
);
4429 if (c
== NULL
&& !par_clauses
)
4431 /* If this was a workshare clause, see if it had been combined
4432 with its parallel. In that case, continue looking for the
4433 clauses also on the parallel statement itself. */
4434 if (is_parallel_ctx (ctx
))
4438 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4441 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4442 OMP_CLAUSE_LASTPRIVATE
);
4448 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4452 oacc_lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
,
4453 tree tid
, tree var
, tree new_var
)
4455 /* The atomic add at the end of the sum creates unnecessary
4456 write contention on accelerators. To work around this,
4457 create an array to store the partial reductions. Later, in
4458 lower_omp_for (for openacc), the values of array will be
4461 tree t
= NULL_TREE
, array
, x
;
4462 tree type
= get_base_type (var
);
4465 /* Now insert the partial reductions into the array. */
4467 /* Find the reduction array. */
4469 tree ptype
= build_pointer_type (type
);
4471 t
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
), ctx
);
4472 t
= build_receiver_ref (t
, false, ctx
->outer
);
4474 array
= create_tmp_var (ptype
);
4475 gimplify_assign (array
, t
, stmt_seqp
);
4477 tree ptr
= create_tmp_var (TREE_TYPE (array
));
4479 /* Find the reduction array. */
4481 /* testing a unary conversion. */
4482 tree offset
= create_tmp_var (sizetype
);
4483 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
4485 t
= create_tmp_var (sizetype
);
4486 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
4488 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, t
);
4489 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4491 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4492 of adding sizeof(var) to the array? */
4493 ptr
= create_tmp_var (ptype
);
4494 stmt
= gimple_build_assign (unshare_expr (ptr
), POINTER_PLUS_EXPR
, array
,
4496 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4498 /* Move the local sum to gfc$sum[i]. */
4499 x
= unshare_expr (build_simple_mem_ref (ptr
));
4500 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
4503 /* Generate code to implement the REDUCTION clauses. */
4506 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4508 gimple_seq sub_seq
= NULL
;
4510 tree x
, c
, tid
= NULL_TREE
;
4513 /* SIMD reductions are handled in lower_rec_input_clauses. */
4514 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4515 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4518 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4519 update in that case, otherwise use a lock. */
4520 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4521 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4523 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4525 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4535 /* Initialize thread info for OpenACC. */
4536 if (is_gimple_omp_oacc (ctx
->stmt
))
4538 /* Get the current thread id. */
4539 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
4540 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)));
4541 gimple stmt
= gimple_build_call (call
, 0);
4542 gimple_call_set_lhs (stmt
, tid
);
4543 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4546 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4548 tree var
, ref
, new_var
;
4549 enum tree_code code
;
4550 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4552 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4555 var
= OMP_CLAUSE_DECL (c
);
4556 new_var
= lookup_decl (var
, ctx
);
4557 if (is_reference (var
))
4558 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4559 ref
= build_outer_var_ref (var
, ctx
);
4560 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4562 /* reduction(-:var) sums up the partial results, so it acts
4563 identically to reduction(+:var). */
4564 if (code
== MINUS_EXPR
)
4567 if (is_gimple_omp_oacc (ctx
->stmt
))
4569 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
));
4571 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
4573 else if (count
== 1)
4575 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4577 addr
= save_expr (addr
);
4578 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4579 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4580 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4581 gimplify_and_add (x
, stmt_seqp
);
4584 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4586 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4588 if (is_reference (var
)
4589 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4591 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4592 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4593 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4594 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4595 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4596 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4597 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4601 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4602 ref
= build_outer_var_ref (var
, ctx
);
4603 gimplify_assign (ref
, x
, &sub_seq
);
4607 if (is_gimple_omp_oacc (ctx
->stmt
))
4610 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4612 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4614 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4616 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4618 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4622 /* Generate code to implement the COPYPRIVATE clauses. */
4625 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4630 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4632 tree var
, new_var
, ref
, x
;
4634 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4636 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4639 var
= OMP_CLAUSE_DECL (c
);
4640 by_ref
= use_pointer_for_field (var
, NULL
);
4642 ref
= build_sender_ref (var
, ctx
);
4643 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4646 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4647 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4649 gimplify_assign (ref
, x
, slist
);
4651 ref
= build_receiver_ref (var
, false, ctx
);
4654 ref
= fold_convert_loc (clause_loc
,
4655 build_pointer_type (TREE_TYPE (new_var
)),
4657 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4659 if (is_reference (var
))
4661 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4662 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4663 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4665 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4666 gimplify_and_add (x
, rlist
);
4671 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4672 and REDUCTION from the sender (aka parent) side. */
4675 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4680 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4682 tree val
, ref
, x
, var
;
4683 bool by_ref
, do_in
= false, do_out
= false;
4684 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4686 switch (OMP_CLAUSE_CODE (c
))
4688 case OMP_CLAUSE_PRIVATE
:
4689 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4692 case OMP_CLAUSE_FIRSTPRIVATE
:
4693 case OMP_CLAUSE_COPYIN
:
4694 case OMP_CLAUSE_LASTPRIVATE
:
4695 case OMP_CLAUSE_REDUCTION
:
4696 case OMP_CLAUSE__LOOPTEMP_
:
4702 val
= OMP_CLAUSE_DECL (c
);
4703 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4705 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4706 && is_global_var (var
))
4708 if (is_variable_sized (val
))
4710 by_ref
= use_pointer_for_field (val
, NULL
);
4712 switch (OMP_CLAUSE_CODE (c
))
4714 case OMP_CLAUSE_PRIVATE
:
4715 case OMP_CLAUSE_FIRSTPRIVATE
:
4716 case OMP_CLAUSE_COPYIN
:
4717 case OMP_CLAUSE__LOOPTEMP_
:
4721 case OMP_CLAUSE_LASTPRIVATE
:
4722 if (by_ref
|| is_reference (val
))
4724 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4731 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4736 case OMP_CLAUSE_REDUCTION
:
4738 do_out
= !(by_ref
|| is_reference (val
));
4747 ref
= build_sender_ref (val
, ctx
);
4748 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4749 gimplify_assign (ref
, x
, ilist
);
4750 if (is_task_ctx (ctx
))
4751 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4756 ref
= build_sender_ref (val
, ctx
);
4757 gimplify_assign (var
, ref
, olist
);
4762 /* Generate code to implement SHARED from the sender (aka parent)
4763 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4764 list things that got automatically shared. */
4767 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4769 tree var
, ovar
, nvar
, f
, x
, record_type
;
4771 if (ctx
->record_type
== NULL
)
4774 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4775 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4777 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4778 nvar
= maybe_lookup_decl (ovar
, ctx
);
4779 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4782 /* If CTX is a nested parallel directive. Find the immediately
4783 enclosing parallel or workshare construct that contains a
4784 mapping for OVAR. */
4785 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4787 if (use_pointer_for_field (ovar
, ctx
))
4789 x
= build_sender_ref (ovar
, ctx
);
4790 var
= build_fold_addr_expr (var
);
4791 gimplify_assign (x
, var
, ilist
);
4795 x
= build_sender_ref (ovar
, ctx
);
4796 gimplify_assign (x
, var
, ilist
);
4798 if (!TREE_READONLY (var
)
4799 /* We don't need to receive a new reference to a result
4800 or parm decl. In fact we may not store to it as we will
4801 invalidate any pending RSO and generate wrong gimple
4803 && !((TREE_CODE (var
) == RESULT_DECL
4804 || TREE_CODE (var
) == PARM_DECL
)
4805 && DECL_BY_REFERENCE (var
)))
4807 x
= build_sender_ref (ovar
, ctx
);
4808 gimplify_assign (var
, x
, olist
);
4815 /* A convenience function to build an empty GIMPLE_COND with just the
4819 gimple_build_cond_empty (tree cond
)
4821 enum tree_code pred_code
;
4824 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4825 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4829 /* Build the function calls to GOMP_parallel_start etc to actually
4830 generate the parallel operation. REGION is the parallel region
4831 being expanded. BB is the block where to insert the code. WS_ARGS
4832 will be set if this is a call to a combined parallel+workshare
4833 construct, it contains the list of additional arguments needed by
4834 the workshare construct. */
4837 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4838 gomp_parallel
*entry_stmt
,
4839 vec
<tree
, va_gc
> *ws_args
)
4841 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4842 gimple_stmt_iterator gsi
;
4844 enum built_in_function start_ix
;
4846 location_t clause_loc
;
4847 vec
<tree
, va_gc
> *args
;
4849 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4851 /* Determine what flavor of GOMP_parallel we will be
4853 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4854 if (is_combined_parallel (region
))
4856 switch (region
->inner
->type
)
4858 case GIMPLE_OMP_FOR
:
4859 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4860 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4861 + (region
->inner
->sched_kind
4862 == OMP_CLAUSE_SCHEDULE_RUNTIME
4863 ? 3 : region
->inner
->sched_kind
));
4864 start_ix
= (enum built_in_function
)start_ix2
;
4866 case GIMPLE_OMP_SECTIONS
:
4867 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4874 /* By default, the value of NUM_THREADS is zero (selected at run time)
4875 and there is no conditional. */
4877 val
= build_int_cst (unsigned_type_node
, 0);
4878 flags
= build_int_cst (unsigned_type_node
, 0);
4880 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4882 cond
= OMP_CLAUSE_IF_EXPR (c
);
4884 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4887 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4888 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4891 clause_loc
= gimple_location (entry_stmt
);
4893 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4895 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4897 /* Ensure 'val' is of the correct type. */
4898 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4900 /* If we found the clause 'if (cond)', build either
4901 (cond != 0) or (cond ? val : 1u). */
4904 cond
= gimple_boolify (cond
);
4906 if (integer_zerop (val
))
4907 val
= fold_build2_loc (clause_loc
,
4908 EQ_EXPR
, unsigned_type_node
, cond
,
4909 build_int_cst (TREE_TYPE (cond
), 0));
4912 basic_block cond_bb
, then_bb
, else_bb
;
4913 edge e
, e_then
, e_else
;
4914 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4916 tmp_var
= create_tmp_var (TREE_TYPE (val
));
4917 if (gimple_in_ssa_p (cfun
))
4919 tmp_then
= make_ssa_name (tmp_var
);
4920 tmp_else
= make_ssa_name (tmp_var
);
4921 tmp_join
= make_ssa_name (tmp_var
);
4930 e
= split_block_after_labels (bb
);
4935 then_bb
= create_empty_bb (cond_bb
);
4936 else_bb
= create_empty_bb (then_bb
);
4937 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4938 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4940 stmt
= gimple_build_cond_empty (cond
);
4941 gsi
= gsi_start_bb (cond_bb
);
4942 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4944 gsi
= gsi_start_bb (then_bb
);
4945 stmt
= gimple_build_assign (tmp_then
, val
);
4946 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4948 gsi
= gsi_start_bb (else_bb
);
4949 stmt
= gimple_build_assign
4950 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4951 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4953 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4954 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4955 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4956 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4957 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4958 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4960 if (gimple_in_ssa_p (cfun
))
4962 gphi
*phi
= create_phi_node (tmp_join
, bb
);
4963 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4964 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4970 gsi
= gsi_start_bb (bb
);
4971 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4972 false, GSI_CONTINUE_LINKING
);
4975 gsi
= gsi_last_bb (bb
);
4976 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4978 t1
= null_pointer_node
;
4980 t1
= build_fold_addr_expr (t
);
4981 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4983 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4984 args
->quick_push (t2
);
4985 args
->quick_push (t1
);
4986 args
->quick_push (val
);
4988 args
->splice (*ws_args
);
4989 args
->quick_push (flags
);
4991 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4992 builtin_decl_explicit (start_ix
), args
);
4994 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4995 false, GSI_CONTINUE_LINKING
);
4998 /* Insert a function call whose name is FUNC_NAME with the information from
4999 ENTRY_STMT into the basic_block BB. */
5002 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
5003 vec
<tree
, va_gc
> *ws_args
)
5006 gimple_stmt_iterator gsi
;
5007 vec
<tree
, va_gc
> *args
;
5009 gcc_assert (vec_safe_length (ws_args
) == 2);
5010 tree func_name
= (*ws_args
)[0];
5011 tree grain
= (*ws_args
)[1];
5013 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
5014 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
5015 gcc_assert (count
!= NULL_TREE
);
5016 count
= OMP_CLAUSE_OPERAND (count
, 0);
5018 gsi
= gsi_last_bb (bb
);
5019 t
= gimple_omp_parallel_data_arg (entry_stmt
);
5021 t1
= null_pointer_node
;
5023 t1
= build_fold_addr_expr (t
);
5024 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5026 vec_alloc (args
, 4);
5027 args
->quick_push (t2
);
5028 args
->quick_push (t1
);
5029 args
->quick_push (count
);
5030 args
->quick_push (grain
);
5031 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
5033 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
5034 GSI_CONTINUE_LINKING
);
5037 /* Build the function call to GOMP_task to actually
5038 generate the task operation. BB is the block where to insert the code. */
5041 expand_task_call (basic_block bb
, gomp_task
*entry_stmt
)
5043 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
5044 gimple_stmt_iterator gsi
;
5045 location_t loc
= gimple_location (entry_stmt
);
5047 clauses
= gimple_omp_task_clauses (entry_stmt
);
5049 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
5051 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
5053 cond
= boolean_true_node
;
5055 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
5056 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
5057 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
5058 flags
= build_int_cst (unsigned_type_node
,
5059 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
5061 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
5064 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
5065 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
5066 build_int_cst (unsigned_type_node
, 2),
5067 build_int_cst (unsigned_type_node
, 0));
5068 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
5071 depend
= OMP_CLAUSE_DECL (depend
);
5073 depend
= build_int_cst (ptr_type_node
, 0);
5075 gsi
= gsi_last_bb (bb
);
5076 t
= gimple_omp_task_data_arg (entry_stmt
);
5078 t2
= null_pointer_node
;
5080 t2
= build_fold_addr_expr_loc (loc
, t
);
5081 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
5082 t
= gimple_omp_task_copy_fn (entry_stmt
);
5084 t3
= null_pointer_node
;
5086 t3
= build_fold_addr_expr_loc (loc
, t
);
5088 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
5090 gimple_omp_task_arg_size (entry_stmt
),
5091 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
5094 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5095 false, GSI_CONTINUE_LINKING
);
5099 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5100 catch handler and return it. This prevents programs from violating the
5101 structured block semantics with throws. */
5104 maybe_catch_exception (gimple_seq body
)
5109 if (!flag_exceptions
)
5112 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
5113 decl
= lang_hooks
.eh_protect_cleanup_actions ();
5115 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
5117 g
= gimple_build_eh_must_not_throw (decl
);
5118 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
5121 return gimple_seq_alloc_with_stmt (g
);
5124 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5127 vec2chain (vec
<tree
, va_gc
> *v
)
5129 tree chain
= NULL_TREE
, t
;
5132 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
5134 DECL_CHAIN (t
) = chain
;
5142 /* Remove barriers in REGION->EXIT's block. Note that this is only
5143 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5144 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5145 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5149 remove_exit_barrier (struct omp_region
*region
)
5151 gimple_stmt_iterator gsi
;
5152 basic_block exit_bb
;
5156 int any_addressable_vars
= -1;
5158 exit_bb
= region
->exit
;
5160 /* If the parallel region doesn't return, we don't have REGION->EXIT
5165 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5166 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5167 statements that can appear in between are extremely limited -- no
5168 memory operations at all. Here, we allow nothing at all, so the
5169 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5170 gsi
= gsi_last_bb (exit_bb
);
5171 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5173 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
5176 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
5178 gsi
= gsi_last_bb (e
->src
);
5179 if (gsi_end_p (gsi
))
5181 stmt
= gsi_stmt (gsi
);
5182 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
5183 && !gimple_omp_return_nowait_p (stmt
))
5185 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5186 in many cases. If there could be tasks queued, the barrier
5187 might be needed to let the tasks run before some local
5188 variable of the parallel that the task uses as shared
5189 runs out of scope. The task can be spawned either
5190 from within current function (this would be easy to check)
5191 or from some function it calls and gets passed an address
5192 of such a variable. */
5193 if (any_addressable_vars
< 0)
5195 gomp_parallel
*parallel_stmt
5196 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
5197 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
5198 tree local_decls
, block
, decl
;
5201 any_addressable_vars
= 0;
5202 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
5203 if (TREE_ADDRESSABLE (decl
))
5205 any_addressable_vars
= 1;
5208 for (block
= gimple_block (stmt
);
5209 !any_addressable_vars
5211 && TREE_CODE (block
) == BLOCK
;
5212 block
= BLOCK_SUPERCONTEXT (block
))
5214 for (local_decls
= BLOCK_VARS (block
);
5216 local_decls
= DECL_CHAIN (local_decls
))
5217 if (TREE_ADDRESSABLE (local_decls
))
5219 any_addressable_vars
= 1;
5222 if (block
== gimple_block (parallel_stmt
))
5226 if (!any_addressable_vars
)
5227 gimple_omp_return_set_nowait (stmt
);
5233 remove_exit_barriers (struct omp_region
*region
)
5235 if (region
->type
== GIMPLE_OMP_PARALLEL
)
5236 remove_exit_barrier (region
);
5240 region
= region
->inner
;
5241 remove_exit_barriers (region
);
5242 while (region
->next
)
5244 region
= region
->next
;
5245 remove_exit_barriers (region
);
5250 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5251 calls. These can't be declared as const functions, but
5252 within one parallel body they are constant, so they can be
5253 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5254 which are declared const. Similarly for task body, except
5255 that in untied task omp_get_thread_num () can change at any task
5256 scheduling point. */
5259 optimize_omp_library_calls (gimple entry_stmt
)
5262 gimple_stmt_iterator gsi
;
5263 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5264 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
5265 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5266 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
5267 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
5268 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
5269 OMP_CLAUSE_UNTIED
) != NULL
);
5271 FOR_EACH_BB_FN (bb
, cfun
)
5272 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5274 gimple call
= gsi_stmt (gsi
);
5277 if (is_gimple_call (call
)
5278 && (decl
= gimple_call_fndecl (call
))
5279 && DECL_EXTERNAL (decl
)
5280 && TREE_PUBLIC (decl
)
5281 && DECL_INITIAL (decl
) == NULL
)
5285 if (DECL_NAME (decl
) == thr_num_id
)
5287 /* In #pragma omp task untied omp_get_thread_num () can change
5288 during the execution of the task region. */
5291 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5293 else if (DECL_NAME (decl
) == num_thr_id
)
5294 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5298 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
5299 || gimple_call_num_args (call
) != 0)
5302 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
5305 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
5306 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
5307 TREE_TYPE (TREE_TYPE (built_in
))))
5310 gimple_call_set_fndecl (call
, built_in
);
5315 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5319 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
5323 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5324 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
5327 if (TREE_CODE (t
) == ADDR_EXPR
)
5328 recompute_tree_invariant_for_addr_expr (t
);
5330 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
5334 /* Prepend TO = FROM assignment before *GSI_P. */
5337 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
5339 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
5340 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
5341 true, GSI_SAME_STMT
);
5342 gimple stmt
= gimple_build_assign (to
, from
);
5343 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
5344 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
5345 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
5347 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
5348 gimple_regimplify_operands (stmt
, &gsi
);
5352 /* Expand the OpenMP parallel or task directive starting at REGION. */
5355 expand_omp_taskreg (struct omp_region
*region
)
5357 basic_block entry_bb
, exit_bb
, new_bb
;
5358 struct function
*child_cfun
;
5359 tree child_fn
, block
, t
;
5360 gimple_stmt_iterator gsi
;
5361 gimple entry_stmt
, stmt
;
5363 vec
<tree
, va_gc
> *ws_args
;
5365 entry_stmt
= last_stmt (region
->entry
);
5366 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
5367 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
5369 entry_bb
= region
->entry
;
5370 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
5371 exit_bb
= region
->cont
;
5373 exit_bb
= region
->exit
;
5377 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
5378 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
5379 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
5382 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5383 and the inner statement contains the name of the built-in function
5385 ws_args
= region
->inner
->ws_args
;
5386 else if (is_combined_parallel (region
))
5387 ws_args
= region
->ws_args
;
5391 if (child_cfun
->cfg
)
5393 /* Due to inlining, it may happen that we have already outlined
5394 the region, in which case all we need to do is make the
5395 sub-graph unreachable and emit the parallel call. */
5396 edge entry_succ_e
, exit_succ_e
;
5398 entry_succ_e
= single_succ_edge (entry_bb
);
5400 gsi
= gsi_last_bb (entry_bb
);
5401 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
5402 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
5403 gsi_remove (&gsi
, true);
5408 exit_succ_e
= single_succ_edge (exit_bb
);
5409 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
5411 remove_edge_and_dominated_blocks (entry_succ_e
);
5415 unsigned srcidx
, dstidx
, num
;
5417 /* If the parallel region needs data sent from the parent
5418 function, then the very first statement (except possible
5419 tree profile counter updates) of the parallel body
5420 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5421 &.OMP_DATA_O is passed as an argument to the child function,
5422 we need to replace it with the argument as seen by the child
5425 In most cases, this will end up being the identity assignment
5426 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5427 a function call that has been inlined, the original PARM_DECL
5428 .OMP_DATA_I may have been converted into a different local
5429 variable. In which case, we need to keep the assignment. */
5430 if (gimple_omp_taskreg_data_arg (entry_stmt
))
5432 basic_block entry_succ_bb
5433 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
5434 : FALLTHRU_EDGE (entry_bb
)->dest
;
5436 gimple parcopy_stmt
= NULL
;
5438 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
5442 gcc_assert (!gsi_end_p (gsi
));
5443 stmt
= gsi_stmt (gsi
);
5444 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
5447 if (gimple_num_ops (stmt
) == 2)
5449 tree arg
= gimple_assign_rhs1 (stmt
);
5451 /* We're ignore the subcode because we're
5452 effectively doing a STRIP_NOPS. */
5454 if (TREE_CODE (arg
) == ADDR_EXPR
5455 && TREE_OPERAND (arg
, 0)
5456 == gimple_omp_taskreg_data_arg (entry_stmt
))
5458 parcopy_stmt
= stmt
;
5464 gcc_assert (parcopy_stmt
!= NULL
);
5465 arg
= DECL_ARGUMENTS (child_fn
);
5467 if (!gimple_in_ssa_p (cfun
))
5469 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5470 gsi_remove (&gsi
, true);
5473 /* ?? Is setting the subcode really necessary ?? */
5474 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5475 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5480 /* If we are in ssa form, we must load the value from the default
5481 definition of the argument. That should not be defined now,
5482 since the argument is not used uninitialized. */
5483 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5484 narg
= make_ssa_name (arg
, gimple_build_nop ());
5485 set_ssa_default_def (cfun
, arg
, narg
);
5486 /* ?? Is setting the subcode really necessary ?? */
5487 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5488 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5489 update_stmt (parcopy_stmt
);
5493 /* Declare local variables needed in CHILD_CFUN. */
5494 block
= DECL_INITIAL (child_fn
);
5495 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5496 /* The gimplifier could record temporaries in parallel/task block
5497 rather than in containing function's local_decls chain,
5498 which would mean cgraph missed finalizing them. Do it now. */
5499 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5500 if (TREE_CODE (t
) == VAR_DECL
5502 && !DECL_EXTERNAL (t
))
5503 varpool_node::finalize_decl (t
);
5504 DECL_SAVED_TREE (child_fn
) = NULL
;
5505 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5506 gimple_set_body (child_fn
, NULL
);
5507 TREE_USED (block
) = 1;
5509 /* Reset DECL_CONTEXT on function arguments. */
5510 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5511 DECL_CONTEXT (t
) = child_fn
;
5513 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5514 so that it can be moved to the child function. */
5515 gsi
= gsi_last_bb (entry_bb
);
5516 stmt
= gsi_stmt (gsi
);
5517 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5518 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5519 e
= split_block (entry_bb
, stmt
);
5520 gsi_remove (&gsi
, true);
5523 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5524 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5527 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
5528 gcc_assert (e2
->dest
== region
->exit
);
5529 remove_edge (BRANCH_EDGE (entry_bb
));
5530 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
5531 gsi
= gsi_last_bb (region
->exit
);
5532 gcc_assert (!gsi_end_p (gsi
)
5533 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5534 gsi_remove (&gsi
, true);
5537 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
5540 gsi
= gsi_last_bb (exit_bb
);
5541 gcc_assert (!gsi_end_p (gsi
)
5542 && (gimple_code (gsi_stmt (gsi
))
5543 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
5544 stmt
= gimple_build_return (NULL
);
5545 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5546 gsi_remove (&gsi
, true);
5549 /* Move the parallel region into CHILD_CFUN. */
5551 if (gimple_in_ssa_p (cfun
))
5553 init_tree_ssa (child_cfun
);
5554 init_ssa_operands (child_cfun
);
5555 child_cfun
->gimple_df
->in_ssa_p
= true;
5559 block
= gimple_block (entry_stmt
);
5561 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5563 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5566 basic_block dest_bb
= e2
->dest
;
5568 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
5570 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
5572 /* When the OMP expansion process cannot guarantee an up-to-date
5573 loop tree arrange for the child function to fixup loops. */
5574 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5575 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5577 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5578 num
= vec_safe_length (child_cfun
->local_decls
);
5579 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5581 t
= (*child_cfun
->local_decls
)[srcidx
];
5582 if (DECL_CONTEXT (t
) == cfun
->decl
)
5584 if (srcidx
!= dstidx
)
5585 (*child_cfun
->local_decls
)[dstidx
] = t
;
5589 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5591 /* Inform the callgraph about the new function. */
5592 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5593 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
5594 node
->parallelized_function
= 1;
5595 cgraph_node::add_new_function (child_fn
, true);
5597 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5598 fixed in a following pass. */
5599 push_cfun (child_cfun
);
5601 optimize_omp_library_calls (entry_stmt
);
5602 cgraph_edge::rebuild_edges ();
5604 /* Some EH regions might become dead, see PR34608. If
5605 pass_cleanup_cfg isn't the first pass to happen with the
5606 new child, these dead EH edges might cause problems.
5607 Clean them up now. */
5608 if (flag_exceptions
)
5611 bool changed
= false;
5613 FOR_EACH_BB_FN (bb
, cfun
)
5614 changed
|= gimple_purge_dead_eh_edges (bb
);
5616 cleanup_tree_cfg ();
5618 if (gimple_in_ssa_p (cfun
))
5619 update_ssa (TODO_update_ssa
);
5623 /* Emit a library call to launch the children threads. */
5625 expand_cilk_for_call (new_bb
,
5626 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5627 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5628 expand_parallel_call (region
, new_bb
,
5629 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5631 expand_task_call (new_bb
, as_a
<gomp_task
*> (entry_stmt
));
5632 if (gimple_in_ssa_p (cfun
))
5633 update_ssa (TODO_update_ssa_only_virtuals
);
5637 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5638 of the combined collapse > 1 loop constructs, generate code like:
5639 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5644 count3 = (adj + N32 - N31) / STEP3;
5645 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5650 count2 = (adj + N22 - N21) / STEP2;
5651 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5656 count1 = (adj + N12 - N11) / STEP1;
5657 count = count1 * count2 * count3;
5658 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5660 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5661 of the combined loop constructs, just initialize COUNTS array
5662 from the _looptemp_ clauses. */
5664 /* NOTE: It *could* be better to moosh all of the BBs together,
5665 creating one larger BB with all the computation and the unexpected
5666 jump at the end. I.e.
5668 bool zero3, zero2, zero1, zero;
5671 count3 = (N32 - N31) /[cl] STEP3;
5673 count2 = (N22 - N21) /[cl] STEP2;
5675 count1 = (N12 - N11) /[cl] STEP1;
5676 zero = zero3 || zero2 || zero1;
5677 count = count1 * count2 * count3;
5678 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5680 After all, we expect the zero=false, and thus we expect to have to
5681 evaluate all of the comparison expressions, so short-circuiting
5682 oughtn't be a win. Since the condition isn't protecting a
5683 denominator, we're not concerned about divide-by-zero, so we can
5684 fully evaluate count even if a numerator turned out to be wrong.
5686 It seems like putting this all together would create much better
5687 scheduling opportunities, and less pressure on the chip's branch
5691 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5692 basic_block
&entry_bb
, tree
*counts
,
5693 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5694 basic_block
&l2_dom_bb
)
5696 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5700 /* Collapsed loops need work for expansion into SSA form. */
5701 gcc_assert (!gimple_in_ssa_p (cfun
));
5703 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5704 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5706 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5707 isn't supposed to be handled, as the inner loop doesn't
5709 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5710 OMP_CLAUSE__LOOPTEMP_
);
5711 gcc_assert (innerc
);
5712 for (i
= 0; i
< fd
->collapse
; i
++)
5714 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5715 OMP_CLAUSE__LOOPTEMP_
);
5716 gcc_assert (innerc
);
5718 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5720 counts
[0] = NULL_TREE
;
5725 for (i
= 0; i
< fd
->collapse
; i
++)
5727 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5729 if (SSA_VAR_P (fd
->loop
.n2
)
5730 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5731 fold_convert (itype
, fd
->loops
[i
].n1
),
5732 fold_convert (itype
, fd
->loops
[i
].n2
)))
5733 == NULL_TREE
|| !integer_onep (t
)))
5737 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5738 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5739 true, GSI_SAME_STMT
);
5740 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5741 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5742 true, GSI_SAME_STMT
);
5743 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5744 NULL_TREE
, NULL_TREE
);
5745 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
5746 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
5747 expand_omp_regimplify_p
, NULL
, NULL
)
5748 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
5749 expand_omp_regimplify_p
, NULL
, NULL
))
5751 *gsi
= gsi_for_stmt (cond_stmt
);
5752 gimple_regimplify_operands (cond_stmt
, gsi
);
5754 e
= split_block (entry_bb
, cond_stmt
);
5755 if (zero_iter_bb
== NULL
)
5757 gassign
*assign_stmt
;
5758 first_zero_iter
= i
;
5759 zero_iter_bb
= create_empty_bb (entry_bb
);
5760 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5761 *gsi
= gsi_after_labels (zero_iter_bb
);
5762 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
5763 build_zero_cst (type
));
5764 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
5765 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5768 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5769 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5770 e
->flags
= EDGE_TRUE_VALUE
;
5771 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5772 if (l2_dom_bb
== NULL
)
5773 l2_dom_bb
= entry_bb
;
5775 *gsi
= gsi_last_bb (entry_bb
);
5778 if (POINTER_TYPE_P (itype
))
5779 itype
= signed_type_for (itype
);
5780 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5782 t
= fold_build2 (PLUS_EXPR
, itype
,
5783 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5784 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5785 fold_convert (itype
, fd
->loops
[i
].n2
));
5786 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5787 fold_convert (itype
, fd
->loops
[i
].n1
));
5788 /* ?? We could probably use CEIL_DIV_EXPR instead of
5789 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5790 generate the same code in the end because generically we
5791 don't know that the values involved must be negative for
5793 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5794 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5795 fold_build1 (NEGATE_EXPR
, itype
, t
),
5796 fold_build1 (NEGATE_EXPR
, itype
,
5797 fold_convert (itype
,
5798 fd
->loops
[i
].step
)));
5800 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5801 fold_convert (itype
, fd
->loops
[i
].step
));
5802 t
= fold_convert (type
, t
);
5803 if (TREE_CODE (t
) == INTEGER_CST
)
5807 counts
[i
] = create_tmp_reg (type
, ".count");
5808 expand_omp_build_assign (gsi
, counts
[i
], t
);
5810 if (SSA_VAR_P (fd
->loop
.n2
))
5815 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5816 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5822 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5824 V3 = N31 + (T % count3) * STEP3;
5826 V2 = N21 + (T % count2) * STEP2;
5828 V1 = N11 + T * STEP1;
5829 if this loop doesn't have an inner loop construct combined with it.
5830 If it does have an inner loop construct combined with it and the
5831 iteration count isn't known constant, store values from counts array
5832 into its _looptemp_ temporaries instead. */
5835 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5836 tree
*counts
, gimple inner_stmt
, tree startvar
)
5839 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5841 /* If fd->loop.n2 is constant, then no propagation of the counts
5842 is needed, they are constant. */
5843 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5846 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5847 ? gimple_omp_parallel_clauses (inner_stmt
)
5848 : gimple_omp_for_clauses (inner_stmt
);
5849 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5850 isn't supposed to be handled, as the inner loop doesn't
5852 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5853 gcc_assert (innerc
);
5854 for (i
= 0; i
< fd
->collapse
; i
++)
5856 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5857 OMP_CLAUSE__LOOPTEMP_
);
5858 gcc_assert (innerc
);
5861 tree tem
= OMP_CLAUSE_DECL (innerc
);
5862 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5863 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5864 false, GSI_CONTINUE_LINKING
);
5865 gassign
*stmt
= gimple_build_assign (tem
, t
);
5866 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5872 tree type
= TREE_TYPE (fd
->loop
.v
);
5873 tree tem
= create_tmp_reg (type
, ".tem");
5874 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
5875 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5877 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5879 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5881 if (POINTER_TYPE_P (vtype
))
5882 itype
= signed_type_for (vtype
);
5884 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5887 t
= fold_convert (itype
, t
);
5888 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5889 fold_convert (itype
, fd
->loops
[i
].step
));
5890 if (POINTER_TYPE_P (vtype
))
5891 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5893 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5894 t
= force_gimple_operand_gsi (gsi
, t
,
5895 DECL_P (fd
->loops
[i
].v
)
5896 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5898 GSI_CONTINUE_LINKING
);
5899 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5900 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5903 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5904 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5905 false, GSI_CONTINUE_LINKING
);
5906 stmt
= gimple_build_assign (tem
, t
);
5907 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5913 /* Helper function for expand_omp_for_*. Generate code like:
5916 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5920 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5927 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5928 basic_block body_bb
)
5930 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5932 gimple_stmt_iterator gsi
;
5938 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5940 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5942 bb
= create_empty_bb (last_bb
);
5943 add_bb_to_loop (bb
, last_bb
->loop_father
);
5944 gsi
= gsi_start_bb (bb
);
5946 if (i
< fd
->collapse
- 1)
5948 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5949 e
->probability
= REG_BR_PROB_BASE
/ 8;
5951 t
= fd
->loops
[i
+ 1].n1
;
5952 t
= force_gimple_operand_gsi (&gsi
, t
,
5953 DECL_P (fd
->loops
[i
+ 1].v
)
5954 && TREE_ADDRESSABLE (fd
->loops
[i
5957 GSI_CONTINUE_LINKING
);
5958 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5959 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5964 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5966 if (POINTER_TYPE_P (vtype
))
5967 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5969 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5970 t
= force_gimple_operand_gsi (&gsi
, t
,
5971 DECL_P (fd
->loops
[i
].v
)
5972 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5973 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5974 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5975 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5979 t
= fd
->loops
[i
].n2
;
5980 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5981 false, GSI_CONTINUE_LINKING
);
5982 tree v
= fd
->loops
[i
].v
;
5983 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5984 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5985 false, GSI_CONTINUE_LINKING
);
5986 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5987 stmt
= gimple_build_cond_empty (t
);
5988 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5989 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5990 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5993 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
6001 /* A subroutine of expand_omp_for. Generate code for a parallel
6002 loop with any schedule. Given parameters:
6004 for (V = N1; V cond N2; V += STEP) BODY;
6006 where COND is "<" or ">", we generate pseudocode
6008 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
6009 if (more) goto L0; else goto L3;
6016 if (V cond iend) goto L1; else goto L2;
6018 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6021 If this is a combined omp parallel loop, instead of the call to
6022 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
6023 If this is gimple_omp_for_combined_p loop, then instead of assigning
6024 V and iend in L0 we assign the first two _looptemp_ clause decls of the
6025 inner GIMPLE_OMP_FOR and V += STEP; and
6026 if (V cond iend) goto L1; else goto L2; are removed.
6028 For collapsed loops, given parameters:
6030 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6031 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6032 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6035 we generate pseudocode
6037 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6042 count3 = (adj + N32 - N31) / STEP3;
6043 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6048 count2 = (adj + N22 - N21) / STEP2;
6049 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6054 count1 = (adj + N12 - N11) / STEP1;
6055 count = count1 * count2 * count3;
6060 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6061 if (more) goto L0; else goto L3;
6065 V3 = N31 + (T % count3) * STEP3;
6067 V2 = N21 + (T % count2) * STEP2;
6069 V1 = N11 + T * STEP1;
6074 if (V < iend) goto L10; else goto L2;
6077 if (V3 cond3 N32) goto L1; else goto L11;
6081 if (V2 cond2 N22) goto L1; else goto L12;
6087 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6093 expand_omp_for_generic (struct omp_region
*region
,
6094 struct omp_for_data
*fd
,
6095 enum built_in_function start_fn
,
6096 enum built_in_function next_fn
,
6099 tree type
, istart0
, iend0
, iend
;
6100 tree t
, vmain
, vback
, bias
= NULL_TREE
;
6101 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
6102 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
6103 gimple_stmt_iterator gsi
;
6104 gassign
*assign_stmt
;
6105 bool in_combined_parallel
= is_combined_parallel (region
);
6106 bool broken_loop
= region
->cont
== NULL
;
6108 tree
*counts
= NULL
;
6111 gcc_assert (!broken_loop
|| !in_combined_parallel
);
6112 gcc_assert (fd
->iter_type
== long_integer_type_node
6113 || !in_combined_parallel
);
6115 type
= TREE_TYPE (fd
->loop
.v
);
6116 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
6117 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
6118 TREE_ADDRESSABLE (istart0
) = 1;
6119 TREE_ADDRESSABLE (iend0
) = 1;
6121 /* See if we need to bias by LLONG_MIN. */
6122 if (fd
->iter_type
== long_long_unsigned_type_node
6123 && TREE_CODE (type
) == INTEGER_TYPE
6124 && !TYPE_UNSIGNED (type
))
6128 if (fd
->loop
.cond_code
== LT_EXPR
)
6131 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6135 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6138 if (TREE_CODE (n1
) != INTEGER_CST
6139 || TREE_CODE (n2
) != INTEGER_CST
6140 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
6141 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
6144 entry_bb
= region
->entry
;
6145 cont_bb
= region
->cont
;
6147 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6148 gcc_assert (broken_loop
6149 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6150 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6151 l1_bb
= single_succ (l0_bb
);
6154 l2_bb
= create_empty_bb (cont_bb
);
6155 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
6156 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6160 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6161 exit_bb
= region
->exit
;
6163 gsi
= gsi_last_bb (entry_bb
);
6165 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6166 if (fd
->collapse
> 1)
6168 int first_zero_iter
= -1;
6169 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
6171 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6172 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6173 zero_iter_bb
, first_zero_iter
,
6178 /* Some counts[i] vars might be uninitialized if
6179 some loop has zero iterations. But the body shouldn't
6180 be executed in that case, so just avoid uninit warnings. */
6181 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
6182 if (SSA_VAR_P (counts
[i
]))
6183 TREE_NO_WARNING (counts
[i
]) = 1;
6185 e
= split_block (entry_bb
, gsi_stmt (gsi
));
6187 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
6188 gsi
= gsi_last_bb (entry_bb
);
6189 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
6190 get_immediate_dominator (CDI_DOMINATORS
,
6194 if (in_combined_parallel
)
6196 /* In a combined parallel loop, emit a call to
6197 GOMP_loop_foo_next. */
6198 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6199 build_fold_addr_expr (istart0
),
6200 build_fold_addr_expr (iend0
));
6204 tree t0
, t1
, t2
, t3
, t4
;
6205 /* If this is not a combined parallel loop, emit a call to
6206 GOMP_loop_foo_start in ENTRY_BB. */
6207 t4
= build_fold_addr_expr (iend0
);
6208 t3
= build_fold_addr_expr (istart0
);
6209 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
6212 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6214 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6215 OMP_CLAUSE__LOOPTEMP_
);
6216 gcc_assert (innerc
);
6217 t0
= OMP_CLAUSE_DECL (innerc
);
6218 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6219 OMP_CLAUSE__LOOPTEMP_
);
6220 gcc_assert (innerc
);
6221 t1
= OMP_CLAUSE_DECL (innerc
);
6223 if (POINTER_TYPE_P (TREE_TYPE (t0
))
6224 && TYPE_PRECISION (TREE_TYPE (t0
))
6225 != TYPE_PRECISION (fd
->iter_type
))
6227 /* Avoid casting pointers to integer of a different size. */
6228 tree itype
= signed_type_for (type
);
6229 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
6230 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
6234 t1
= fold_convert (fd
->iter_type
, t1
);
6235 t0
= fold_convert (fd
->iter_type
, t0
);
6239 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
6240 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
6242 if (fd
->iter_type
== long_integer_type_node
)
6246 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6247 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6248 6, t0
, t1
, t2
, t
, t3
, t4
);
6251 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6252 5, t0
, t1
, t2
, t3
, t4
);
6260 /* The GOMP_loop_ull_*start functions have additional boolean
6261 argument, true for < loops and false for > loops.
6262 In Fortran, the C bool type can be different from
6263 boolean_type_node. */
6264 bfn_decl
= builtin_decl_explicit (start_fn
);
6265 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
6266 t5
= build_int_cst (c_bool_type
,
6267 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
6270 tree bfn_decl
= builtin_decl_explicit (start_fn
);
6271 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6272 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
6275 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6276 6, t5
, t0
, t1
, t2
, t3
, t4
);
6279 if (TREE_TYPE (t
) != boolean_type_node
)
6280 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6281 t
, build_int_cst (TREE_TYPE (t
), 0));
6282 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6283 true, GSI_SAME_STMT
);
6284 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6286 /* Remove the GIMPLE_OMP_FOR statement. */
6287 gsi_remove (&gsi
, true);
6289 /* Iteration setup for sequential loop goes in L0_BB. */
6290 tree startvar
= fd
->loop
.v
;
6291 tree endvar
= NULL_TREE
;
6293 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6295 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
6296 && gimple_omp_for_kind (inner_stmt
)
6297 == GF_OMP_FOR_KIND_SIMD
);
6298 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
6299 OMP_CLAUSE__LOOPTEMP_
);
6300 gcc_assert (innerc
);
6301 startvar
= OMP_CLAUSE_DECL (innerc
);
6302 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6303 OMP_CLAUSE__LOOPTEMP_
);
6304 gcc_assert (innerc
);
6305 endvar
= OMP_CLAUSE_DECL (innerc
);
6308 gsi
= gsi_start_bb (l0_bb
);
6311 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6312 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6313 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6314 t
= fold_convert (TREE_TYPE (startvar
), t
);
6315 t
= force_gimple_operand_gsi (&gsi
, t
,
6317 && TREE_ADDRESSABLE (startvar
),
6318 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6319 assign_stmt
= gimple_build_assign (startvar
, t
);
6320 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6324 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6325 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6326 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6327 t
= fold_convert (TREE_TYPE (startvar
), t
);
6328 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6329 false, GSI_CONTINUE_LINKING
);
6332 assign_stmt
= gimple_build_assign (endvar
, iend
);
6333 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6334 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
6335 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
6337 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
6338 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6340 if (fd
->collapse
> 1)
6341 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6345 /* Code to control the increment and predicate for the sequential
6346 loop goes in the CONT_BB. */
6347 gsi
= gsi_last_bb (cont_bb
);
6348 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6349 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6350 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6351 vback
= gimple_omp_continue_control_def (cont_stmt
);
6353 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6355 if (POINTER_TYPE_P (type
))
6356 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
6358 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
6359 t
= force_gimple_operand_gsi (&gsi
, t
,
6361 && TREE_ADDRESSABLE (vback
),
6362 NULL_TREE
, true, GSI_SAME_STMT
);
6363 assign_stmt
= gimple_build_assign (vback
, t
);
6364 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6366 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6367 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
6369 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6370 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6373 /* Remove GIMPLE_OMP_CONTINUE. */
6374 gsi_remove (&gsi
, true);
6376 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6377 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
6379 /* Emit code to get the next parallel iteration in L2_BB. */
6380 gsi
= gsi_start_bb (l2_bb
);
6382 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6383 build_fold_addr_expr (istart0
),
6384 build_fold_addr_expr (iend0
));
6385 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6386 false, GSI_CONTINUE_LINKING
);
6387 if (TREE_TYPE (t
) != boolean_type_node
)
6388 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6389 t
, build_int_cst (TREE_TYPE (t
), 0));
6390 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6391 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
6394 /* Add the loop cleanup function. */
6395 gsi
= gsi_last_bb (exit_bb
);
6396 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6397 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
6398 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6399 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
6401 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
6402 gcall
*call_stmt
= gimple_build_call (t
, 0);
6403 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6404 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
6405 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
6406 gsi_remove (&gsi
, true);
6408 /* Connect the new blocks. */
6409 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
6410 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
6416 e
= find_edge (cont_bb
, l3_bb
);
6417 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
6419 phis
= phi_nodes (l3_bb
);
6420 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6422 gimple phi
= gsi_stmt (gsi
);
6423 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
6424 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
6428 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
6429 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
6430 e
= find_edge (cont_bb
, l1_bb
);
6431 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6436 else if (fd
->collapse
> 1)
6439 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6442 e
->flags
= EDGE_TRUE_VALUE
;
6445 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6446 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
6450 e
= find_edge (cont_bb
, l2_bb
);
6451 e
->flags
= EDGE_FALLTHRU
;
6453 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6455 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
6456 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
6457 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
6458 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
6459 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
6460 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
6461 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
6462 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
6464 struct loop
*outer_loop
= alloc_loop ();
6465 outer_loop
->header
= l0_bb
;
6466 outer_loop
->latch
= l2_bb
;
6467 add_loop (outer_loop
, l0_bb
->loop_father
);
6469 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6471 struct loop
*loop
= alloc_loop ();
6472 loop
->header
= l1_bb
;
6473 /* The loop may have multiple latches. */
6474 add_loop (loop
, outer_loop
);
6480 /* A subroutine of expand_omp_for. Generate code for a parallel
6481 loop with static schedule and no specified chunk size. Given
6484 for (V = N1; V cond N2; V += STEP) BODY;
6486 where COND is "<" or ">", we generate pseudocode
6488 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6493 if ((__typeof (V)) -1 > 0 && cond is >)
6494 n = -(adj + N2 - N1) / -STEP;
6496 n = (adj + N2 - N1) / STEP;
6499 if (threadid < tt) goto L3; else goto L4;
6504 s0 = q * threadid + tt;
6507 if (s0 >= e0) goto L2; else goto L0;
6513 if (V cond e) goto L1;
6518 expand_omp_for_static_nochunk (struct omp_region
*region
,
6519 struct omp_for_data
*fd
,
6522 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6523 tree type
, itype
, vmain
, vback
;
6524 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6525 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6527 gimple_stmt_iterator gsi
;
6529 bool broken_loop
= region
->cont
== NULL
;
6530 tree
*counts
= NULL
;
6533 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6534 != GF_OMP_FOR_KIND_OACC_LOOP
)
6537 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6538 if (POINTER_TYPE_P (type
))
6539 itype
= signed_type_for (type
);
6541 entry_bb
= region
->entry
;
6542 cont_bb
= region
->cont
;
6543 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6544 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6545 gcc_assert (broken_loop
6546 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6547 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6548 body_bb
= single_succ (seq_start_bb
);
6551 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6552 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6554 exit_bb
= region
->exit
;
6556 /* Iteration space partitioning goes in ENTRY_BB. */
6557 gsi
= gsi_last_bb (entry_bb
);
6558 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6560 if (fd
->collapse
> 1)
6562 int first_zero_iter
= -1;
6563 basic_block l2_dom_bb
= NULL
;
6565 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6566 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6567 fin_bb
, first_zero_iter
,
6571 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6572 t
= integer_one_node
;
6574 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6575 fold_convert (type
, fd
->loop
.n1
),
6576 fold_convert (type
, fd
->loop
.n2
));
6577 if (fd
->collapse
== 1
6578 && TYPE_UNSIGNED (type
)
6579 && (t
== NULL_TREE
|| !integer_onep (t
)))
6581 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6582 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6583 true, GSI_SAME_STMT
);
6584 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6585 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6586 true, GSI_SAME_STMT
);
6587 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6588 NULL_TREE
, NULL_TREE
);
6589 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6590 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6591 expand_omp_regimplify_p
, NULL
, NULL
)
6592 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6593 expand_omp_regimplify_p
, NULL
, NULL
))
6595 gsi
= gsi_for_stmt (cond_stmt
);
6596 gimple_regimplify_operands (cond_stmt
, &gsi
);
6598 ep
= split_block (entry_bb
, cond_stmt
);
6599 ep
->flags
= EDGE_TRUE_VALUE
;
6600 entry_bb
= ep
->dest
;
6601 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6602 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6603 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6604 if (gimple_in_ssa_p (cfun
))
6606 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6607 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6608 !gsi_end_p (gpi
); gsi_next (&gpi
))
6610 gphi
*phi
= gpi
.phi ();
6611 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6612 ep
, UNKNOWN_LOCATION
);
6615 gsi
= gsi_last_bb (entry_bb
);
6618 switch (gimple_omp_for_kind (fd
->for_stmt
))
6620 case GF_OMP_FOR_KIND_FOR
:
6621 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6622 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6624 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6625 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6626 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6628 case GF_OMP_FOR_KIND_OACC_LOOP
:
6629 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6630 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
6635 nthreads
= build_call_expr (nthreads
, 0);
6636 nthreads
= fold_convert (itype
, nthreads
);
6637 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
6638 true, GSI_SAME_STMT
);
6639 threadid
= build_call_expr (threadid
, 0);
6640 threadid
= fold_convert (itype
, threadid
);
6641 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
6642 true, GSI_SAME_STMT
);
6646 step
= fd
->loop
.step
;
6647 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6649 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6650 OMP_CLAUSE__LOOPTEMP_
);
6651 gcc_assert (innerc
);
6652 n1
= OMP_CLAUSE_DECL (innerc
);
6653 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6654 OMP_CLAUSE__LOOPTEMP_
);
6655 gcc_assert (innerc
);
6656 n2
= OMP_CLAUSE_DECL (innerc
);
6658 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6659 true, NULL_TREE
, true, GSI_SAME_STMT
);
6660 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6661 true, NULL_TREE
, true, GSI_SAME_STMT
);
6662 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6663 true, NULL_TREE
, true, GSI_SAME_STMT
);
6665 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6666 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6667 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6668 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6669 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6670 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6671 fold_build1 (NEGATE_EXPR
, itype
, t
),
6672 fold_build1 (NEGATE_EXPR
, itype
, step
));
6674 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6675 t
= fold_convert (itype
, t
);
6676 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6678 q
= create_tmp_reg (itype
, "q");
6679 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6680 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6681 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6683 tt
= create_tmp_reg (itype
, "tt");
6684 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6685 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6686 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6688 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6689 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6690 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6692 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
6693 gsi
= gsi_last_bb (second_bb
);
6694 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6696 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6698 gassign
*assign_stmt
6699 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
6700 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6702 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
6703 gsi
= gsi_last_bb (third_bb
);
6704 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6706 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6707 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6708 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6710 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6711 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6713 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6714 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6716 /* Remove the GIMPLE_OMP_FOR statement. */
6717 gsi_remove (&gsi
, true);
6719 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6720 gsi
= gsi_start_bb (seq_start_bb
);
6722 tree startvar
= fd
->loop
.v
;
6723 tree endvar
= NULL_TREE
;
6725 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6727 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6728 ? gimple_omp_parallel_clauses (inner_stmt
)
6729 : gimple_omp_for_clauses (inner_stmt
);
6730 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6731 gcc_assert (innerc
);
6732 startvar
= OMP_CLAUSE_DECL (innerc
);
6733 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6734 OMP_CLAUSE__LOOPTEMP_
);
6735 gcc_assert (innerc
);
6736 endvar
= OMP_CLAUSE_DECL (innerc
);
6738 t
= fold_convert (itype
, s0
);
6739 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6740 if (POINTER_TYPE_P (type
))
6741 t
= fold_build_pointer_plus (n1
, t
);
6743 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6744 t
= fold_convert (TREE_TYPE (startvar
), t
);
6745 t
= force_gimple_operand_gsi (&gsi
, t
,
6747 && TREE_ADDRESSABLE (startvar
),
6748 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6749 assign_stmt
= gimple_build_assign (startvar
, t
);
6750 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6752 t
= fold_convert (itype
, e0
);
6753 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6754 if (POINTER_TYPE_P (type
))
6755 t
= fold_build_pointer_plus (n1
, t
);
6757 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6758 t
= fold_convert (TREE_TYPE (startvar
), t
);
6759 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6760 false, GSI_CONTINUE_LINKING
);
6763 assign_stmt
= gimple_build_assign (endvar
, e
);
6764 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6765 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6766 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6768 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
6769 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6771 if (fd
->collapse
> 1)
6772 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6776 /* The code controlling the sequential loop replaces the
6777 GIMPLE_OMP_CONTINUE. */
6778 gsi
= gsi_last_bb (cont_bb
);
6779 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6780 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6781 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6782 vback
= gimple_omp_continue_control_def (cont_stmt
);
6784 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6786 if (POINTER_TYPE_P (type
))
6787 t
= fold_build_pointer_plus (vmain
, step
);
6789 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6790 t
= force_gimple_operand_gsi (&gsi
, t
,
6792 && TREE_ADDRESSABLE (vback
),
6793 NULL_TREE
, true, GSI_SAME_STMT
);
6794 assign_stmt
= gimple_build_assign (vback
, t
);
6795 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6797 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6798 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6800 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6803 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6804 gsi_remove (&gsi
, true);
6806 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6807 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6810 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6811 gsi
= gsi_last_bb (exit_bb
);
6812 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6814 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6815 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
6816 gcc_checking_assert (t
== NULL_TREE
);
6818 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6820 gsi_remove (&gsi
, true);
6822 /* Connect all the blocks. */
6823 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6824 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6825 ep
= find_edge (entry_bb
, second_bb
);
6826 ep
->flags
= EDGE_TRUE_VALUE
;
6827 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6828 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6829 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6833 ep
= find_edge (cont_bb
, body_bb
);
6834 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6839 else if (fd
->collapse
> 1)
6842 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6845 ep
->flags
= EDGE_TRUE_VALUE
;
6846 find_edge (cont_bb
, fin_bb
)->flags
6847 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6850 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6851 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6852 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6854 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6855 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6856 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6857 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6859 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6861 struct loop
*loop
= alloc_loop ();
6862 loop
->header
= body_bb
;
6863 if (collapse_bb
== NULL
)
6864 loop
->latch
= cont_bb
;
6865 add_loop (loop
, body_bb
->loop_father
);
6870 /* A subroutine of expand_omp_for. Generate code for a parallel
6871 loop with static schedule and a specified chunk size. Given
6874 for (V = N1; V cond N2; V += STEP) BODY;
6876 where COND is "<" or ">", we generate pseudocode
6878 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6883 if ((__typeof (V)) -1 > 0 && cond is >)
6884 n = -(adj + N2 - N1) / -STEP;
6886 n = (adj + N2 - N1) / STEP;
6888 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6889 here so that V is defined
6890 if the loop is not entered
6892 s0 = (trip * nthreads + threadid) * CHUNK;
6893 e0 = min(s0 + CHUNK, n);
6894 if (s0 < n) goto L1; else goto L4;
6901 if (V cond e) goto L2; else goto L3;
6909 expand_omp_for_static_chunk (struct omp_region
*region
,
6910 struct omp_for_data
*fd
, gimple inner_stmt
)
6912 tree n
, s0
, e0
, e
, t
;
6913 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6914 tree type
, itype
, vmain
, vback
, vextra
;
6915 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6916 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6917 gimple_stmt_iterator gsi
;
6919 bool broken_loop
= region
->cont
== NULL
;
6920 tree
*counts
= NULL
;
6923 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6924 != GF_OMP_FOR_KIND_OACC_LOOP
)
6927 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6928 if (POINTER_TYPE_P (type
))
6929 itype
= signed_type_for (type
);
6931 entry_bb
= region
->entry
;
6932 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6934 iter_part_bb
= se
->dest
;
6935 cont_bb
= region
->cont
;
6936 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6937 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6938 gcc_assert (broken_loop
6939 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6940 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6941 body_bb
= single_succ (seq_start_bb
);
6944 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6945 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6946 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6948 exit_bb
= region
->exit
;
6950 /* Trip and adjustment setup goes in ENTRY_BB. */
6951 gsi
= gsi_last_bb (entry_bb
);
6952 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6954 if (fd
->collapse
> 1)
6956 int first_zero_iter
= -1;
6957 basic_block l2_dom_bb
= NULL
;
6959 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6960 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6961 fin_bb
, first_zero_iter
,
6965 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6966 t
= integer_one_node
;
6968 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6969 fold_convert (type
, fd
->loop
.n1
),
6970 fold_convert (type
, fd
->loop
.n2
));
6971 if (fd
->collapse
== 1
6972 && TYPE_UNSIGNED (type
)
6973 && (t
== NULL_TREE
|| !integer_onep (t
)))
6975 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6976 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6977 true, GSI_SAME_STMT
);
6978 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6979 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6980 true, GSI_SAME_STMT
);
6981 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6982 NULL_TREE
, NULL_TREE
);
6983 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6984 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6985 expand_omp_regimplify_p
, NULL
, NULL
)
6986 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6987 expand_omp_regimplify_p
, NULL
, NULL
))
6989 gsi
= gsi_for_stmt (cond_stmt
);
6990 gimple_regimplify_operands (cond_stmt
, &gsi
);
6992 se
= split_block (entry_bb
, cond_stmt
);
6993 se
->flags
= EDGE_TRUE_VALUE
;
6994 entry_bb
= se
->dest
;
6995 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6996 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6997 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6998 if (gimple_in_ssa_p (cfun
))
7000 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
7001 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
7002 !gsi_end_p (gpi
); gsi_next (&gpi
))
7004 gphi
*phi
= gpi
.phi ();
7005 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
7006 se
, UNKNOWN_LOCATION
);
7009 gsi
= gsi_last_bb (entry_bb
);
7012 switch (gimple_omp_for_kind (fd
->for_stmt
))
7014 case GF_OMP_FOR_KIND_FOR
:
7015 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7016 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7018 case GF_OMP_FOR_KIND_DISTRIBUTE
:
7019 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
7020 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
7022 case GF_OMP_FOR_KIND_OACC_LOOP
:
7023 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
7024 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
7029 nthreads
= build_call_expr (nthreads
, 0);
7030 nthreads
= fold_convert (itype
, nthreads
);
7031 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
7032 true, GSI_SAME_STMT
);
7033 threadid
= build_call_expr (threadid
, 0);
7034 threadid
= fold_convert (itype
, threadid
);
7035 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
7036 true, GSI_SAME_STMT
);
7040 step
= fd
->loop
.step
;
7041 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7043 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7044 OMP_CLAUSE__LOOPTEMP_
);
7045 gcc_assert (innerc
);
7046 n1
= OMP_CLAUSE_DECL (innerc
);
7047 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7048 OMP_CLAUSE__LOOPTEMP_
);
7049 gcc_assert (innerc
);
7050 n2
= OMP_CLAUSE_DECL (innerc
);
7052 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
7053 true, NULL_TREE
, true, GSI_SAME_STMT
);
7054 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
7055 true, NULL_TREE
, true, GSI_SAME_STMT
);
7056 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
7057 true, NULL_TREE
, true, GSI_SAME_STMT
);
7059 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
7060 true, NULL_TREE
, true, GSI_SAME_STMT
);
7062 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
7063 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
7064 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
7065 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
7066 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
7067 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7068 fold_build1 (NEGATE_EXPR
, itype
, t
),
7069 fold_build1 (NEGATE_EXPR
, itype
, step
));
7071 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
7072 t
= fold_convert (itype
, t
);
7073 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7074 true, GSI_SAME_STMT
);
7076 trip_var
= create_tmp_reg (itype
, ".trip");
7077 if (gimple_in_ssa_p (cfun
))
7079 trip_init
= make_ssa_name (trip_var
);
7080 trip_main
= make_ssa_name (trip_var
);
7081 trip_back
= make_ssa_name (trip_var
);
7085 trip_init
= trip_var
;
7086 trip_main
= trip_var
;
7087 trip_back
= trip_var
;
7090 gassign
*assign_stmt
7091 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
7092 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7094 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
7095 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7096 if (POINTER_TYPE_P (type
))
7097 t
= fold_build_pointer_plus (n1
, t
);
7099 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7100 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7101 true, GSI_SAME_STMT
);
7103 /* Remove the GIMPLE_OMP_FOR. */
7104 gsi_remove (&gsi
, true);
7106 /* Iteration space partitioning goes in ITER_PART_BB. */
7107 gsi
= gsi_last_bb (iter_part_bb
);
7109 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
7110 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
7111 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
7112 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7113 false, GSI_CONTINUE_LINKING
);
7115 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
7116 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
7117 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7118 false, GSI_CONTINUE_LINKING
);
7120 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
7121 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
7123 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7124 gsi
= gsi_start_bb (seq_start_bb
);
7126 tree startvar
= fd
->loop
.v
;
7127 tree endvar
= NULL_TREE
;
7129 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7131 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
7132 ? gimple_omp_parallel_clauses (inner_stmt
)
7133 : gimple_omp_for_clauses (inner_stmt
);
7134 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7135 gcc_assert (innerc
);
7136 startvar
= OMP_CLAUSE_DECL (innerc
);
7137 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7138 OMP_CLAUSE__LOOPTEMP_
);
7139 gcc_assert (innerc
);
7140 endvar
= OMP_CLAUSE_DECL (innerc
);
7143 t
= fold_convert (itype
, s0
);
7144 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7145 if (POINTER_TYPE_P (type
))
7146 t
= fold_build_pointer_plus (n1
, t
);
7148 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7149 t
= fold_convert (TREE_TYPE (startvar
), t
);
7150 t
= force_gimple_operand_gsi (&gsi
, t
,
7152 && TREE_ADDRESSABLE (startvar
),
7153 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7154 assign_stmt
= gimple_build_assign (startvar
, t
);
7155 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7157 t
= fold_convert (itype
, e0
);
7158 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7159 if (POINTER_TYPE_P (type
))
7160 t
= fold_build_pointer_plus (n1
, t
);
7162 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7163 t
= fold_convert (TREE_TYPE (startvar
), t
);
7164 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7165 false, GSI_CONTINUE_LINKING
);
7168 assign_stmt
= gimple_build_assign (endvar
, e
);
7169 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7170 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
7171 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
7173 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
7174 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7176 if (fd
->collapse
> 1)
7177 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
7181 /* The code controlling the sequential loop goes in CONT_BB,
7182 replacing the GIMPLE_OMP_CONTINUE. */
7183 gsi
= gsi_last_bb (cont_bb
);
7184 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
7185 vmain
= gimple_omp_continue_control_use (cont_stmt
);
7186 vback
= gimple_omp_continue_control_def (cont_stmt
);
7188 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7190 if (POINTER_TYPE_P (type
))
7191 t
= fold_build_pointer_plus (vmain
, step
);
7193 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
7194 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
7195 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7196 true, GSI_SAME_STMT
);
7197 assign_stmt
= gimple_build_assign (vback
, t
);
7198 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7200 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
7201 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
7203 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
7206 /* Remove GIMPLE_OMP_CONTINUE. */
7207 gsi_remove (&gsi
, true);
7209 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
7210 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
7212 /* Trip update code goes into TRIP_UPDATE_BB. */
7213 gsi
= gsi_start_bb (trip_update_bb
);
7215 t
= build_int_cst (itype
, 1);
7216 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
7217 assign_stmt
= gimple_build_assign (trip_back
, t
);
7218 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7221 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7222 gsi
= gsi_last_bb (exit_bb
);
7223 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
7225 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
7226 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
7227 gcc_checking_assert (t
== NULL_TREE
);
7229 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
7231 gsi_remove (&gsi
, true);
7233 /* Connect the new blocks. */
7234 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
7235 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
7239 se
= find_edge (cont_bb
, body_bb
);
7240 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7245 else if (fd
->collapse
> 1)
7248 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
7251 se
->flags
= EDGE_TRUE_VALUE
;
7252 find_edge (cont_bb
, trip_update_bb
)->flags
7253 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
7255 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
7258 if (gimple_in_ssa_p (cfun
))
7266 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
7268 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7269 remove arguments of the phi nodes in fin_bb. We need to create
7270 appropriate phi nodes in iter_part_bb instead. */
7271 se
= single_pred_edge (fin_bb
);
7272 re
= single_succ_edge (trip_update_bb
);
7273 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
7274 ene
= single_succ_edge (entry_bb
);
7276 psi
= gsi_start_phis (fin_bb
);
7277 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
7278 gsi_next (&psi
), ++i
)
7281 source_location locus
;
7284 t
= gimple_phi_result (phi
);
7285 gcc_assert (t
== redirect_edge_var_map_result (vm
));
7286 nphi
= create_phi_node (t
, iter_part_bb
);
7288 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
7289 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
7291 /* A special case -- fd->loop.v is not yet computed in
7292 iter_part_bb, we need to use vextra instead. */
7293 if (t
== fd
->loop
.v
)
7295 add_phi_arg (nphi
, t
, ene
, locus
);
7296 locus
= redirect_edge_var_map_location (vm
);
7297 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
7299 gcc_assert (gsi_end_p (psi
) && i
== head
->length ());
7300 redirect_edge_var_map_clear (re
);
7303 psi
= gsi_start_phis (fin_bb
);
7304 if (gsi_end_p (psi
))
7306 remove_phi_node (&psi
, false);
7309 /* Make phi node for trip. */
7310 phi
= create_phi_node (trip_main
, iter_part_bb
);
7311 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
7313 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
7318 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
7319 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
7320 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
7321 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
7322 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
7323 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
7324 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
7325 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
7326 recompute_dominator (CDI_DOMINATORS
, body_bb
));
7330 struct loop
*trip_loop
= alloc_loop ();
7331 trip_loop
->header
= iter_part_bb
;
7332 trip_loop
->latch
= trip_update_bb
;
7333 add_loop (trip_loop
, iter_part_bb
->loop_father
);
7335 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7337 struct loop
*loop
= alloc_loop ();
7338 loop
->header
= body_bb
;
7339 if (collapse_bb
== NULL
)
7340 loop
->latch
= cont_bb
;
7341 add_loop (loop
, trip_loop
);
7346 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7348 for (V = N1; V cond N2; V += STEP) BODY;
7350 where COND is "<" or ">" or "!=", we generate pseudocode
7352 for (ind_var = low; ind_var < high; ind_var++)
7354 V = n1 + (ind_var * STEP)
7359 In the above pseudocode, low and high are function parameters of the
7360 child function. In the function below, we are inserting a temp.
7361 variable that will be making a call to two OMP functions that will not be
7362 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7363 with _Cilk_for). These functions are replaced with low and high
7364 by the function that handles taskreg. */
7368 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
7370 bool broken_loop
= region
->cont
== NULL
;
7371 basic_block entry_bb
= region
->entry
;
7372 basic_block cont_bb
= region
->cont
;
7374 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7375 gcc_assert (broken_loop
7376 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7377 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7378 basic_block l1_bb
, l2_bb
;
7382 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7383 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7384 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7385 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7389 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7390 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7391 l2_bb
= single_succ (l1_bb
);
7393 basic_block exit_bb
= region
->exit
;
7394 basic_block l2_dom_bb
= NULL
;
7396 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
7398 /* Below statements until the "tree high_val = ..." are pseudo statements
7399 used to pass information to be used by expand_omp_taskreg.
7400 low_val and high_val will be replaced by the __low and __high
7401 parameter from the child function.
7403 The call_exprs part is a place-holder, it is mainly used
7404 to distinctly identify to the top-level part that this is
7405 where we should put low and high (reasoning given in header
7409 = gimple_omp_parallel_child_fn (
7410 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
7411 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
7412 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
7414 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
7416 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
7419 gcc_assert (low_val
&& high_val
);
7421 tree type
= TREE_TYPE (low_val
);
7422 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
7423 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7425 /* Not needed in SSA form right now. */
7426 gcc_assert (!gimple_in_ssa_p (cfun
));
7427 if (l2_dom_bb
== NULL
)
7433 gimple stmt
= gimple_build_assign (ind_var
, n1
);
7435 /* Replace the GIMPLE_OMP_FOR statement. */
7436 gsi_replace (&gsi
, stmt
, true);
7440 /* Code to control the increment goes in the CONT_BB. */
7441 gsi
= gsi_last_bb (cont_bb
);
7442 stmt
= gsi_stmt (gsi
);
7443 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7444 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
7445 build_one_cst (type
));
7447 /* Replace GIMPLE_OMP_CONTINUE. */
7448 gsi_replace (&gsi
, stmt
, true);
7451 /* Emit the condition in L1_BB. */
7452 gsi
= gsi_after_labels (l1_bb
);
7453 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
7454 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
7456 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
7457 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7458 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7460 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7461 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
7462 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
7463 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7465 /* The condition is always '<' since the runtime will fill in the low
7467 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
7468 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7470 /* Remove GIMPLE_OMP_RETURN. */
7471 gsi
= gsi_last_bb (exit_bb
);
7472 gsi_remove (&gsi
, true);
7474 /* Connect the new blocks. */
7475 remove_edge (FALLTHRU_EDGE (entry_bb
));
7480 remove_edge (BRANCH_EDGE (entry_bb
));
7481 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7483 e
= BRANCH_EDGE (l1_bb
);
7484 ne
= FALLTHRU_EDGE (l1_bb
);
7485 e
->flags
= EDGE_TRUE_VALUE
;
7489 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7491 ne
= single_succ_edge (l1_bb
);
7492 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7495 ne
->flags
= EDGE_FALSE_VALUE
;
7496 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7497 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7499 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7500 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7501 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7505 struct loop
*loop
= alloc_loop ();
7506 loop
->header
= l1_bb
;
7507 loop
->latch
= cont_bb
;
7508 add_loop (loop
, l1_bb
->loop_father
);
7509 loop
->safelen
= INT_MAX
;
7512 /* Pick the correct library function based on the precision of the
7513 induction variable type. */
7514 tree lib_fun
= NULL_TREE
;
7515 if (TYPE_PRECISION (type
) == 32)
7516 lib_fun
= cilk_for_32_fndecl
;
7517 else if (TYPE_PRECISION (type
) == 64)
7518 lib_fun
= cilk_for_64_fndecl
;
7522 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7524 /* WS_ARGS contains the library function flavor to call:
7525 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7526 user-defined grain value. If the user does not define one, then zero
7527 is passed in by the parser. */
7528 vec_alloc (region
->ws_args
, 2);
7529 region
->ws_args
->quick_push (lib_fun
);
7530 region
->ws_args
->quick_push (fd
->chunk_size
);
7533 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7534 loop. Given parameters:
7536 for (V = N1; V cond N2; V += STEP) BODY;
7538 where COND is "<" or ">", we generate pseudocode
7546 if (V cond N2) goto L0; else goto L2;
7549 For collapsed loops, given parameters:
7551 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7552 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7553 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7556 we generate pseudocode
7562 count3 = (adj + N32 - N31) / STEP3;
7567 count2 = (adj + N22 - N21) / STEP2;
7572 count1 = (adj + N12 - N11) / STEP1;
7573 count = count1 * count2 * count3;
7583 V2 += (V3 cond3 N32) ? 0 : STEP2;
7584 V3 = (V3 cond3 N32) ? V3 : N31;
7585 V1 += (V2 cond2 N22) ? 0 : STEP1;
7586 V2 = (V2 cond2 N22) ? V2 : N21;
7588 if (V < count) goto L0; else goto L2;
7594 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7597 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7598 gimple_stmt_iterator gsi
;
7601 bool broken_loop
= region
->cont
== NULL
;
7603 tree
*counts
= NULL
;
7605 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7606 OMP_CLAUSE_SAFELEN
);
7607 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7608 OMP_CLAUSE__SIMDUID_
);
7611 type
= TREE_TYPE (fd
->loop
.v
);
7612 entry_bb
= region
->entry
;
7613 cont_bb
= region
->cont
;
7614 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7615 gcc_assert (broken_loop
7616 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7617 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7620 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7621 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7622 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7623 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7627 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7628 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7629 l2_bb
= single_succ (l1_bb
);
7631 exit_bb
= region
->exit
;
7634 gsi
= gsi_last_bb (entry_bb
);
7636 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7637 /* Not needed in SSA form right now. */
7638 gcc_assert (!gimple_in_ssa_p (cfun
));
7639 if (fd
->collapse
> 1)
7641 int first_zero_iter
= -1;
7642 basic_block zero_iter_bb
= l2_bb
;
7644 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7645 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7646 zero_iter_bb
, first_zero_iter
,
7649 if (l2_dom_bb
== NULL
)
7654 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7656 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7657 OMP_CLAUSE__LOOPTEMP_
);
7658 gcc_assert (innerc
);
7659 n1
= OMP_CLAUSE_DECL (innerc
);
7660 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7661 OMP_CLAUSE__LOOPTEMP_
);
7662 gcc_assert (innerc
);
7663 n2
= OMP_CLAUSE_DECL (innerc
);
7664 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7665 fold_convert (type
, n1
));
7666 if (fd
->collapse
> 1)
7669 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7675 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7676 fold_convert (type
, fd
->loop
.n1
));
7677 if (fd
->collapse
> 1)
7678 for (i
= 0; i
< fd
->collapse
; i
++)
7680 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7681 if (POINTER_TYPE_P (itype
))
7682 itype
= signed_type_for (itype
);
7683 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7684 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7688 /* Remove the GIMPLE_OMP_FOR statement. */
7689 gsi_remove (&gsi
, true);
7693 /* Code to control the increment goes in the CONT_BB. */
7694 gsi
= gsi_last_bb (cont_bb
);
7695 stmt
= gsi_stmt (gsi
);
7696 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7698 if (POINTER_TYPE_P (type
))
7699 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7701 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7702 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7704 if (fd
->collapse
> 1)
7706 i
= fd
->collapse
- 1;
7707 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7709 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7710 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7714 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7716 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7719 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7721 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7723 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7724 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7725 if (POINTER_TYPE_P (itype2
))
7726 itype2
= signed_type_for (itype2
);
7727 t
= build3 (COND_EXPR
, itype2
,
7728 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7730 fold_convert (itype
, fd
->loops
[i
].n2
)),
7731 build_int_cst (itype2
, 0),
7732 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7733 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7734 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7736 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7737 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7739 t
= build3 (COND_EXPR
, itype
,
7740 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7742 fold_convert (itype
, fd
->loops
[i
].n2
)),
7744 fold_convert (itype
, fd
->loops
[i
].n1
));
7745 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7749 /* Remove GIMPLE_OMP_CONTINUE. */
7750 gsi_remove (&gsi
, true);
7753 /* Emit the condition in L1_BB. */
7754 gsi
= gsi_start_bb (l1_bb
);
7756 t
= fold_convert (type
, n2
);
7757 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7758 false, GSI_CONTINUE_LINKING
);
7759 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7760 cond_stmt
= gimple_build_cond_empty (t
);
7761 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
7762 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7764 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7767 gsi
= gsi_for_stmt (cond_stmt
);
7768 gimple_regimplify_operands (cond_stmt
, &gsi
);
7771 /* Remove GIMPLE_OMP_RETURN. */
7772 gsi
= gsi_last_bb (exit_bb
);
7773 gsi_remove (&gsi
, true);
7775 /* Connect the new blocks. */
7776 remove_edge (FALLTHRU_EDGE (entry_bb
));
7780 remove_edge (BRANCH_EDGE (entry_bb
));
7781 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7783 e
= BRANCH_EDGE (l1_bb
);
7784 ne
= FALLTHRU_EDGE (l1_bb
);
7785 e
->flags
= EDGE_TRUE_VALUE
;
7789 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7791 ne
= single_succ_edge (l1_bb
);
7792 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7795 ne
->flags
= EDGE_FALSE_VALUE
;
7796 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7797 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7799 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7800 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7801 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7805 struct loop
*loop
= alloc_loop ();
7806 loop
->header
= l1_bb
;
7807 loop
->latch
= cont_bb
;
7808 add_loop (loop
, l1_bb
->loop_father
);
7809 if (safelen
== NULL_TREE
)
7810 loop
->safelen
= INT_MAX
;
7813 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7814 if (TREE_CODE (safelen
) != INTEGER_CST
)
7816 else if (!tree_fits_uhwi_p (safelen
)
7817 || tree_to_uhwi (safelen
) > INT_MAX
)
7818 loop
->safelen
= INT_MAX
;
7820 loop
->safelen
= tree_to_uhwi (safelen
);
7821 if (loop
->safelen
== 1)
7826 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7827 cfun
->has_simduid_loops
= true;
7829 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7831 if ((flag_tree_loop_vectorize
7832 || (!global_options_set
.x_flag_tree_loop_vectorize
7833 && !global_options_set
.x_flag_tree_vectorize
))
7834 && flag_tree_loop_optimize
7835 && loop
->safelen
> 1)
7837 loop
->force_vectorize
= true;
7838 cfun
->has_force_vectorize_loops
= true;
7844 /* Expand the OMP loop defined by REGION. */
7847 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7849 struct omp_for_data fd
;
7850 struct omp_for_data_loop
*loops
;
7853 = (struct omp_for_data_loop
*)
7854 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7855 * sizeof (struct omp_for_data_loop
));
7856 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
7858 region
->sched_kind
= fd
.sched_kind
;
7860 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7861 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7862 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7865 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7866 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7867 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7870 /* If there isn't a continue then this is a degerate case where
7871 the introduction of abnormal edges during lowering will prevent
7872 original loops from being detected. Fix that up. */
7873 loops_state_set (LOOPS_NEED_FIXUP
);
7875 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7876 expand_omp_simd (region
, &fd
);
7877 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7878 expand_cilk_for (region
, &fd
);
7879 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7880 && !fd
.have_ordered
)
7882 if (fd
.chunk_size
== NULL
)
7883 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7885 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7889 int fn_index
, start_ix
, next_ix
;
7891 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7892 == GF_OMP_FOR_KIND_FOR
);
7893 if (fd
.chunk_size
== NULL
7894 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7895 fd
.chunk_size
= integer_zero_node
;
7896 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7897 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7898 ? 3 : fd
.sched_kind
;
7899 fn_index
+= fd
.have_ordered
* 4;
7900 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7901 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7902 if (fd
.iter_type
== long_long_unsigned_type_node
)
7904 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7905 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7906 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7907 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7909 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7910 (enum built_in_function
) next_ix
, inner_stmt
);
7913 if (gimple_in_ssa_p (cfun
))
7914 update_ssa (TODO_update_ssa_only_virtuals
);
7918 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7920 v = GOMP_sections_start (n);
7937 v = GOMP_sections_next ();
7942 If this is a combined parallel sections, replace the call to
7943 GOMP_sections_start with call to GOMP_sections_next. */
7946 expand_omp_sections (struct omp_region
*region
)
7948 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7950 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7951 gimple_stmt_iterator si
, switch_si
;
7952 gomp_sections
*sections_stmt
;
7954 gomp_continue
*cont
;
7957 struct omp_region
*inner
;
7959 bool exit_reachable
= region
->cont
!= NULL
;
7961 gcc_assert (region
->exit
!= NULL
);
7962 entry_bb
= region
->entry
;
7963 l0_bb
= single_succ (entry_bb
);
7964 l1_bb
= region
->cont
;
7965 l2_bb
= region
->exit
;
7966 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7967 l2
= gimple_block_label (l2_bb
);
7970 /* This can happen if there are reductions. */
7971 len
= EDGE_COUNT (l0_bb
->succs
);
7972 gcc_assert (len
> 0);
7973 e
= EDGE_SUCC (l0_bb
, len
- 1);
7974 si
= gsi_last_bb (e
->dest
);
7977 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7978 l2
= gimple_block_label (e
->dest
);
7980 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7982 si
= gsi_last_bb (e
->dest
);
7984 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7986 l2
= gimple_block_label (e
->dest
);
7992 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7994 default_bb
= create_empty_bb (l0_bb
);
7996 /* We will build a switch() with enough cases for all the
7997 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7998 and a default case to abort if something goes wrong. */
7999 len
= EDGE_COUNT (l0_bb
->succs
);
8001 /* Use vec::quick_push on label_vec throughout, since we know the size
8003 auto_vec
<tree
> label_vec (len
);
8005 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
8006 GIMPLE_OMP_SECTIONS statement. */
8007 si
= gsi_last_bb (entry_bb
);
8008 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
8009 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
8010 vin
= gimple_omp_sections_control (sections_stmt
);
8011 if (!is_combined_parallel (region
))
8013 /* If we are not inside a combined parallel+sections region,
8014 call GOMP_sections_start. */
8015 t
= build_int_cst (unsigned_type_node
, len
- 1);
8016 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
8017 stmt
= gimple_build_call (u
, 1, t
);
8021 /* Otherwise, call GOMP_sections_next. */
8022 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8023 stmt
= gimple_build_call (u
, 0);
8025 gimple_call_set_lhs (stmt
, vin
);
8026 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8027 gsi_remove (&si
, true);
8029 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8031 switch_si
= gsi_last_bb (l0_bb
);
8032 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
8035 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
8036 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
8037 vmain
= gimple_omp_continue_control_use (cont
);
8038 vnext
= gimple_omp_continue_control_def (cont
);
8046 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
8047 label_vec
.quick_push (t
);
8050 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8051 for (inner
= region
->inner
, casei
= 1;
8053 inner
= inner
->next
, i
++, casei
++)
8055 basic_block s_entry_bb
, s_exit_bb
;
8057 /* Skip optional reduction region. */
8058 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
8065 s_entry_bb
= inner
->entry
;
8066 s_exit_bb
= inner
->exit
;
8068 t
= gimple_block_label (s_entry_bb
);
8069 u
= build_int_cst (unsigned_type_node
, casei
);
8070 u
= build_case_label (u
, NULL
, t
);
8071 label_vec
.quick_push (u
);
8073 si
= gsi_last_bb (s_entry_bb
);
8074 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
8075 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
8076 gsi_remove (&si
, true);
8077 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
8079 if (s_exit_bb
== NULL
)
8082 si
= gsi_last_bb (s_exit_bb
);
8083 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8084 gsi_remove (&si
, true);
8086 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
8089 /* Error handling code goes in DEFAULT_BB. */
8090 t
= gimple_block_label (default_bb
);
8091 u
= build_case_label (NULL
, NULL
, t
);
8092 make_edge (l0_bb
, default_bb
, 0);
8093 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
8095 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
8096 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
8097 gsi_remove (&switch_si
, true);
8099 si
= gsi_start_bb (default_bb
);
8100 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
8101 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
8107 /* Code to get the next section goes in L1_BB. */
8108 si
= gsi_last_bb (l1_bb
);
8109 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
8111 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8112 stmt
= gimple_build_call (bfn_decl
, 0);
8113 gimple_call_set_lhs (stmt
, vnext
);
8114 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8115 gsi_remove (&si
, true);
8117 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
8120 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8121 si
= gsi_last_bb (l2_bb
);
8122 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
8123 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
8124 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
8125 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
8127 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
8128 stmt
= gimple_build_call (t
, 0);
8129 if (gimple_omp_return_lhs (gsi_stmt (si
)))
8130 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
8131 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8132 gsi_remove (&si
, true);
8134 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
8138 /* Expand code for an OpenMP single directive. We've already expanded
8139 much of the code, here we simply place the GOMP_barrier call. */
8142 expand_omp_single (struct omp_region
*region
)
8144 basic_block entry_bb
, exit_bb
;
8145 gimple_stmt_iterator si
;
8147 entry_bb
= region
->entry
;
8148 exit_bb
= region
->exit
;
8150 si
= gsi_last_bb (entry_bb
);
8151 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
8152 gsi_remove (&si
, true);
8153 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8155 si
= gsi_last_bb (exit_bb
);
8156 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
8158 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
8159 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
8161 gsi_remove (&si
, true);
8162 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8166 /* Generic expansion for OpenMP synchronization directives: master,
8167 ordered and critical. All we need to do here is remove the entry
8168 and exit markers for REGION. */
8171 expand_omp_synch (struct omp_region
*region
)
8173 basic_block entry_bb
, exit_bb
;
8174 gimple_stmt_iterator si
;
8176 entry_bb
= region
->entry
;
8177 exit_bb
= region
->exit
;
8179 si
= gsi_last_bb (entry_bb
);
8180 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
8181 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
8182 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
8183 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
8184 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
8185 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
8186 gsi_remove (&si
, true);
8187 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8191 si
= gsi_last_bb (exit_bb
);
8192 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8193 gsi_remove (&si
, true);
8194 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8198 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8199 operation as a normal volatile load. */
8202 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
8203 tree loaded_val
, int index
)
8205 enum built_in_function tmpbase
;
8206 gimple_stmt_iterator gsi
;
8207 basic_block store_bb
;
8210 tree decl
, call
, type
, itype
;
8212 gsi
= gsi_last_bb (load_bb
);
8213 stmt
= gsi_stmt (gsi
);
8214 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8215 loc
= gimple_location (stmt
);
8217 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8218 is smaller than word size, then expand_atomic_load assumes that the load
8219 is atomic. We could avoid the builtin entirely in this case. */
8221 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8222 decl
= builtin_decl_explicit (tmpbase
);
8223 if (decl
== NULL_TREE
)
8226 type
= TREE_TYPE (loaded_val
);
8227 itype
= TREE_TYPE (TREE_TYPE (decl
));
8229 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
8230 build_int_cst (NULL
,
8231 gimple_omp_atomic_seq_cst_p (stmt
)
8233 : MEMMODEL_RELAXED
));
8234 if (!useless_type_conversion_p (type
, itype
))
8235 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8236 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8238 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8239 gsi_remove (&gsi
, true);
8241 store_bb
= single_succ (load_bb
);
8242 gsi
= gsi_last_bb (store_bb
);
8243 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8244 gsi_remove (&gsi
, true);
8246 if (gimple_in_ssa_p (cfun
))
8247 update_ssa (TODO_update_ssa_no_phi
);
8252 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8253 operation as a normal volatile store. */
8256 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
8257 tree loaded_val
, tree stored_val
, int index
)
8259 enum built_in_function tmpbase
;
8260 gimple_stmt_iterator gsi
;
8261 basic_block store_bb
= single_succ (load_bb
);
8264 tree decl
, call
, type
, itype
;
8268 gsi
= gsi_last_bb (load_bb
);
8269 stmt
= gsi_stmt (gsi
);
8270 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8272 /* If the load value is needed, then this isn't a store but an exchange. */
8273 exchange
= gimple_omp_atomic_need_value_p (stmt
);
8275 gsi
= gsi_last_bb (store_bb
);
8276 stmt
= gsi_stmt (gsi
);
8277 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
8278 loc
= gimple_location (stmt
);
8280 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8281 is smaller than word size, then expand_atomic_store assumes that the store
8282 is atomic. We could avoid the builtin entirely in this case. */
8284 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
8285 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
8286 decl
= builtin_decl_explicit (tmpbase
);
8287 if (decl
== NULL_TREE
)
8290 type
= TREE_TYPE (stored_val
);
8292 /* Dig out the type of the function's second argument. */
8293 itype
= TREE_TYPE (decl
);
8294 itype
= TYPE_ARG_TYPES (itype
);
8295 itype
= TREE_CHAIN (itype
);
8296 itype
= TREE_VALUE (itype
);
8297 imode
= TYPE_MODE (itype
);
8299 if (exchange
&& !can_atomic_exchange_p (imode
, true))
8302 if (!useless_type_conversion_p (itype
, type
))
8303 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
8304 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
8305 build_int_cst (NULL
,
8306 gimple_omp_atomic_seq_cst_p (stmt
)
8308 : MEMMODEL_RELAXED
));
8311 if (!useless_type_conversion_p (type
, itype
))
8312 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8313 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8316 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8317 gsi_remove (&gsi
, true);
8319 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8320 gsi
= gsi_last_bb (load_bb
);
8321 gsi_remove (&gsi
, true);
8323 if (gimple_in_ssa_p (cfun
))
8324 update_ssa (TODO_update_ssa_no_phi
);
8329 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8330 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8331 size of the data type, and thus usable to find the index of the builtin
8332 decl. Returns false if the expression is not of the proper form. */
8335 expand_omp_atomic_fetch_op (basic_block load_bb
,
8336 tree addr
, tree loaded_val
,
8337 tree stored_val
, int index
)
8339 enum built_in_function oldbase
, newbase
, tmpbase
;
8340 tree decl
, itype
, call
;
8342 basic_block store_bb
= single_succ (load_bb
);
8343 gimple_stmt_iterator gsi
;
8346 enum tree_code code
;
8347 bool need_old
, need_new
;
8351 /* We expect to find the following sequences:
8354 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8357 val = tmp OP something; (or: something OP tmp)
8358 GIMPLE_OMP_STORE (val)
8360 ???FIXME: Allow a more flexible sequence.
8361 Perhaps use data flow to pick the statements.
8365 gsi
= gsi_after_labels (store_bb
);
8366 stmt
= gsi_stmt (gsi
);
8367 loc
= gimple_location (stmt
);
8368 if (!is_gimple_assign (stmt
))
8371 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
8373 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
8374 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
8375 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
8376 gcc_checking_assert (!need_old
|| !need_new
);
8378 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
8381 /* Check for one of the supported fetch-op operations. */
8382 code
= gimple_assign_rhs_code (stmt
);
8386 case POINTER_PLUS_EXPR
:
8387 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
8388 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
8391 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
8392 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
8395 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
8396 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
8399 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
8400 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
8403 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
8404 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
8410 /* Make sure the expression is of the proper form. */
8411 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
8412 rhs
= gimple_assign_rhs2 (stmt
);
8413 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
8414 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
8415 rhs
= gimple_assign_rhs1 (stmt
);
8419 tmpbase
= ((enum built_in_function
)
8420 ((need_new
? newbase
: oldbase
) + index
+ 1));
8421 decl
= builtin_decl_explicit (tmpbase
);
8422 if (decl
== NULL_TREE
)
8424 itype
= TREE_TYPE (TREE_TYPE (decl
));
8425 imode
= TYPE_MODE (itype
);
8427 /* We could test all of the various optabs involved, but the fact of the
8428 matter is that (with the exception of i486 vs i586 and xadd) all targets
8429 that support any atomic operaton optab also implements compare-and-swap.
8430 Let optabs.c take care of expanding any compare-and-swap loop. */
8431 if (!can_compare_and_swap_p (imode
, true))
8434 gsi
= gsi_last_bb (load_bb
);
8435 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8437 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8438 It only requires that the operation happen atomically. Thus we can
8439 use the RELAXED memory model. */
8440 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
8441 fold_convert_loc (loc
, itype
, rhs
),
8442 build_int_cst (NULL
,
8443 seq_cst
? MEMMODEL_SEQ_CST
8444 : MEMMODEL_RELAXED
));
8446 if (need_old
|| need_new
)
8448 lhs
= need_old
? loaded_val
: stored_val
;
8449 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
8450 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
8453 call
= fold_convert_loc (loc
, void_type_node
, call
);
8454 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8455 gsi_remove (&gsi
, true);
8457 gsi
= gsi_last_bb (store_bb
);
8458 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8459 gsi_remove (&gsi
, true);
8460 gsi
= gsi_last_bb (store_bb
);
8461 gsi_remove (&gsi
, true);
8463 if (gimple_in_ssa_p (cfun
))
8464 update_ssa (TODO_update_ssa_no_phi
);
8469 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8473 newval = rhs; // with oldval replacing *addr in rhs
8474 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8475 if (oldval != newval)
8478 INDEX is log2 of the size of the data type, and thus usable to find the
8479 index of the builtin decl. */
8482 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
8483 tree addr
, tree loaded_val
, tree stored_val
,
8486 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
8487 tree type
, itype
, cmpxchg
, iaddr
;
8488 gimple_stmt_iterator si
;
8489 basic_block loop_header
= single_succ (load_bb
);
8492 enum built_in_function fncode
;
8494 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8495 order to use the RELAXED memory model effectively. */
8496 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8498 cmpxchg
= builtin_decl_explicit (fncode
);
8499 if (cmpxchg
== NULL_TREE
)
8501 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8502 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
8504 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
8507 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8508 si
= gsi_last_bb (load_bb
);
8509 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8511 /* For floating-point values, we'll need to view-convert them to integers
8512 so that we can perform the atomic compare and swap. Simplify the
8513 following code by always setting up the "i"ntegral variables. */
8514 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
8518 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
8521 = force_gimple_operand_gsi (&si
,
8522 fold_convert (TREE_TYPE (iaddr
), addr
),
8523 false, NULL_TREE
, true, GSI_SAME_STMT
);
8524 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
8525 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8526 loadedi
= create_tmp_var (itype
);
8527 if (gimple_in_ssa_p (cfun
))
8528 loadedi
= make_ssa_name (loadedi
);
8533 loadedi
= loaded_val
;
8536 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8537 tree loaddecl
= builtin_decl_explicit (fncode
);
8540 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8541 build_call_expr (loaddecl
, 2, iaddr
,
8542 build_int_cst (NULL_TREE
,
8543 MEMMODEL_RELAXED
)));
8545 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8546 build_int_cst (TREE_TYPE (iaddr
), 0));
8549 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8552 /* Move the value to the LOADEDI temporary. */
8553 if (gimple_in_ssa_p (cfun
))
8555 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8556 phi
= create_phi_node (loadedi
, loop_header
);
8557 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8561 gsi_insert_before (&si
,
8562 gimple_build_assign (loadedi
, initial
),
8564 if (loadedi
!= loaded_val
)
8566 gimple_stmt_iterator gsi2
;
8569 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8570 gsi2
= gsi_start_bb (loop_header
);
8571 if (gimple_in_ssa_p (cfun
))
8574 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8575 true, GSI_SAME_STMT
);
8576 stmt
= gimple_build_assign (loaded_val
, x
);
8577 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8581 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8582 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8583 true, GSI_SAME_STMT
);
8586 gsi_remove (&si
, true);
8588 si
= gsi_last_bb (store_bb
);
8589 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8592 storedi
= stored_val
;
8595 force_gimple_operand_gsi (&si
,
8596 build1 (VIEW_CONVERT_EXPR
, itype
,
8597 stored_val
), true, NULL_TREE
, true,
8600 /* Build the compare&swap statement. */
8601 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8602 new_storedi
= force_gimple_operand_gsi (&si
,
8603 fold_convert (TREE_TYPE (loadedi
),
8606 true, GSI_SAME_STMT
);
8608 if (gimple_in_ssa_p (cfun
))
8612 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
8613 stmt
= gimple_build_assign (old_vali
, loadedi
);
8614 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8616 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8617 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8620 /* Note that we always perform the comparison as an integer, even for
8621 floating point. This allows the atomic operation to properly
8622 succeed even with NaNs and -0.0. */
8623 stmt
= gimple_build_cond_empty
8624 (build2 (NE_EXPR
, boolean_type_node
,
8625 new_storedi
, old_vali
));
8626 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8629 e
= single_succ_edge (store_bb
);
8630 e
->flags
&= ~EDGE_FALLTHRU
;
8631 e
->flags
|= EDGE_FALSE_VALUE
;
8633 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8635 /* Copy the new value to loadedi (we already did that before the condition
8636 if we are not in SSA). */
8637 if (gimple_in_ssa_p (cfun
))
8639 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8640 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8643 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8644 gsi_remove (&si
, true);
8646 struct loop
*loop
= alloc_loop ();
8647 loop
->header
= loop_header
;
8648 loop
->latch
= store_bb
;
8649 add_loop (loop
, loop_header
->loop_father
);
8651 if (gimple_in_ssa_p (cfun
))
8652 update_ssa (TODO_update_ssa_no_phi
);
8657 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8659 GOMP_atomic_start ();
8663 The result is not globally atomic, but works so long as all parallel
8664 references are within #pragma omp atomic directives. According to
8665 responses received from omp@openmp.org, appears to be within spec.
8666 Which makes sense, since that's how several other compilers handle
8667 this situation as well.
8668 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8669 expanding. STORED_VAL is the operand of the matching
8670 GIMPLE_OMP_ATOMIC_STORE.
8673 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8677 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8682 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8683 tree addr
, tree loaded_val
, tree stored_val
)
8685 gimple_stmt_iterator si
;
8689 si
= gsi_last_bb (load_bb
);
8690 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8692 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8693 t
= build_call_expr (t
, 0);
8694 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8696 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8697 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8698 gsi_remove (&si
, true);
8700 si
= gsi_last_bb (store_bb
);
8701 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8703 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8705 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8707 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8708 t
= build_call_expr (t
, 0);
8709 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8710 gsi_remove (&si
, true);
8712 if (gimple_in_ssa_p (cfun
))
8713 update_ssa (TODO_update_ssa_no_phi
);
8717 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8718 using expand_omp_atomic_fetch_op. If it failed, we try to
8719 call expand_omp_atomic_pipeline, and if it fails too, the
8720 ultimate fallback is wrapping the operation in a mutex
8721 (expand_omp_atomic_mutex). REGION is the atomic region built
8722 by build_omp_regions_1(). */
8725 expand_omp_atomic (struct omp_region
*region
)
8727 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8728 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
8729 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
8730 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8731 tree addr
= gimple_omp_atomic_load_rhs (load
);
8732 tree stored_val
= gimple_omp_atomic_store_val (store
);
8733 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8734 HOST_WIDE_INT index
;
8736 /* Make sure the type is one of the supported sizes. */
8737 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8738 index
= exact_log2 (index
);
8739 if (index
>= 0 && index
<= 4)
8741 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8743 /* __sync builtins require strict data alignment. */
8744 if (exact_log2 (align
) >= index
)
8747 if (loaded_val
== stored_val
8748 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8749 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8750 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8751 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8755 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8756 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8757 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8758 && store_bb
== single_succ (load_bb
)
8759 && first_stmt (store_bb
) == store
8760 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8764 /* When possible, use specialized atomic update functions. */
8765 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8766 && store_bb
== single_succ (load_bb
)
8767 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8768 loaded_val
, stored_val
, index
))
8771 /* If we don't have specialized __sync builtins, try and implement
8772 as a compare and swap loop. */
8773 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8774 loaded_val
, stored_val
, index
))
8779 /* The ultimate fallback is wrapping the operation in a mutex. */
8780 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8784 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8787 expand_omp_target (struct omp_region
*region
)
8789 basic_block entry_bb
, exit_bb
, new_bb
;
8790 struct function
*child_cfun
;
8791 tree child_fn
, block
, t
;
8792 gimple_stmt_iterator gsi
;
8793 gomp_target
*entry_stmt
;
8796 bool offloaded
, data_region
;
8798 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
8799 new_bb
= region
->entry
;
8801 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
8802 switch (gimple_omp_target_kind (entry_stmt
))
8804 case GF_OMP_TARGET_KIND_REGION
:
8805 case GF_OMP_TARGET_KIND_UPDATE
:
8806 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8807 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8808 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8809 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8810 data_region
= false;
8812 case GF_OMP_TARGET_KIND_DATA
:
8813 case GF_OMP_TARGET_KIND_OACC_DATA
:
8820 child_fn
= NULL_TREE
;
8824 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8825 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8828 /* Supported by expand_omp_taskreg, but not here. */
8829 if (child_cfun
!= NULL
)
8830 gcc_checking_assert (!child_cfun
->cfg
);
8831 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
8833 entry_bb
= region
->entry
;
8834 exit_bb
= region
->exit
;
8838 unsigned srcidx
, dstidx
, num
;
8840 /* If the offloading region needs data sent from the parent
8841 function, then the very first statement (except possible
8842 tree profile counter updates) of the offloading body
8843 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8844 &.OMP_DATA_O is passed as an argument to the child function,
8845 we need to replace it with the argument as seen by the child
8848 In most cases, this will end up being the identity assignment
8849 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8850 a function call that has been inlined, the original PARM_DECL
8851 .OMP_DATA_I may have been converted into a different local
8852 variable. In which case, we need to keep the assignment. */
8853 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
8856 basic_block entry_succ_bb
= single_succ (entry_bb
);
8857 gimple_stmt_iterator gsi
;
8859 gimple tgtcopy_stmt
= NULL
;
8860 tree sender
= TREE_VEC_ELT (data_arg
, 0);
8862 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8864 gcc_assert (!gsi_end_p (gsi
));
8865 stmt
= gsi_stmt (gsi
);
8866 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8869 if (gimple_num_ops (stmt
) == 2)
8871 tree arg
= gimple_assign_rhs1 (stmt
);
8873 /* We're ignoring the subcode because we're
8874 effectively doing a STRIP_NOPS. */
8876 if (TREE_CODE (arg
) == ADDR_EXPR
8877 && TREE_OPERAND (arg
, 0) == sender
)
8879 tgtcopy_stmt
= stmt
;
8885 gcc_assert (tgtcopy_stmt
!= NULL
);
8886 arg
= DECL_ARGUMENTS (child_fn
);
8888 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8889 gsi_remove (&gsi
, true);
8892 /* Declare local variables needed in CHILD_CFUN. */
8893 block
= DECL_INITIAL (child_fn
);
8894 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8895 /* The gimplifier could record temporaries in the offloading block
8896 rather than in containing function's local_decls chain,
8897 which would mean cgraph missed finalizing them. Do it now. */
8898 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8899 if (TREE_CODE (t
) == VAR_DECL
8901 && !DECL_EXTERNAL (t
))
8902 varpool_node::finalize_decl (t
);
8903 DECL_SAVED_TREE (child_fn
) = NULL
;
8904 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8905 gimple_set_body (child_fn
, NULL
);
8906 TREE_USED (block
) = 1;
8908 /* Reset DECL_CONTEXT on function arguments. */
8909 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8910 DECL_CONTEXT (t
) = child_fn
;
8912 /* Split ENTRY_BB at GIMPLE_*,
8913 so that it can be moved to the child function. */
8914 gsi
= gsi_last_bb (entry_bb
);
8915 stmt
= gsi_stmt (gsi
);
8917 && gimple_code (stmt
) == gimple_code (entry_stmt
));
8918 e
= split_block (entry_bb
, stmt
);
8919 gsi_remove (&gsi
, true);
8921 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8923 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8926 gsi
= gsi_last_bb (exit_bb
);
8927 gcc_assert (!gsi_end_p (gsi
)
8928 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8929 stmt
= gimple_build_return (NULL
);
8930 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8931 gsi_remove (&gsi
, true);
8934 /* Move the offloading region into CHILD_CFUN. */
8936 block
= gimple_block (entry_stmt
);
8938 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8940 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8941 /* When the OMP expansion process cannot guarantee an up-to-date
8942 loop tree arrange for the child function to fixup loops. */
8943 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8944 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8946 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8947 num
= vec_safe_length (child_cfun
->local_decls
);
8948 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8950 t
= (*child_cfun
->local_decls
)[srcidx
];
8951 if (DECL_CONTEXT (t
) == cfun
->decl
)
8953 if (srcidx
!= dstidx
)
8954 (*child_cfun
->local_decls
)[dstidx
] = t
;
8958 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8960 /* Inform the callgraph about the new function. */
8961 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8962 cgraph_node::add_new_function (child_fn
, true);
8964 #ifdef ENABLE_OFFLOADING
8965 /* Add the new function to the offload table. */
8966 vec_safe_push (offload_funcs
, child_fn
);
8969 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8970 fixed in a following pass. */
8971 push_cfun (child_cfun
);
8972 cgraph_edge::rebuild_edges ();
8974 #ifdef ENABLE_OFFLOADING
8975 /* Prevent IPA from removing child_fn as unreachable, since there are no
8976 refs from the parent function to child_fn in offload LTO mode. */
8977 struct cgraph_node
*node
= cgraph_node::get (child_fn
);
8978 node
->mark_force_output ();
8981 /* Some EH regions might become dead, see PR34608. If
8982 pass_cleanup_cfg isn't the first pass to happen with the
8983 new child, these dead EH edges might cause problems.
8984 Clean them up now. */
8985 if (flag_exceptions
)
8988 bool changed
= false;
8990 FOR_EACH_BB_FN (bb
, cfun
)
8991 changed
|= gimple_purge_dead_eh_edges (bb
);
8993 cleanup_tree_cfg ();
8998 /* Emit a library call to launch the offloading region, or do data
9000 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
9001 enum built_in_function start_ix
;
9002 location_t clause_loc
;
9004 switch (gimple_omp_target_kind (entry_stmt
))
9006 case GF_OMP_TARGET_KIND_REGION
:
9007 start_ix
= BUILT_IN_GOMP_TARGET
;
9009 case GF_OMP_TARGET_KIND_DATA
:
9010 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
9012 case GF_OMP_TARGET_KIND_UPDATE
:
9013 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
9015 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9016 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9017 start_ix
= BUILT_IN_GOACC_PARALLEL
;
9019 case GF_OMP_TARGET_KIND_OACC_DATA
:
9020 start_ix
= BUILT_IN_GOACC_DATA_START
;
9022 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9023 start_ix
= BUILT_IN_GOACC_UPDATE
;
9025 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9026 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
9032 clauses
= gimple_omp_target_clauses (entry_stmt
);
9034 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
9035 library choose) and there is no conditional. */
9037 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
9039 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
9041 cond
= OMP_CLAUSE_IF_EXPR (c
);
9043 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
9046 /* Even if we pass it to all library function calls, it is currently only
9047 defined/used for the OpenMP target ones. */
9048 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
9049 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
9050 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
);
9052 device
= OMP_CLAUSE_DEVICE_ID (c
);
9053 clause_loc
= OMP_CLAUSE_LOCATION (c
);
9056 clause_loc
= gimple_location (entry_stmt
);
9058 /* Ensure 'device' is of the correct type. */
9059 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
9061 /* If we found the clause 'if (cond)', build
9062 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
9065 cond
= gimple_boolify (cond
);
9067 basic_block cond_bb
, then_bb
, else_bb
;
9071 tmp_var
= create_tmp_var (TREE_TYPE (device
));
9073 e
= split_block_after_labels (new_bb
);
9076 gsi
= gsi_last_bb (new_bb
);
9078 e
= split_block (new_bb
, gsi_stmt (gsi
));
9084 then_bb
= create_empty_bb (cond_bb
);
9085 else_bb
= create_empty_bb (then_bb
);
9086 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
9087 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
9089 stmt
= gimple_build_cond_empty (cond
);
9090 gsi
= gsi_last_bb (cond_bb
);
9091 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9093 gsi
= gsi_start_bb (then_bb
);
9094 stmt
= gimple_build_assign (tmp_var
, device
);
9095 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9097 gsi
= gsi_start_bb (else_bb
);
9098 stmt
= gimple_build_assign (tmp_var
,
9099 build_int_cst (integer_type_node
,
9100 GOMP_DEVICE_HOST_FALLBACK
));
9101 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9103 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
9104 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
9105 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
9106 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
9107 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
9108 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
9113 gsi
= gsi_last_bb (new_bb
);
9114 t
= gimple_omp_target_data_arg (entry_stmt
);
9117 t1
= size_zero_node
;
9118 t2
= build_zero_cst (ptr_type_node
);
9124 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
9125 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
9126 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
9127 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
9128 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
9132 /* The maximum number used by any start_ix, without varargs. */
9133 auto_vec
<tree
, 11> args
;
9134 args
.quick_push (device
);
9136 args
.quick_push (build_fold_addr_expr (child_fn
));
9139 case BUILT_IN_GOMP_TARGET
:
9140 case BUILT_IN_GOMP_TARGET_DATA
:
9141 case BUILT_IN_GOMP_TARGET_UPDATE
:
9142 /* This const void * is part of the current ABI, but we're not actually
9144 args
.quick_push (build_zero_cst (ptr_type_node
));
9146 case BUILT_IN_GOACC_DATA_START
:
9147 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9148 case BUILT_IN_GOACC_PARALLEL
:
9149 case BUILT_IN_GOACC_UPDATE
:
9154 args
.quick_push (t1
);
9155 args
.quick_push (t2
);
9156 args
.quick_push (t3
);
9157 args
.quick_push (t4
);
9160 case BUILT_IN_GOACC_DATA_START
:
9161 case BUILT_IN_GOMP_TARGET
:
9162 case BUILT_IN_GOMP_TARGET_DATA
:
9163 case BUILT_IN_GOMP_TARGET_UPDATE
:
9165 case BUILT_IN_GOACC_PARALLEL
:
9167 tree t_num_gangs
, t_num_workers
, t_vector_length
;
9169 /* Default values for num_gangs, num_workers, and vector_length. */
9170 t_num_gangs
= t_num_workers
= t_vector_length
9171 = fold_convert_loc (gimple_location (entry_stmt
),
9172 integer_type_node
, integer_one_node
);
9173 /* ..., but if present, use the value specified by the respective
9174 clause, making sure that are of the correct type. */
9175 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
9177 t_num_gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9179 OMP_CLAUSE_NUM_GANGS_EXPR (c
));
9180 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_WORKERS
);
9182 t_num_workers
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9184 OMP_CLAUSE_NUM_WORKERS_EXPR (c
));
9185 c
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
9187 t_vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9189 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c
));
9190 args
.quick_push (t_num_gangs
);
9191 args
.quick_push (t_num_workers
);
9192 args
.quick_push (t_vector_length
);
9195 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9196 case BUILT_IN_GOACC_UPDATE
:
9201 /* Default values for t_async. */
9202 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
9204 build_int_cst (integer_type_node
,
9206 /* ..., but if present, use the value specified by the respective
9207 clause, making sure that is of the correct type. */
9208 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
9210 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9212 OMP_CLAUSE_ASYNC_EXPR (c
));
9214 args
.quick_push (t_async
);
9215 /* Save the index, and... */
9216 t_wait_idx
= args
.length ();
9217 /* ... push a default value. */
9218 args
.quick_push (fold_convert_loc (gimple_location (entry_stmt
),
9220 integer_zero_node
));
9221 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
9226 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
9228 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
9230 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9232 OMP_CLAUSE_WAIT_EXPR (c
)));
9237 /* Now that we know the number, replace the default value. */
9238 args
.ordered_remove (t_wait_idx
);
9239 args
.quick_insert (t_wait_idx
,
9240 fold_convert_loc (gimple_location (entry_stmt
),
9242 build_int_cst (integer_type_node
, n
)));
9250 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
9251 gimple_set_location (g
, gimple_location (entry_stmt
));
9252 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
9256 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
9257 gsi_remove (&gsi
, true);
9262 gsi
= gsi_last_bb (region
->exit
);
9264 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
9265 gsi_remove (&gsi
, true);
9270 /* Expand the parallel region tree rooted at REGION. Expansion
9271 proceeds in depth-first order. Innermost regions are expanded
9272 first. This way, parallel regions that require a new function to
9273 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9274 internal dependencies in their body. */
9277 expand_omp (struct omp_region
*region
)
9281 location_t saved_location
;
9282 gimple inner_stmt
= NULL
;
9284 /* First, determine whether this is a combined parallel+workshare
9286 if (region
->type
== GIMPLE_OMP_PARALLEL
)
9287 determine_parallel_type (region
);
9289 if (region
->type
== GIMPLE_OMP_FOR
9290 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
9291 inner_stmt
= last_stmt (region
->inner
->entry
);
9294 expand_omp (region
->inner
);
9296 saved_location
= input_location
;
9297 if (gimple_has_location (last_stmt (region
->entry
)))
9298 input_location
= gimple_location (last_stmt (region
->entry
));
9300 switch (region
->type
)
9302 case GIMPLE_OMP_PARALLEL
:
9303 case GIMPLE_OMP_TASK
:
9304 expand_omp_taskreg (region
);
9307 case GIMPLE_OMP_FOR
:
9308 expand_omp_for (region
, inner_stmt
);
9311 case GIMPLE_OMP_SECTIONS
:
9312 expand_omp_sections (region
);
9315 case GIMPLE_OMP_SECTION
:
9316 /* Individual omp sections are handled together with their
9317 parent GIMPLE_OMP_SECTIONS region. */
9320 case GIMPLE_OMP_SINGLE
:
9321 expand_omp_single (region
);
9324 case GIMPLE_OMP_MASTER
:
9325 case GIMPLE_OMP_TASKGROUP
:
9326 case GIMPLE_OMP_ORDERED
:
9327 case GIMPLE_OMP_CRITICAL
:
9328 case GIMPLE_OMP_TEAMS
:
9329 expand_omp_synch (region
);
9332 case GIMPLE_OMP_ATOMIC_LOAD
:
9333 expand_omp_atomic (region
);
9336 case GIMPLE_OMP_TARGET
:
9337 expand_omp_target (region
);
9344 input_location
= saved_location
;
9345 region
= region
->next
;
9350 /* Helper for build_omp_regions. Scan the dominator tree starting at
9351 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9352 true, the function ends once a single tree is built (otherwise, whole
9353 forest of OMP constructs may be built). */
9356 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
9359 gimple_stmt_iterator gsi
;
9363 gsi
= gsi_last_bb (bb
);
9364 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
9366 struct omp_region
*region
;
9367 enum gimple_code code
;
9369 stmt
= gsi_stmt (gsi
);
9370 code
= gimple_code (stmt
);
9371 if (code
== GIMPLE_OMP_RETURN
)
9373 /* STMT is the return point out of region PARENT. Mark it
9374 as the exit point and make PARENT the immediately
9375 enclosing region. */
9376 gcc_assert (parent
);
9379 parent
= parent
->outer
;
9381 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
9383 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9384 GIMPLE_OMP_RETURN, but matches with
9385 GIMPLE_OMP_ATOMIC_LOAD. */
9386 gcc_assert (parent
);
9387 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
9390 parent
= parent
->outer
;
9392 else if (code
== GIMPLE_OMP_CONTINUE
)
9394 gcc_assert (parent
);
9397 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
9399 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9400 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9404 region
= new_omp_region (bb
, code
, parent
);
9406 if (code
== GIMPLE_OMP_TARGET
)
9408 switch (gimple_omp_target_kind (stmt
))
9410 case GF_OMP_TARGET_KIND_REGION
:
9411 case GF_OMP_TARGET_KIND_DATA
:
9412 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9413 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9414 case GF_OMP_TARGET_KIND_OACC_DATA
:
9416 case GF_OMP_TARGET_KIND_UPDATE
:
9417 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9418 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9419 /* ..., other than for those stand-alone directives... */
9426 /* ..., this directive becomes the parent for a new region. */
9432 if (single_tree
&& !parent
)
9435 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
9437 son
= next_dom_son (CDI_DOMINATORS
, son
))
9438 build_omp_regions_1 (son
, parent
, single_tree
);
9441 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9445 build_omp_regions_root (basic_block root
)
9447 gcc_assert (root_omp_region
== NULL
);
9448 build_omp_regions_1 (root
, NULL
, true);
9449 gcc_assert (root_omp_region
!= NULL
);
9452 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9455 omp_expand_local (basic_block head
)
9457 build_omp_regions_root (head
);
9458 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
9460 fprintf (dump_file
, "\nOMP region tree\n\n");
9461 dump_omp_region (dump_file
, root_omp_region
, 0);
9462 fprintf (dump_file
, "\n");
9465 remove_exit_barriers (root_omp_region
);
9466 expand_omp (root_omp_region
);
9468 free_omp_regions ();
9471 /* Scan the CFG and build a tree of OMP regions. Return the root of
9472 the OMP region tree. */
9475 build_omp_regions (void)
9477 gcc_assert (root_omp_region
== NULL
);
9478 calculate_dominance_info (CDI_DOMINATORS
);
9479 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
9482 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9485 execute_expand_omp (void)
9487 build_omp_regions ();
9489 if (!root_omp_region
)
9494 fprintf (dump_file
, "\nOMP region tree\n\n");
9495 dump_omp_region (dump_file
, root_omp_region
, 0);
9496 fprintf (dump_file
, "\n");
9499 remove_exit_barriers (root_omp_region
);
9501 expand_omp (root_omp_region
);
9503 cleanup_tree_cfg ();
9505 free_omp_regions ();
9510 /* OMP expansion -- the default pass, run before creation of SSA form. */
9514 const pass_data pass_data_expand_omp
=
9516 GIMPLE_PASS
, /* type */
9517 "ompexp", /* name */
9518 OPTGROUP_NONE
, /* optinfo_flags */
9519 TV_NONE
, /* tv_id */
9520 PROP_gimple_any
, /* properties_required */
9521 PROP_gimple_eomp
, /* properties_provided */
9522 0, /* properties_destroyed */
9523 0, /* todo_flags_start */
9524 0, /* todo_flags_finish */
9527 class pass_expand_omp
: public gimple_opt_pass
9530 pass_expand_omp (gcc::context
*ctxt
)
9531 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
9534 /* opt_pass methods: */
9535 virtual unsigned int execute (function
*)
9537 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
9538 || flag_openmp_simd
!= 0)
9541 /* This pass always runs, to provide PROP_gimple_eomp.
9542 But often, there is nothing to do. */
9546 return execute_expand_omp ();
9549 }; // class pass_expand_omp
9554 make_pass_expand_omp (gcc::context
*ctxt
)
9556 return new pass_expand_omp (ctxt
);
9561 const pass_data pass_data_expand_omp_ssa
=
9563 GIMPLE_PASS
, /* type */
9564 "ompexpssa", /* name */
9565 OPTGROUP_NONE
, /* optinfo_flags */
9566 TV_NONE
, /* tv_id */
9567 PROP_cfg
| PROP_ssa
, /* properties_required */
9568 PROP_gimple_eomp
, /* properties_provided */
9569 0, /* properties_destroyed */
9570 0, /* todo_flags_start */
9571 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
9574 class pass_expand_omp_ssa
: public gimple_opt_pass
9577 pass_expand_omp_ssa (gcc::context
*ctxt
)
9578 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
9581 /* opt_pass methods: */
9582 virtual bool gate (function
*fun
)
9584 return !(fun
->curr_properties
& PROP_gimple_eomp
);
9586 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
9588 }; // class pass_expand_omp_ssa
9593 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
9595 return new pass_expand_omp_ssa (ctxt
);
9598 /* Routines to lower OMP directives into OMP-GIMPLE. */
9600 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9601 convert it to gimple. */
9603 oacc_gimple_assign (tree dest
, tree_code op
, tree src
, gimple_seq
*seq
)
9607 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
9609 stmt
= gimple_build_assign (dest
, op
, dest
, src
);
9610 gimple_seq_add_stmt (seq
, stmt
);
9614 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9615 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9616 gimplify_assign (t
, rdest
, seq
);
9619 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9620 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9621 gimplify_assign (t
, idest
, seq
);
9624 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9625 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9626 gimplify_assign (t
, rsrc
, seq
);
9629 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9630 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9631 gimplify_assign (t
, isrc
, seq
);
9634 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9635 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9638 if (op
== PLUS_EXPR
)
9640 stmt
= gimple_build_assign (r
, op
, rdest
, rsrc
);
9641 gimple_seq_add_stmt (seq
, stmt
);
9643 stmt
= gimple_build_assign (i
, op
, idest
, isrc
);
9644 gimple_seq_add_stmt (seq
, stmt
);
9646 else if (op
== MULT_EXPR
)
9648 /* Let x = a + ib = dest, y = c + id = src.
9649 x * y = (ac - bd) + i(ad + bc) */
9650 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9651 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9652 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9653 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9655 stmt
= gimple_build_assign (ac
, MULT_EXPR
, rdest
, rsrc
);
9656 gimple_seq_add_stmt (seq
, stmt
);
9658 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, isrc
);
9659 gimple_seq_add_stmt (seq
, stmt
);
9661 stmt
= gimple_build_assign (r
, MINUS_EXPR
, ac
, bd
);
9662 gimple_seq_add_stmt (seq
, stmt
);
9664 stmt
= gimple_build_assign (ad
, MULT_EXPR
, rdest
, isrc
);
9665 gimple_seq_add_stmt (seq
, stmt
);
9667 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, rsrc
);
9668 gimple_seq_add_stmt (seq
, stmt
);
9670 stmt
= gimple_build_assign (i
, PLUS_EXPR
, ad
, bc
);
9671 gimple_seq_add_stmt (seq
, stmt
);
9676 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
9677 gimplify_assign (dest
, result
, seq
);
9680 /* Helper function to initialize local data for the reduction arrays.
9681 The reduction arrays need to be placed inside the calling function
9682 for accelerators, or else the host won't be able to preform the final
9686 oacc_initialize_reduction_data (tree clauses
, tree nthreads
,
9687 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9693 /* Find the innermost OpenACC parallel context. */
9694 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
9695 && (gimple_omp_target_kind (ctx
->stmt
)
9696 == GF_OMP_TARGET_KIND_OACC_PARALLEL
))
9700 gcc_checking_assert (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
9701 && (gimple_omp_target_kind (octx
->stmt
)
9702 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
9704 /* Extract the clauses. */
9705 oc
= gimple_omp_target_clauses (octx
->stmt
);
9707 /* Find the last outer clause. */
9708 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
9711 /* Allocate arrays for each reduction variable. */
9712 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9714 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9717 tree var
= OMP_CLAUSE_DECL (c
);
9718 tree type
= get_base_type (var
);
9719 tree array
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
),
9723 /* Calculate size of the reduction array. */
9724 t
= create_tmp_var (TREE_TYPE (nthreads
));
9725 stmt
= gimple_build_assign (t
, MULT_EXPR
, nthreads
,
9726 fold_convert (TREE_TYPE (nthreads
),
9727 TYPE_SIZE_UNIT (type
)));
9728 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9730 size
= create_tmp_var (sizetype
);
9731 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
9733 /* Now allocate memory for it. */
9734 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
9735 stmt
= gimple_build_call (call
, 1, size
);
9736 gimple_call_set_lhs (stmt
, array
);
9737 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9739 /* Map this array into the accelerator. */
9741 /* Add the reduction array to the list of clauses. */
9743 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
9744 OMP_CLAUSE_SET_MAP_KIND (t
, GOMP_MAP_FORCE_FROM
);
9745 OMP_CLAUSE_DECL (t
) = x
;
9746 OMP_CLAUSE_CHAIN (t
) = NULL
;
9748 OMP_CLAUSE_CHAIN (oc
) = t
;
9750 gimple_omp_target_set_clauses (as_a
<gomp_target
*> (octx
->stmt
), t
);
9751 OMP_CLAUSE_SIZE (t
) = size
;
9756 /* Helper function to process the array of partial reductions. Nthreads
9757 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9758 cannot be used here, because nthreads on the host may be different than
9759 on the accelerator. */
9762 oacc_finalize_reduction_data (tree clauses
, tree nthreads
,
9763 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9765 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
9770 let var = the original reduction variable
9771 let array = reduction variable array
9773 for (i = 0; i < nthreads; i++)
9777 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
9778 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
9779 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
9781 /* Create and initialize an index variable. */
9782 tree ix
= create_tmp_var (sizetype
);
9783 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
9786 /* Insert the loop header label here. */
9787 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
9789 /* Exit loop if ix >= nthreads. */
9790 x
= create_tmp_var (sizetype
);
9791 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
9792 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
9793 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9795 /* Insert the loop body label here. */
9796 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
9798 /* Collapse each reduction array, one element at a time. */
9799 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9801 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9804 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
9806 /* reduction(-:var) sums up the partial results, so it acts
9807 identically to reduction(+:var). */
9808 if (reduction_code
== MINUS_EXPR
)
9809 reduction_code
= PLUS_EXPR
;
9811 /* Set up reduction variable var. */
9812 var
= OMP_CLAUSE_DECL (c
);
9813 type
= get_base_type (var
);
9814 array
= lookup_oacc_reduction (oacc_get_reduction_array_id
9815 (OMP_CLAUSE_DECL (c
)), ctx
);
9817 /* Calculate the array offset. */
9818 tree offset
= create_tmp_var (sizetype
);
9819 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
9820 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, ix
);
9821 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9823 tree ptr
= create_tmp_var (TREE_TYPE (array
));
9824 stmt
= gimple_build_assign (ptr
, POINTER_PLUS_EXPR
, array
, offset
);
9825 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9827 /* Extract array[ix] into mem. */
9828 tree mem
= create_tmp_var (type
);
9829 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
9831 /* Find the original reduction variable. */
9832 if (is_reference (var
))
9833 var
= build_simple_mem_ref (var
);
9835 tree t
= create_tmp_var (type
);
9837 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
9838 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9840 /* var = var op mem */
9841 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
9843 case TRUTH_ANDIF_EXPR
:
9844 case TRUTH_ORIF_EXPR
:
9845 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
9847 gimplify_and_add (t
, stmt_seqp
);
9850 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9851 oacc_gimple_assign (t
, OMP_CLAUSE_REDUCTION_CODE (c
), mem
,
9855 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
9856 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
9857 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9860 /* Increment the induction variable. */
9861 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
9862 stmt
= gimple_build_assign (ix
, PLUS_EXPR
, ix
, one
);
9863 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9865 /* Go back to the top of the loop. */
9866 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
9868 /* Place the loop exit label here. */
9869 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
9872 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9873 scan that for reductions. */
9876 oacc_process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
9877 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
9879 gimple_stmt_iterator gsi
;
9880 gimple_seq inner
= NULL
;
9882 /* A collapse clause may have inserted a new bind block. */
9883 gsi
= gsi_start (*body
);
9884 while (!gsi_end_p (gsi
))
9886 gimple stmt
= gsi_stmt (gsi
);
9887 if (gbind
*bind_stmt
= dyn_cast
<gbind
*> (stmt
))
9889 inner
= gimple_bind_body (bind_stmt
);
9891 gsi
= gsi_start (*body
);
9893 else if (dyn_cast
<gomp_for
*> (stmt
))
9899 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9901 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
9903 bool reduction_found
= false;
9905 gimple stmt
= gsi_stmt (gsi
);
9907 switch (gimple_code (stmt
))
9909 case GIMPLE_OMP_FOR
:
9910 clauses
= gimple_omp_for_clauses (stmt
);
9912 /* Search for a reduction clause. */
9913 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9914 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
9916 reduction_found
= true;
9920 if (!reduction_found
)
9923 ctx
= maybe_lookup_ctx (stmt
);
9926 /* Extract the number of threads. */
9927 nthreads
= create_tmp_var (sizetype
);
9928 t
= oacc_max_threads (ctx
);
9929 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
9931 /* Determine if this is kernel will be executed on the host. */
9932 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
9933 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
9934 stmt
= gimple_build_call (call
, 0);
9935 gimple_call_set_lhs (stmt
, acc_device
);
9936 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9938 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9939 acc_device_host
= create_tmp_var (integer_type_node
,
9940 ".acc_device_host");
9941 gimplify_assign (acc_device_host
,
9942 build_int_cst (integer_type_node
,
9946 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9947 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9949 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9951 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9952 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9953 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9956 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9958 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9959 gimplify_assign (acc_device_host
,
9960 build_int_cst (integer_type_node
,
9961 GOMP_DEVICE_HOST_NONSHM
),
9964 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9965 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9967 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9969 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9970 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9971 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9974 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9976 oacc_initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
,
9978 oacc_finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
9981 // Scan for other directives which support reduction here.
9987 /* If ctx is a worksharing context inside of a cancellable parallel
9988 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
9989 and conditional branch to parallel's cancel_label to handle
9990 cancellation in the implicit barrier. */
9993 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
9995 gimple omp_return
= gimple_seq_last_stmt (*body
);
9996 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
9997 if (gimple_omp_return_nowait_p (omp_return
))
10000 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
10001 && ctx
->outer
->cancellable
)
10003 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
10004 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
10005 tree lhs
= create_tmp_var (c_bool_type
);
10006 gimple_omp_return_set_lhs (omp_return
, lhs
);
10007 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10008 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
10009 fold_convert (c_bool_type
,
10010 boolean_false_node
),
10011 ctx
->outer
->cancel_label
, fallthru_label
);
10012 gimple_seq_add_stmt (body
, g
);
10013 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
10017 /* Lower the OpenMP sections directive in the current statement in GSI_P.
10018 CTX is the enclosing OMP context for the current statement. */
10021 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10023 tree block
, control
;
10024 gimple_stmt_iterator tgsi
;
10025 gomp_sections
*stmt
;
10027 gbind
*new_stmt
, *bind
;
10028 gimple_seq ilist
, dlist
, olist
, new_body
;
10030 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
10032 push_gimplify_context ();
10036 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
10037 &ilist
, &dlist
, ctx
, NULL
);
10039 new_body
= gimple_omp_body (stmt
);
10040 gimple_omp_set_body (stmt
, NULL
);
10041 tgsi
= gsi_start (new_body
);
10042 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
10047 sec_start
= gsi_stmt (tgsi
);
10048 sctx
= maybe_lookup_ctx (sec_start
);
10051 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
10052 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
10053 GSI_CONTINUE_LINKING
);
10054 gimple_omp_set_body (sec_start
, NULL
);
10056 if (gsi_one_before_end_p (tgsi
))
10058 gimple_seq l
= NULL
;
10059 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
10061 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
10062 gimple_omp_section_set_last (sec_start
);
10065 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
10066 GSI_CONTINUE_LINKING
);
10069 block
= make_node (BLOCK
);
10070 bind
= gimple_build_bind (NULL
, new_body
, block
);
10073 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
10075 block
= make_node (BLOCK
);
10076 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10077 gsi_replace (gsi_p
, new_stmt
, true);
10079 pop_gimplify_context (new_stmt
);
10080 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10081 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10082 if (BLOCK_VARS (block
))
10083 TREE_USED (block
) = 1;
10086 gimple_seq_add_seq (&new_body
, ilist
);
10087 gimple_seq_add_stmt (&new_body
, stmt
);
10088 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
10089 gimple_seq_add_stmt (&new_body
, bind
);
10091 control
= create_tmp_var (unsigned_type_node
, ".section");
10092 t
= gimple_build_omp_continue (control
, control
);
10093 gimple_omp_sections_set_control (stmt
, control
);
10094 gimple_seq_add_stmt (&new_body
, t
);
10096 gimple_seq_add_seq (&new_body
, olist
);
10097 if (ctx
->cancellable
)
10098 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10099 gimple_seq_add_seq (&new_body
, dlist
);
10101 new_body
= maybe_catch_exception (new_body
);
10103 t
= gimple_build_omp_return
10104 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
10105 OMP_CLAUSE_NOWAIT
));
10106 gimple_seq_add_stmt (&new_body
, t
);
10107 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
10109 gimple_bind_set_body (new_stmt
, new_body
);
10113 /* A subroutine of lower_omp_single. Expand the simple form of
10114 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10116 if (GOMP_single_start ())
10118 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10120 FIXME. It may be better to delay expanding the logic of this until
10121 pass_expand_omp. The expanded logic may make the job more difficult
10122 to a synchronization analysis pass. */
10125 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
10127 location_t loc
= gimple_location (single_stmt
);
10128 tree tlabel
= create_artificial_label (loc
);
10129 tree flabel
= create_artificial_label (loc
);
10133 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
10134 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
10135 call
= gimple_build_call (decl
, 0);
10136 gimple_call_set_lhs (call
, lhs
);
10137 gimple_seq_add_stmt (pre_p
, call
);
10139 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
10140 fold_convert_loc (loc
, TREE_TYPE (lhs
),
10141 boolean_true_node
),
10143 gimple_seq_add_stmt (pre_p
, cond
);
10144 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
10145 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10146 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
10150 /* A subroutine of lower_omp_single. Expand the simple form of
10151 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10153 #pragma omp single copyprivate (a, b, c)
10155 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10158 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10164 GOMP_single_copy_end (©out);
10175 FIXME. It may be better to delay expanding the logic of this until
10176 pass_expand_omp. The expanded logic may make the job more difficult
10177 to a synchronization analysis pass. */
10180 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
10183 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
10184 gimple_seq copyin_seq
;
10185 location_t loc
= gimple_location (single_stmt
);
10187 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
10189 ptr_type
= build_pointer_type (ctx
->record_type
);
10190 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
10192 l0
= create_artificial_label (loc
);
10193 l1
= create_artificial_label (loc
);
10194 l2
= create_artificial_label (loc
);
10196 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
10197 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
10198 t
= fold_convert_loc (loc
, ptr_type
, t
);
10199 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
10201 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
10202 build_int_cst (ptr_type
, 0));
10203 t
= build3 (COND_EXPR
, void_type_node
, t
,
10204 build_and_jump (&l0
), build_and_jump (&l1
));
10205 gimplify_and_add (t
, pre_p
);
10207 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
10209 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10212 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
10215 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10216 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
10217 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
10218 gimplify_and_add (t
, pre_p
);
10220 t
= build_and_jump (&l2
);
10221 gimplify_and_add (t
, pre_p
);
10223 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
10225 gimple_seq_add_seq (pre_p
, copyin_seq
);
10227 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
10231 /* Expand code for an OpenMP single directive. */
10234 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10238 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
10240 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
10242 push_gimplify_context ();
10244 block
= make_node (BLOCK
);
10245 bind
= gimple_build_bind (NULL
, NULL
, block
);
10246 gsi_replace (gsi_p
, bind
, true);
10249 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
10250 &bind_body
, &dlist
, ctx
, NULL
);
10251 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
10253 gimple_seq_add_stmt (&bind_body
, single_stmt
);
10255 if (ctx
->record_type
)
10256 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
10258 lower_omp_single_simple (single_stmt
, &bind_body
);
10260 gimple_omp_set_body (single_stmt
, NULL
);
10262 gimple_seq_add_seq (&bind_body
, dlist
);
10264 bind_body
= maybe_catch_exception (bind_body
);
10266 t
= gimple_build_omp_return
10267 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
10268 OMP_CLAUSE_NOWAIT
));
10269 gimple_seq_add_stmt (&bind_body_tail
, t
);
10270 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
10271 if (ctx
->record_type
)
10273 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
10274 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10275 TREE_THIS_VOLATILE (clobber
) = 1;
10276 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
10277 clobber
), GSI_SAME_STMT
);
10279 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
10280 gimple_bind_set_body (bind
, bind_body
);
10282 pop_gimplify_context (bind
);
10284 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10285 BLOCK_VARS (block
) = ctx
->block_vars
;
10286 if (BLOCK_VARS (block
))
10287 TREE_USED (block
) = 1;
10291 /* Expand code for an OpenMP master directive. */
10294 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10296 tree block
, lab
= NULL
, x
, bfn_decl
;
10297 gimple stmt
= gsi_stmt (*gsi_p
);
10299 location_t loc
= gimple_location (stmt
);
10302 push_gimplify_context ();
10304 block
= make_node (BLOCK
);
10305 bind
= gimple_build_bind (NULL
, NULL
, block
);
10306 gsi_replace (gsi_p
, bind
, true);
10307 gimple_bind_add_stmt (bind
, stmt
);
10309 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
10310 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
10311 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
10312 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
10314 gimplify_and_add (x
, &tseq
);
10315 gimple_bind_add_seq (bind
, tseq
);
10317 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10318 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10319 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10320 gimple_omp_set_body (stmt
, NULL
);
10322 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
10324 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10326 pop_gimplify_context (bind
);
10328 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10329 BLOCK_VARS (block
) = ctx
->block_vars
;
10333 /* Expand code for an OpenMP taskgroup directive. */
10336 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10338 gimple stmt
= gsi_stmt (*gsi_p
);
10341 tree block
= make_node (BLOCK
);
10343 bind
= gimple_build_bind (NULL
, NULL
, block
);
10344 gsi_replace (gsi_p
, bind
, true);
10345 gimple_bind_add_stmt (bind
, stmt
);
10347 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
10349 gimple_bind_add_stmt (bind
, x
);
10351 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10352 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10353 gimple_omp_set_body (stmt
, NULL
);
10355 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10357 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10358 BLOCK_VARS (block
) = ctx
->block_vars
;
10362 /* Expand code for an OpenMP ordered directive. */
10365 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10368 gimple stmt
= gsi_stmt (*gsi_p
);
10372 push_gimplify_context ();
10374 block
= make_node (BLOCK
);
10375 bind
= gimple_build_bind (NULL
, NULL
, block
);
10376 gsi_replace (gsi_p
, bind
, true);
10377 gimple_bind_add_stmt (bind
, stmt
);
10379 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
10381 gimple_bind_add_stmt (bind
, x
);
10383 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10384 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10385 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10386 gimple_omp_set_body (stmt
, NULL
);
10388 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
10389 gimple_bind_add_stmt (bind
, x
);
10391 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10393 pop_gimplify_context (bind
);
10395 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10396 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10400 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10401 substitution of a couple of function calls. But in the NAMED case,
10402 requires that languages coordinate a symbol name. It is therefore
10403 best put here in common code. */
10405 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
10408 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10411 tree name
, lock
, unlock
;
10412 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
10414 location_t loc
= gimple_location (stmt
);
10417 name
= gimple_omp_critical_name (stmt
);
10422 if (!critical_name_mutexes
)
10423 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
10425 tree
*n
= critical_name_mutexes
->get (name
);
10430 decl
= create_tmp_var_raw (ptr_type_node
);
10432 new_str
= ACONCAT ((".gomp_critical_user_",
10433 IDENTIFIER_POINTER (name
), NULL
));
10434 DECL_NAME (decl
) = get_identifier (new_str
);
10435 TREE_PUBLIC (decl
) = 1;
10436 TREE_STATIC (decl
) = 1;
10437 DECL_COMMON (decl
) = 1;
10438 DECL_ARTIFICIAL (decl
) = 1;
10439 DECL_IGNORED_P (decl
) = 1;
10441 varpool_node::finalize_decl (decl
);
10443 critical_name_mutexes
->put (name
, decl
);
10448 /* If '#pragma omp critical' is inside offloaded region or
10449 inside function marked as offloadable, the symbol must be
10450 marked as offloadable too. */
10452 if (cgraph_node::get (current_function_decl
)->offloadable
)
10453 varpool_node::get_create (decl
)->offloadable
= 1;
10455 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
10456 if (is_gimple_omp_offloaded (octx
->stmt
))
10458 varpool_node::get_create (decl
)->offloadable
= 1;
10462 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
10463 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
10465 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
10466 unlock
= build_call_expr_loc (loc
, unlock
, 1,
10467 build_fold_addr_expr_loc (loc
, decl
));
10471 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
10472 lock
= build_call_expr_loc (loc
, lock
, 0);
10474 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
10475 unlock
= build_call_expr_loc (loc
, unlock
, 0);
10478 push_gimplify_context ();
10480 block
= make_node (BLOCK
);
10481 bind
= gimple_build_bind (NULL
, NULL
, block
);
10482 gsi_replace (gsi_p
, bind
, true);
10483 gimple_bind_add_stmt (bind
, stmt
);
10485 tbody
= gimple_bind_body (bind
);
10486 gimplify_and_add (lock
, &tbody
);
10487 gimple_bind_set_body (bind
, tbody
);
10489 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10490 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10491 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10492 gimple_omp_set_body (stmt
, NULL
);
10494 tbody
= gimple_bind_body (bind
);
10495 gimplify_and_add (unlock
, &tbody
);
10496 gimple_bind_set_body (bind
, tbody
);
10498 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10500 pop_gimplify_context (bind
);
10501 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10502 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10506 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10507 for a lastprivate clause. Given a loop control predicate of (V
10508 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10509 is appended to *DLIST, iterator initialization is appended to
10513 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
10514 gimple_seq
*dlist
, struct omp_context
*ctx
)
10516 tree clauses
, cond
, vinit
;
10517 enum tree_code cond_code
;
10520 cond_code
= fd
->loop
.cond_code
;
10521 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
10523 /* When possible, use a strict equality expression. This can let VRP
10524 type optimizations deduce the value and remove a copy. */
10525 if (tree_fits_shwi_p (fd
->loop
.step
))
10527 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
10528 if (step
== 1 || step
== -1)
10529 cond_code
= EQ_EXPR
;
10532 tree n2
= fd
->loop
.n2
;
10533 if (fd
->collapse
> 1
10534 && TREE_CODE (n2
) != INTEGER_CST
10535 && gimple_omp_for_combined_into_p (fd
->for_stmt
)
10536 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
10538 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
10539 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
)
10541 struct omp_for_data outer_fd
;
10542 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
10543 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
10546 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
10548 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
10550 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
10551 if (!gimple_seq_empty_p (stmts
))
10553 gimple_seq_add_seq (&stmts
, *dlist
);
10556 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10557 vinit
= fd
->loop
.n1
;
10558 if (cond_code
== EQ_EXPR
10559 && tree_fits_shwi_p (fd
->loop
.n2
)
10560 && ! integer_zerop (fd
->loop
.n2
))
10561 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
10563 vinit
= unshare_expr (vinit
);
10565 /* Initialize the iterator variable, so that threads that don't execute
10566 any iterations don't execute the lastprivate clauses by accident. */
10567 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
10572 /* Lower code for an OMP loop directive. */
10575 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10577 tree
*rhs_p
, block
;
10578 struct omp_for_data fd
, *fdp
= NULL
;
10579 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
10581 gimple_seq omp_for_body
, body
, dlist
;
10584 push_gimplify_context ();
10586 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
10588 block
= make_node (BLOCK
);
10589 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10590 /* Replace at gsi right away, so that 'stmt' is no member
10591 of a sequence anymore as we're going to add to to a different
10593 gsi_replace (gsi_p
, new_stmt
, true);
10595 /* Move declaration of temporaries in the loop body before we make
10597 omp_for_body
= gimple_omp_body (stmt
);
10598 if (!gimple_seq_empty_p (omp_for_body
)
10599 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
10602 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
10603 tree vars
= gimple_bind_vars (inner_bind
);
10604 gimple_bind_append_vars (new_stmt
, vars
);
10605 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10606 keep them on the inner_bind and it's block. */
10607 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
10608 if (gimple_bind_block (inner_bind
))
10609 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
10612 if (gimple_omp_for_combined_into_p (stmt
))
10614 extract_omp_for_data (stmt
, &fd
, NULL
);
10617 /* We need two temporaries with fd.loop.v type (istart/iend)
10618 and then (fd.collapse - 1) temporaries with the same
10619 type for count2 ... countN-1 vars if not constant. */
10621 tree type
= fd
.iter_type
;
10622 if (fd
.collapse
> 1
10623 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
10624 count
+= fd
.collapse
- 1;
10625 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
10626 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
10627 tree clauses
= *pc
;
10630 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
10631 OMP_CLAUSE__LOOPTEMP_
);
10632 for (i
= 0; i
< count
; i
++)
10637 gcc_assert (outerc
);
10638 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
10639 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
10640 OMP_CLAUSE__LOOPTEMP_
);
10644 temp
= create_tmp_var (type
);
10645 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
10647 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
10648 OMP_CLAUSE_DECL (*pc
) = temp
;
10649 pc
= &OMP_CLAUSE_CHAIN (*pc
);
10654 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10657 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
10659 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
10661 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10663 /* Lower the header expressions. At this point, we can assume that
10664 the header is of the form:
10666 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10668 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10669 using the .omp_data_s mapping, if needed. */
10670 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
10672 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
10673 if (!is_gimple_min_invariant (*rhs_p
))
10674 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10676 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
10677 if (!is_gimple_min_invariant (*rhs_p
))
10678 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10680 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
10681 if (!is_gimple_min_invariant (*rhs_p
))
10682 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10685 /* Once lowered, extract the bounds and clauses. */
10686 extract_omp_for_data (stmt
, &fd
, NULL
);
10688 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
10690 gimple_seq_add_stmt (&body
, stmt
);
10691 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
10693 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
10696 /* After the loop, add exit clauses. */
10697 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
10699 if (ctx
->cancellable
)
10700 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
10702 gimple_seq_add_seq (&body
, dlist
);
10704 body
= maybe_catch_exception (body
);
10706 /* Region exit marker goes at the end of the loop body. */
10707 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
10708 maybe_add_implicit_barrier_cancel (ctx
, &body
);
10709 pop_gimplify_context (new_stmt
);
10711 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10712 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
10713 if (BLOCK_VARS (block
))
10714 TREE_USED (block
) = 1;
10716 gimple_bind_set_body (new_stmt
, body
);
10717 gimple_omp_set_body (stmt
, NULL
);
10718 gimple_omp_for_set_pre_body (stmt
, NULL
);
10721 /* Callback for walk_stmts. Check if the current statement only contains
10722 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10725 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
10726 bool *handled_ops_p
,
10727 struct walk_stmt_info
*wi
)
10729 int *info
= (int *) wi
->info
;
10730 gimple stmt
= gsi_stmt (*gsi_p
);
10732 *handled_ops_p
= true;
10733 switch (gimple_code (stmt
))
10737 case GIMPLE_OMP_FOR
:
10738 case GIMPLE_OMP_SECTIONS
:
10739 *info
= *info
== 0 ? 1 : -1;
10748 struct omp_taskcopy_context
10750 /* This field must be at the beginning, as we do "inheritance": Some
10751 callback functions for tree-inline.c (e.g., omp_copy_decl)
10752 receive a copy_body_data pointer that is up-casted to an
10753 omp_context pointer. */
10759 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
10761 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
10763 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
10764 return create_tmp_var (TREE_TYPE (var
));
10770 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
10772 tree name
, new_fields
= NULL
, type
, f
;
10774 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
10775 name
= DECL_NAME (TYPE_NAME (orig_type
));
10776 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
10777 TYPE_DECL
, name
, type
);
10778 TYPE_NAME (type
) = name
;
10780 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
10782 tree new_f
= copy_node (f
);
10783 DECL_CONTEXT (new_f
) = type
;
10784 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
10785 TREE_CHAIN (new_f
) = new_fields
;
10786 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10787 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10788 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
10790 new_fields
= new_f
;
10791 tcctx
->cb
.decl_map
->put (f
, new_f
);
10793 TYPE_FIELDS (type
) = nreverse (new_fields
);
10794 layout_type (type
);
10798 /* Create task copyfn. */
10801 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
10803 struct function
*child_cfun
;
10804 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
10805 tree record_type
, srecord_type
, bind
, list
;
10806 bool record_needs_remap
= false, srecord_needs_remap
= false;
10808 struct omp_taskcopy_context tcctx
;
10809 location_t loc
= gimple_location (task_stmt
);
10811 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
10812 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
10813 gcc_assert (child_cfun
->cfg
== NULL
);
10814 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
10816 /* Reset DECL_CONTEXT on function arguments. */
10817 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
10818 DECL_CONTEXT (t
) = child_fn
;
10820 /* Populate the function. */
10821 push_gimplify_context ();
10822 push_cfun (child_cfun
);
10824 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
10825 TREE_SIDE_EFFECTS (bind
) = 1;
10827 DECL_SAVED_TREE (child_fn
) = bind
;
10828 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
10830 /* Remap src and dst argument types if needed. */
10831 record_type
= ctx
->record_type
;
10832 srecord_type
= ctx
->srecord_type
;
10833 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
10834 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10836 record_needs_remap
= true;
10839 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
10840 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10842 srecord_needs_remap
= true;
10846 if (record_needs_remap
|| srecord_needs_remap
)
10848 memset (&tcctx
, '\0', sizeof (tcctx
));
10849 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
10850 tcctx
.cb
.dst_fn
= child_fn
;
10851 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
10852 gcc_checking_assert (tcctx
.cb
.src_node
);
10853 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
10854 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
10855 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
10856 tcctx
.cb
.eh_lp_nr
= 0;
10857 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
10858 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
10861 if (record_needs_remap
)
10862 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
10863 if (srecord_needs_remap
)
10864 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
10867 tcctx
.cb
.decl_map
= NULL
;
10869 arg
= DECL_ARGUMENTS (child_fn
);
10870 TREE_TYPE (arg
) = build_pointer_type (record_type
);
10871 sarg
= DECL_CHAIN (arg
);
10872 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
10874 /* First pass: initialize temporaries used in record_type and srecord_type
10875 sizes and field offsets. */
10876 if (tcctx
.cb
.decl_map
)
10877 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10878 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10882 decl
= OMP_CLAUSE_DECL (c
);
10883 p
= tcctx
.cb
.decl_map
->get (decl
);
10886 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10887 sf
= (tree
) n
->value
;
10888 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10889 src
= build_simple_mem_ref_loc (loc
, sarg
);
10890 src
= omp_build_component_ref (src
, sf
);
10891 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
10892 append_to_statement_list (t
, &list
);
10895 /* Second pass: copy shared var pointers and copy construct non-VLA
10896 firstprivate vars. */
10897 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10898 switch (OMP_CLAUSE_CODE (c
))
10900 case OMP_CLAUSE_SHARED
:
10901 decl
= OMP_CLAUSE_DECL (c
);
10902 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10905 f
= (tree
) n
->value
;
10906 if (tcctx
.cb
.decl_map
)
10907 f
= *tcctx
.cb
.decl_map
->get (f
);
10908 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10909 sf
= (tree
) n
->value
;
10910 if (tcctx
.cb
.decl_map
)
10911 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10912 src
= build_simple_mem_ref_loc (loc
, sarg
);
10913 src
= omp_build_component_ref (src
, sf
);
10914 dst
= build_simple_mem_ref_loc (loc
, arg
);
10915 dst
= omp_build_component_ref (dst
, f
);
10916 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10917 append_to_statement_list (t
, &list
);
10919 case OMP_CLAUSE_FIRSTPRIVATE
:
10920 decl
= OMP_CLAUSE_DECL (c
);
10921 if (is_variable_sized (decl
))
10923 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10926 f
= (tree
) n
->value
;
10927 if (tcctx
.cb
.decl_map
)
10928 f
= *tcctx
.cb
.decl_map
->get (f
);
10929 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10932 sf
= (tree
) n
->value
;
10933 if (tcctx
.cb
.decl_map
)
10934 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10935 src
= build_simple_mem_ref_loc (loc
, sarg
);
10936 src
= omp_build_component_ref (src
, sf
);
10937 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
10938 src
= build_simple_mem_ref_loc (loc
, src
);
10942 dst
= build_simple_mem_ref_loc (loc
, arg
);
10943 dst
= omp_build_component_ref (dst
, f
);
10944 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10945 append_to_statement_list (t
, &list
);
10947 case OMP_CLAUSE_PRIVATE
:
10948 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
10950 decl
= OMP_CLAUSE_DECL (c
);
10951 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10952 f
= (tree
) n
->value
;
10953 if (tcctx
.cb
.decl_map
)
10954 f
= *tcctx
.cb
.decl_map
->get (f
);
10955 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10958 sf
= (tree
) n
->value
;
10959 if (tcctx
.cb
.decl_map
)
10960 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10961 src
= build_simple_mem_ref_loc (loc
, sarg
);
10962 src
= omp_build_component_ref (src
, sf
);
10963 if (use_pointer_for_field (decl
, NULL
))
10964 src
= build_simple_mem_ref_loc (loc
, src
);
10968 dst
= build_simple_mem_ref_loc (loc
, arg
);
10969 dst
= omp_build_component_ref (dst
, f
);
10970 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10971 append_to_statement_list (t
, &list
);
10977 /* Last pass: handle VLA firstprivates. */
10978 if (tcctx
.cb
.decl_map
)
10979 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10980 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10984 decl
= OMP_CLAUSE_DECL (c
);
10985 if (!is_variable_sized (decl
))
10987 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10990 f
= (tree
) n
->value
;
10991 f
= *tcctx
.cb
.decl_map
->get (f
);
10992 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
10993 ind
= DECL_VALUE_EXPR (decl
);
10994 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
10995 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
10996 n
= splay_tree_lookup (ctx
->sfield_map
,
10997 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10998 sf
= (tree
) n
->value
;
10999 sf
= *tcctx
.cb
.decl_map
->get (sf
);
11000 src
= build_simple_mem_ref_loc (loc
, sarg
);
11001 src
= omp_build_component_ref (src
, sf
);
11002 src
= build_simple_mem_ref_loc (loc
, src
);
11003 dst
= build_simple_mem_ref_loc (loc
, arg
);
11004 dst
= omp_build_component_ref (dst
, f
);
11005 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
11006 append_to_statement_list (t
, &list
);
11007 n
= splay_tree_lookup (ctx
->field_map
,
11008 (splay_tree_key
) TREE_OPERAND (ind
, 0));
11009 df
= (tree
) n
->value
;
11010 df
= *tcctx
.cb
.decl_map
->get (df
);
11011 ptr
= build_simple_mem_ref_loc (loc
, arg
);
11012 ptr
= omp_build_component_ref (ptr
, df
);
11013 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
11014 build_fold_addr_expr_loc (loc
, dst
));
11015 append_to_statement_list (t
, &list
);
11018 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
11019 append_to_statement_list (t
, &list
);
11021 if (tcctx
.cb
.decl_map
)
11022 delete tcctx
.cb
.decl_map
;
11023 pop_gimplify_context (NULL
);
11024 BIND_EXPR_BODY (bind
) = list
;
11029 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
11033 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
11035 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
11036 OMP_CLAUSE_DEPEND
);
11037 gcc_assert (clauses
);
11038 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11039 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
11040 switch (OMP_CLAUSE_DEPEND_KIND (c
))
11042 case OMP_CLAUSE_DEPEND_IN
:
11045 case OMP_CLAUSE_DEPEND_OUT
:
11046 case OMP_CLAUSE_DEPEND_INOUT
:
11050 gcc_unreachable ();
11052 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
11053 tree array
= create_tmp_var (type
);
11054 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
11056 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
11057 gimple_seq_add_stmt (iseq
, g
);
11058 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
11060 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
11061 gimple_seq_add_stmt (iseq
, g
);
11062 for (i
= 0; i
< 2; i
++)
11064 if ((i
? n_in
: n_out
) == 0)
11066 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11067 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
11068 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
11070 tree t
= OMP_CLAUSE_DECL (c
);
11071 t
= fold_convert (ptr_type_node
, t
);
11072 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
11073 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
11074 NULL_TREE
, NULL_TREE
);
11075 g
= gimple_build_assign (r
, t
);
11076 gimple_seq_add_stmt (iseq
, g
);
11079 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
11080 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
11081 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
11082 OMP_CLAUSE_CHAIN (c
) = *p
;
11084 tree clobber
= build_constructor (type
, NULL
);
11085 TREE_THIS_VOLATILE (clobber
) = 1;
11086 g
= gimple_build_assign (array
, clobber
);
11087 gimple_seq_add_stmt (oseq
, g
);
11090 /* Lower the OpenMP parallel or task directive in the current statement
11091 in GSI_P. CTX holds context information for the directive. */
11094 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11098 gimple stmt
= gsi_stmt (*gsi_p
);
11099 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
11100 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
11101 location_t loc
= gimple_location (stmt
);
11103 clauses
= gimple_omp_taskreg_clauses (stmt
);
11105 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
11106 par_body
= gimple_bind_body (par_bind
);
11107 child_fn
= ctx
->cb
.dst_fn
;
11108 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11109 && !gimple_omp_parallel_combined_p (stmt
))
11111 struct walk_stmt_info wi
;
11114 memset (&wi
, 0, sizeof (wi
));
11116 wi
.val_only
= true;
11117 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
11119 gimple_omp_parallel_set_combined_p (stmt
, true);
11121 gimple_seq dep_ilist
= NULL
;
11122 gimple_seq dep_olist
= NULL
;
11123 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11124 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
11126 push_gimplify_context ();
11127 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11128 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
11131 if (ctx
->srecord_type
)
11132 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
11134 push_gimplify_context ();
11139 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
11140 lower_omp (&par_body
, ctx
);
11141 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
11142 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
11144 /* Declare all the variables created by mapping and the variables
11145 declared in the scope of the parallel body. */
11146 record_vars_into (ctx
->block_vars
, child_fn
);
11147 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
11149 if (ctx
->record_type
)
11152 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
11153 : ctx
->record_type
, ".omp_data_o");
11154 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11155 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11156 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
11161 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
11162 lower_send_shared_vars (&ilist
, &olist
, ctx
);
11164 if (ctx
->record_type
)
11166 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
11167 TREE_THIS_VOLATILE (clobber
) = 1;
11168 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11172 /* Once all the expansions are done, sequence all the different
11173 fragments inside gimple_omp_body. */
11177 if (ctx
->record_type
)
11179 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11180 /* fixup_child_record_type might have changed receiver_decl's type. */
11181 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11182 gimple_seq_add_stmt (&new_body
,
11183 gimple_build_assign (ctx
->receiver_decl
, t
));
11186 gimple_seq_add_seq (&new_body
, par_ilist
);
11187 gimple_seq_add_seq (&new_body
, par_body
);
11188 gimple_seq_add_seq (&new_body
, par_rlist
);
11189 if (ctx
->cancellable
)
11190 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
11191 gimple_seq_add_seq (&new_body
, par_olist
);
11192 new_body
= maybe_catch_exception (new_body
);
11193 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
11194 gimple_seq_add_stmt (&new_body
,
11195 gimple_build_omp_continue (integer_zero_node
,
11196 integer_zero_node
));
11197 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11198 gimple_omp_set_body (stmt
, new_body
);
11200 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
11201 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
11202 gimple_bind_add_seq (bind
, ilist
);
11203 gimple_bind_add_stmt (bind
, stmt
);
11204 gimple_bind_add_seq (bind
, olist
);
11206 pop_gimplify_context (NULL
);
11210 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11211 gimple_bind_add_stmt (dep_bind
, bind
);
11212 gimple_bind_add_seq (dep_bind
, dep_olist
);
11213 pop_gimplify_context (dep_bind
);
11217 /* Lower the GIMPLE_OMP_TARGET in the current statement
11218 in GSI_P. CTX holds context information for the directive. */
11221 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11224 tree child_fn
, t
, c
;
11225 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
11226 gbind
*tgt_bind
, *bind
;
11227 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
11228 location_t loc
= gimple_location (stmt
);
11229 bool offloaded
, data_region
;
11230 unsigned int map_cnt
= 0;
11232 offloaded
= is_gimple_omp_offloaded (stmt
);
11233 switch (gimple_omp_target_kind (stmt
))
11235 case GF_OMP_TARGET_KIND_REGION
:
11236 case GF_OMP_TARGET_KIND_UPDATE
:
11237 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11238 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11239 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11240 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11241 data_region
= false;
11243 case GF_OMP_TARGET_KIND_DATA
:
11244 case GF_OMP_TARGET_KIND_OACC_DATA
:
11245 data_region
= true;
11248 gcc_unreachable ();
11251 clauses
= gimple_omp_target_clauses (stmt
);
11257 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
11258 tgt_body
= gimple_bind_body (tgt_bind
);
11260 else if (data_region
)
11261 tgt_body
= gimple_omp_body (stmt
);
11262 child_fn
= ctx
->cb
.dst_fn
;
11264 push_gimplify_context ();
11269 && is_gimple_omp_oacc (stmt
))
11270 oacc_process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
11272 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11273 switch (OMP_CLAUSE_CODE (c
))
11279 case OMP_CLAUSE_MAP
:
11280 #ifdef ENABLE_CHECKING
11281 /* First check what we're prepared to handle in the following. */
11282 switch (OMP_CLAUSE_MAP_KIND (c
))
11284 case GOMP_MAP_ALLOC
:
11286 case GOMP_MAP_FROM
:
11287 case GOMP_MAP_TOFROM
:
11288 case GOMP_MAP_POINTER
:
11289 case GOMP_MAP_TO_PSET
:
11291 case GOMP_MAP_FORCE_ALLOC
:
11292 case GOMP_MAP_FORCE_TO
:
11293 case GOMP_MAP_FORCE_FROM
:
11294 case GOMP_MAP_FORCE_TOFROM
:
11295 case GOMP_MAP_FORCE_PRESENT
:
11296 case GOMP_MAP_FORCE_DEALLOC
:
11297 case GOMP_MAP_FORCE_DEVICEPTR
:
11298 gcc_assert (is_gimple_omp_oacc (stmt
));
11301 gcc_unreachable ();
11305 case OMP_CLAUSE_TO
:
11306 case OMP_CLAUSE_FROM
:
11307 var
= OMP_CLAUSE_DECL (c
);
11310 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11311 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11316 if (DECL_SIZE (var
)
11317 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
11319 tree var2
= DECL_VALUE_EXPR (var
);
11320 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
11321 var2
= TREE_OPERAND (var2
, 0);
11322 gcc_assert (DECL_P (var2
));
11326 if (!maybe_lookup_field (var
, ctx
))
11331 x
= build_receiver_ref (var
, true, ctx
);
11332 tree new_var
= lookup_decl (var
, ctx
);
11333 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11334 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11335 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11336 x
= build_simple_mem_ref (x
);
11337 SET_DECL_VALUE_EXPR (new_var
, x
);
11338 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11345 target_nesting_level
++;
11346 lower_omp (&tgt_body
, ctx
);
11347 target_nesting_level
--;
11349 else if (data_region
)
11350 lower_omp (&tgt_body
, ctx
);
11354 /* Declare all the variables created by mapping and the variables
11355 declared in the scope of the target body. */
11356 record_vars_into (ctx
->block_vars
, child_fn
);
11357 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
11362 if (ctx
->record_type
)
11365 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
11366 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11367 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11368 t
= make_tree_vec (3);
11369 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
11370 TREE_VEC_ELT (t
, 1)
11371 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
11372 ".omp_data_sizes");
11373 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
11374 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
11375 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
11378 if (is_gimple_omp_oacc (stmt
))
11380 tkind_type
= short_unsigned_type_node
;
11385 tkind_type
= unsigned_char_type_node
;
11388 TREE_VEC_ELT (t
, 2)
11389 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
11390 ".omp_data_kinds");
11391 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
11392 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
11393 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
11394 gimple_omp_target_set_data_arg (stmt
, t
);
11396 vec
<constructor_elt
, va_gc
> *vsize
;
11397 vec
<constructor_elt
, va_gc
> *vkind
;
11398 vec_alloc (vsize
, map_cnt
);
11399 vec_alloc (vkind
, map_cnt
);
11400 unsigned int map_idx
= 0;
11402 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11403 switch (OMP_CLAUSE_CODE (c
))
11409 case OMP_CLAUSE_MAP
:
11410 case OMP_CLAUSE_TO
:
11411 case OMP_CLAUSE_FROM
:
11413 ovar
= OMP_CLAUSE_DECL (c
);
11414 if (!DECL_P (ovar
))
11416 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11417 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11419 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
11420 == get_base_address (ovar
));
11421 nc
= OMP_CLAUSE_CHAIN (c
);
11422 ovar
= OMP_CLAUSE_DECL (nc
);
11426 tree x
= build_sender_ref (ovar
, ctx
);
11428 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
11429 gimplify_assign (x
, v
, &ilist
);
11435 if (DECL_SIZE (ovar
)
11436 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
11438 tree ovar2
= DECL_VALUE_EXPR (ovar
);
11439 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
11440 ovar2
= TREE_OPERAND (ovar2
, 0);
11441 gcc_assert (DECL_P (ovar2
));
11444 if (!maybe_lookup_field (ovar
, ctx
))
11448 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
11449 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
11450 talign
= DECL_ALIGN_UNIT (ovar
);
11453 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11454 tree x
= build_sender_ref (ovar
, ctx
);
11455 if (maybe_lookup_oacc_reduction (var
, ctx
))
11457 gcc_checking_assert (offloaded
11458 && is_gimple_omp_oacc (stmt
));
11459 gimplify_assign (x
, var
, &ilist
);
11461 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11462 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11463 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11464 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
11466 gcc_assert (offloaded
);
11468 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
11469 mark_addressable (avar
);
11470 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
11471 talign
= DECL_ALIGN_UNIT (avar
);
11472 avar
= build_fold_addr_expr (avar
);
11473 gimplify_assign (x
, avar
, &ilist
);
11475 else if (is_gimple_reg (var
))
11477 gcc_assert (offloaded
);
11478 tree avar
= create_tmp_var (TREE_TYPE (var
));
11479 mark_addressable (avar
);
11480 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
11481 if (GOMP_MAP_COPY_TO_P (map_kind
)
11482 || map_kind
== GOMP_MAP_POINTER
11483 || map_kind
== GOMP_MAP_TO_PSET
11484 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11485 gimplify_assign (avar
, var
, &ilist
);
11486 avar
= build_fold_addr_expr (avar
);
11487 gimplify_assign (x
, avar
, &ilist
);
11488 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
11489 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11490 && !TYPE_READONLY (TREE_TYPE (var
)))
11492 x
= build_sender_ref (ovar
, ctx
);
11493 x
= build_simple_mem_ref (x
);
11494 gimplify_assign (var
, x
, &olist
);
11499 var
= build_fold_addr_expr (var
);
11500 gimplify_assign (x
, var
, &ilist
);
11503 tree s
= OMP_CLAUSE_SIZE (c
);
11504 if (s
== NULL_TREE
)
11505 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11506 s
= fold_convert (size_type_node
, s
);
11507 tree purpose
= size_int (map_idx
++);
11508 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11509 if (TREE_CODE (s
) != INTEGER_CST
)
11510 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11512 unsigned HOST_WIDE_INT tkind
;
11513 switch (OMP_CLAUSE_CODE (c
))
11515 case OMP_CLAUSE_MAP
:
11516 tkind
= OMP_CLAUSE_MAP_KIND (c
);
11518 case OMP_CLAUSE_TO
:
11519 tkind
= GOMP_MAP_TO
;
11521 case OMP_CLAUSE_FROM
:
11522 tkind
= GOMP_MAP_FROM
;
11525 gcc_unreachable ();
11527 gcc_checking_assert (tkind
11528 < (HOST_WIDE_INT_C (1U) << talign_shift
));
11529 talign
= ceil_log2 (talign
);
11530 tkind
|= talign
<< talign_shift
;
11531 gcc_checking_assert (tkind
11532 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11533 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
11534 build_int_cstu (tkind_type
, tkind
));
11539 gcc_assert (map_idx
== map_cnt
);
11541 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
11542 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
11543 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
11544 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
11545 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
11547 gimple_seq initlist
= NULL
;
11548 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
11549 TREE_VEC_ELT (t
, 1)),
11550 &initlist
, true, NULL_TREE
);
11551 gimple_seq_add_seq (&ilist
, initlist
);
11553 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
11555 TREE_THIS_VOLATILE (clobber
) = 1;
11556 gimple_seq_add_stmt (&olist
,
11557 gimple_build_assign (TREE_VEC_ELT (t
, 1),
11561 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
11562 TREE_THIS_VOLATILE (clobber
) = 1;
11563 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11567 /* Once all the expansions are done, sequence all the different
11568 fragments inside gimple_omp_body. */
11573 && ctx
->record_type
)
11575 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11576 /* fixup_child_record_type might have changed receiver_decl's type. */
11577 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11578 gimple_seq_add_stmt (&new_body
,
11579 gimple_build_assign (ctx
->receiver_decl
, t
));
11584 gimple_seq_add_seq (&new_body
, tgt_body
);
11585 new_body
= maybe_catch_exception (new_body
);
11587 else if (data_region
)
11588 new_body
= tgt_body
;
11589 if (offloaded
|| data_region
)
11591 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11592 gimple_omp_set_body (stmt
, new_body
);
11595 bind
= gimple_build_bind (NULL
, NULL
,
11596 tgt_bind
? gimple_bind_block (tgt_bind
)
11598 gsi_replace (gsi_p
, bind
, true);
11599 gimple_bind_add_seq (bind
, irlist
);
11600 gimple_bind_add_seq (bind
, ilist
);
11601 gimple_bind_add_stmt (bind
, stmt
);
11602 gimple_bind_add_seq (bind
, olist
);
11603 gimple_bind_add_seq (bind
, orlist
);
11605 pop_gimplify_context (NULL
);
11608 /* Expand code for an OpenMP teams directive. */
11611 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11613 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
11614 push_gimplify_context ();
11616 tree block
= make_node (BLOCK
);
11617 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
11618 gsi_replace (gsi_p
, bind
, true);
11619 gimple_seq bind_body
= NULL
;
11620 gimple_seq dlist
= NULL
;
11621 gimple_seq olist
= NULL
;
11623 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11624 OMP_CLAUSE_NUM_TEAMS
);
11625 if (num_teams
== NULL_TREE
)
11626 num_teams
= build_int_cst (unsigned_type_node
, 0);
11629 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
11630 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
11631 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
11633 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11634 OMP_CLAUSE_THREAD_LIMIT
);
11635 if (thread_limit
== NULL_TREE
)
11636 thread_limit
= build_int_cst (unsigned_type_node
, 0);
11639 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
11640 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
11641 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
11645 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
11646 &bind_body
, &dlist
, ctx
, NULL
);
11647 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
11648 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
11649 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
11651 location_t loc
= gimple_location (teams_stmt
);
11652 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
11653 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
11654 gimple_set_location (call
, loc
);
11655 gimple_seq_add_stmt (&bind_body
, call
);
11657 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
11658 gimple_omp_set_body (teams_stmt
, NULL
);
11659 gimple_seq_add_seq (&bind_body
, olist
);
11660 gimple_seq_add_seq (&bind_body
, dlist
);
11661 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
11662 gimple_bind_set_body (bind
, bind_body
);
11664 pop_gimplify_context (bind
);
11666 gimple_bind_append_vars (bind
, ctx
->block_vars
);
11667 BLOCK_VARS (block
) = ctx
->block_vars
;
11668 if (BLOCK_VARS (block
))
11669 TREE_USED (block
) = 1;
11673 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11674 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11675 of OMP context, but with task_shared_vars set. */
11678 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
11683 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11684 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
11687 if (task_shared_vars
11689 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
11692 /* If a global variable has been privatized, TREE_CONSTANT on
11693 ADDR_EXPR might be wrong. */
11694 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
11695 recompute_tree_invariant_for_addr_expr (t
);
11697 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
11702 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11704 gimple stmt
= gsi_stmt (*gsi_p
);
11705 struct walk_stmt_info wi
;
11708 if (gimple_has_location (stmt
))
11709 input_location
= gimple_location (stmt
);
11711 if (task_shared_vars
)
11712 memset (&wi
, '\0', sizeof (wi
));
11714 /* If we have issued syntax errors, avoid doing any heavy lifting.
11715 Just replace the OMP directives with a NOP to avoid
11716 confusing RTL expansion. */
11717 if (seen_error () && is_gimple_omp (stmt
))
11719 gsi_replace (gsi_p
, gimple_build_nop (), true);
11723 switch (gimple_code (stmt
))
11727 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
11728 if ((ctx
|| task_shared_vars
)
11729 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
11730 lower_omp_regimplify_p
,
11731 ctx
? NULL
: &wi
, NULL
)
11732 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
11733 lower_omp_regimplify_p
,
11734 ctx
? NULL
: &wi
, NULL
)))
11735 gimple_regimplify_operands (cond_stmt
, gsi_p
);
11739 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
11741 case GIMPLE_EH_FILTER
:
11742 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
11745 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
11746 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
11748 case GIMPLE_TRANSACTION
:
11749 lower_omp (gimple_transaction_body_ptr (
11750 as_a
<gtransaction
*> (stmt
)),
11754 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
11756 case GIMPLE_OMP_PARALLEL
:
11757 case GIMPLE_OMP_TASK
:
11758 ctx
= maybe_lookup_ctx (stmt
);
11760 if (ctx
->cancellable
)
11761 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11762 lower_omp_taskreg (gsi_p
, ctx
);
11764 case GIMPLE_OMP_FOR
:
11765 ctx
= maybe_lookup_ctx (stmt
);
11767 if (ctx
->cancellable
)
11768 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11769 lower_omp_for (gsi_p
, ctx
);
11771 case GIMPLE_OMP_SECTIONS
:
11772 ctx
= maybe_lookup_ctx (stmt
);
11774 if (ctx
->cancellable
)
11775 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11776 lower_omp_sections (gsi_p
, ctx
);
11778 case GIMPLE_OMP_SINGLE
:
11779 ctx
= maybe_lookup_ctx (stmt
);
11781 lower_omp_single (gsi_p
, ctx
);
11783 case GIMPLE_OMP_MASTER
:
11784 ctx
= maybe_lookup_ctx (stmt
);
11786 lower_omp_master (gsi_p
, ctx
);
11788 case GIMPLE_OMP_TASKGROUP
:
11789 ctx
= maybe_lookup_ctx (stmt
);
11791 lower_omp_taskgroup (gsi_p
, ctx
);
11793 case GIMPLE_OMP_ORDERED
:
11794 ctx
= maybe_lookup_ctx (stmt
);
11796 lower_omp_ordered (gsi_p
, ctx
);
11798 case GIMPLE_OMP_CRITICAL
:
11799 ctx
= maybe_lookup_ctx (stmt
);
11801 lower_omp_critical (gsi_p
, ctx
);
11803 case GIMPLE_OMP_ATOMIC_LOAD
:
11804 if ((ctx
|| task_shared_vars
)
11805 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
11806 as_a
<gomp_atomic_load
*> (stmt
)),
11807 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
11808 gimple_regimplify_operands (stmt
, gsi_p
);
11810 case GIMPLE_OMP_TARGET
:
11811 ctx
= maybe_lookup_ctx (stmt
);
11813 lower_omp_target (gsi_p
, ctx
);
11815 case GIMPLE_OMP_TEAMS
:
11816 ctx
= maybe_lookup_ctx (stmt
);
11818 lower_omp_teams (gsi_p
, ctx
);
11822 call_stmt
= as_a
<gcall
*> (stmt
);
11823 fndecl
= gimple_call_fndecl (call_stmt
);
11825 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
11826 switch (DECL_FUNCTION_CODE (fndecl
))
11828 case BUILT_IN_GOMP_BARRIER
:
11832 case BUILT_IN_GOMP_CANCEL
:
11833 case BUILT_IN_GOMP_CANCELLATION_POINT
:
11836 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
11837 cctx
= cctx
->outer
;
11838 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
11839 if (!cctx
->cancellable
)
11841 if (DECL_FUNCTION_CODE (fndecl
)
11842 == BUILT_IN_GOMP_CANCELLATION_POINT
)
11844 stmt
= gimple_build_nop ();
11845 gsi_replace (gsi_p
, stmt
, false);
11849 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
11851 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
11852 gimple_call_set_fndecl (call_stmt
, fndecl
);
11853 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
11856 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
11857 gimple_call_set_lhs (call_stmt
, lhs
);
11858 tree fallthru_label
;
11859 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
11861 g
= gimple_build_label (fallthru_label
);
11862 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11863 g
= gimple_build_cond (NE_EXPR
, lhs
,
11864 fold_convert (TREE_TYPE (lhs
),
11865 boolean_false_node
),
11866 cctx
->cancel_label
, fallthru_label
);
11867 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11874 if ((ctx
|| task_shared_vars
)
11875 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
11878 /* Just remove clobbers, this should happen only if we have
11879 "privatized" local addressable variables in SIMD regions,
11880 the clobber isn't needed in that case and gimplifying address
11881 of the ARRAY_REF into a pointer and creating MEM_REF based
11882 clobber would create worse code than we get with the clobber
11884 if (gimple_clobber_p (stmt
))
11886 gsi_replace (gsi_p
, gimple_build_nop (), true);
11889 gimple_regimplify_operands (stmt
, gsi_p
);
11896 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
11898 location_t saved_location
= input_location
;
11899 gimple_stmt_iterator gsi
;
11900 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11901 lower_omp_1 (&gsi
, ctx
);
11902 /* During gimplification, we haven't folded statments inside offloading
11903 regions (gimplify.c:maybe_fold_stmt); do that now. */
11904 if (target_nesting_level
)
11905 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11907 input_location
= saved_location
;
11910 /* Main entry point. */
11912 static unsigned int
11913 execute_lower_omp (void)
11919 /* This pass always runs, to provide PROP_gimple_lomp.
11920 But often, there is nothing to do. */
11921 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
11922 && flag_openmp_simd
== 0)
11925 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
11926 delete_omp_context
);
11928 body
= gimple_body (current_function_decl
);
11929 scan_omp (&body
, NULL
);
11930 gcc_assert (taskreg_nesting_level
== 0);
11931 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
11932 finish_taskreg_scan (ctx
);
11933 taskreg_contexts
.release ();
11935 if (all_contexts
->root
)
11937 if (task_shared_vars
)
11938 push_gimplify_context ();
11939 lower_omp (&body
, NULL
);
11940 if (task_shared_vars
)
11941 pop_gimplify_context (NULL
);
11946 splay_tree_delete (all_contexts
);
11947 all_contexts
= NULL
;
11949 BITMAP_FREE (task_shared_vars
);
11955 const pass_data pass_data_lower_omp
=
11957 GIMPLE_PASS
, /* type */
11958 "omplower", /* name */
11959 OPTGROUP_NONE
, /* optinfo_flags */
11960 TV_NONE
, /* tv_id */
11961 PROP_gimple_any
, /* properties_required */
11962 PROP_gimple_lomp
, /* properties_provided */
11963 0, /* properties_destroyed */
11964 0, /* todo_flags_start */
11965 0, /* todo_flags_finish */
11968 class pass_lower_omp
: public gimple_opt_pass
11971 pass_lower_omp (gcc::context
*ctxt
)
11972 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
11975 /* opt_pass methods: */
11976 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
11978 }; // class pass_lower_omp
11980 } // anon namespace
11983 make_pass_lower_omp (gcc::context
*ctxt
)
11985 return new pass_lower_omp (ctxt
);
11988 /* The following is a utility to diagnose structured block violations.
11989 It is not part of the "omplower" pass, as that's invoked too late. It
11990 should be invoked by the respective front ends after gimplification. */
11992 static splay_tree all_labels
;
11994 /* Check for mismatched contexts and generate an error if needed. Return
11995 true if an error is detected. */
11998 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
11999 gimple branch_ctx
, gimple label_ctx
)
12001 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
12002 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
12004 if (label_ctx
== branch_ctx
)
12007 const char* kind
= NULL
;
12012 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
12013 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
12015 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
12016 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
12017 kind
= "Cilk Plus";
12021 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
12022 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
12024 gcc_checking_assert (kind
== NULL
);
12030 gcc_checking_assert (flag_openmp
);
12035 Previously we kept track of the label's entire context in diagnose_sb_[12]
12036 so we could traverse it and issue a correct "exit" or "enter" error
12037 message upon a structured block violation.
12039 We built the context by building a list with tree_cons'ing, but there is
12040 no easy counterpart in gimple tuples. It seems like far too much work
12041 for issuing exit/enter error messages. If someone really misses the
12042 distinct error message... patches welcome.
12046 /* Try to avoid confusing the user by producing and error message
12047 with correct "exit" or "enter" verbiage. We prefer "exit"
12048 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12049 if (branch_ctx
== NULL
)
12055 if (TREE_VALUE (label_ctx
) == branch_ctx
)
12060 label_ctx
= TREE_CHAIN (label_ctx
);
12065 error ("invalid exit from %s structured block", kind
);
12067 error ("invalid entry to %s structured block", kind
);
12070 /* If it's obvious we have an invalid entry, be specific about the error. */
12071 if (branch_ctx
== NULL
)
12072 error ("invalid entry to %s structured block", kind
);
12075 /* Otherwise, be vague and lazy, but efficient. */
12076 error ("invalid branch to/from %s structured block", kind
);
12079 gsi_replace (gsi_p
, gimple_build_nop (), false);
12083 /* Pass 1: Create a minimal tree of structured blocks, and record
12084 where each label is found. */
12087 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12088 struct walk_stmt_info
*wi
)
12090 gimple context
= (gimple
) wi
->info
;
12091 gimple inner_context
;
12092 gimple stmt
= gsi_stmt (*gsi_p
);
12094 *handled_ops_p
= true;
12096 switch (gimple_code (stmt
))
12100 case GIMPLE_OMP_PARALLEL
:
12101 case GIMPLE_OMP_TASK
:
12102 case GIMPLE_OMP_SECTIONS
:
12103 case GIMPLE_OMP_SINGLE
:
12104 case GIMPLE_OMP_SECTION
:
12105 case GIMPLE_OMP_MASTER
:
12106 case GIMPLE_OMP_ORDERED
:
12107 case GIMPLE_OMP_CRITICAL
:
12108 case GIMPLE_OMP_TARGET
:
12109 case GIMPLE_OMP_TEAMS
:
12110 case GIMPLE_OMP_TASKGROUP
:
12111 /* The minimal context here is just the current OMP construct. */
12112 inner_context
= stmt
;
12113 wi
->info
= inner_context
;
12114 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12115 wi
->info
= context
;
12118 case GIMPLE_OMP_FOR
:
12119 inner_context
= stmt
;
12120 wi
->info
= inner_context
;
12121 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12123 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
12124 diagnose_sb_1
, NULL
, wi
);
12125 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12126 wi
->info
= context
;
12130 splay_tree_insert (all_labels
,
12131 (splay_tree_key
) gimple_label_label (
12132 as_a
<glabel
*> (stmt
)),
12133 (splay_tree_value
) context
);
12143 /* Pass 2: Check each branch and see if its context differs from that of
12144 the destination label's context. */
12147 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12148 struct walk_stmt_info
*wi
)
12150 gimple context
= (gimple
) wi
->info
;
12152 gimple stmt
= gsi_stmt (*gsi_p
);
12154 *handled_ops_p
= true;
12156 switch (gimple_code (stmt
))
12160 case GIMPLE_OMP_PARALLEL
:
12161 case GIMPLE_OMP_TASK
:
12162 case GIMPLE_OMP_SECTIONS
:
12163 case GIMPLE_OMP_SINGLE
:
12164 case GIMPLE_OMP_SECTION
:
12165 case GIMPLE_OMP_MASTER
:
12166 case GIMPLE_OMP_ORDERED
:
12167 case GIMPLE_OMP_CRITICAL
:
12168 case GIMPLE_OMP_TARGET
:
12169 case GIMPLE_OMP_TEAMS
:
12170 case GIMPLE_OMP_TASKGROUP
:
12172 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12173 wi
->info
= context
;
12176 case GIMPLE_OMP_FOR
:
12178 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12180 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
12181 diagnose_sb_2
, NULL
, wi
);
12182 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12183 wi
->info
= context
;
12188 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
12189 tree lab
= gimple_cond_true_label (cond_stmt
);
12192 n
= splay_tree_lookup (all_labels
,
12193 (splay_tree_key
) lab
);
12194 diagnose_sb_0 (gsi_p
, context
,
12195 n
? (gimple
) n
->value
: NULL
);
12197 lab
= gimple_cond_false_label (cond_stmt
);
12200 n
= splay_tree_lookup (all_labels
,
12201 (splay_tree_key
) lab
);
12202 diagnose_sb_0 (gsi_p
, context
,
12203 n
? (gimple
) n
->value
: NULL
);
12210 tree lab
= gimple_goto_dest (stmt
);
12211 if (TREE_CODE (lab
) != LABEL_DECL
)
12214 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12215 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
12219 case GIMPLE_SWITCH
:
12221 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
12223 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
12225 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
12226 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12227 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
12233 case GIMPLE_RETURN
:
12234 diagnose_sb_0 (gsi_p
, context
, NULL
);
12244 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12247 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
12250 gimple last
= last_stmt (bb
);
12251 enum gimple_code code
= gimple_code (last
);
12252 struct omp_region
*cur_region
= *region
;
12253 bool fallthru
= false;
12257 case GIMPLE_OMP_PARALLEL
:
12258 case GIMPLE_OMP_TASK
:
12259 case GIMPLE_OMP_FOR
:
12260 case GIMPLE_OMP_SINGLE
:
12261 case GIMPLE_OMP_TEAMS
:
12262 case GIMPLE_OMP_MASTER
:
12263 case GIMPLE_OMP_TASKGROUP
:
12264 case GIMPLE_OMP_ORDERED
:
12265 case GIMPLE_OMP_CRITICAL
:
12266 case GIMPLE_OMP_SECTION
:
12267 cur_region
= new_omp_region (bb
, code
, cur_region
);
12271 case GIMPLE_OMP_TARGET
:
12272 cur_region
= new_omp_region (bb
, code
, cur_region
);
12274 switch (gimple_omp_target_kind (last
))
12276 case GF_OMP_TARGET_KIND_REGION
:
12277 case GF_OMP_TARGET_KIND_DATA
:
12278 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12279 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12280 case GF_OMP_TARGET_KIND_OACC_DATA
:
12282 case GF_OMP_TARGET_KIND_UPDATE
:
12283 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12284 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12285 cur_region
= cur_region
->outer
;
12288 gcc_unreachable ();
12292 case GIMPLE_OMP_SECTIONS
:
12293 cur_region
= new_omp_region (bb
, code
, cur_region
);
12297 case GIMPLE_OMP_SECTIONS_SWITCH
:
12301 case GIMPLE_OMP_ATOMIC_LOAD
:
12302 case GIMPLE_OMP_ATOMIC_STORE
:
12306 case GIMPLE_OMP_RETURN
:
12307 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12308 somewhere other than the next block. This will be
12310 cur_region
->exit
= bb
;
12311 if (cur_region
->type
== GIMPLE_OMP_TASK
)
12312 /* Add an edge corresponding to not scheduling the task
12314 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
12315 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
12316 cur_region
= cur_region
->outer
;
12319 case GIMPLE_OMP_CONTINUE
:
12320 cur_region
->cont
= bb
;
12321 switch (cur_region
->type
)
12323 case GIMPLE_OMP_FOR
:
12324 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12325 succs edges as abnormal to prevent splitting
12327 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
12328 /* Make the loopback edge. */
12329 make_edge (bb
, single_succ (cur_region
->entry
),
12332 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12333 corresponds to the case that the body of the loop
12334 is not executed at all. */
12335 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
12336 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
12340 case GIMPLE_OMP_SECTIONS
:
12341 /* Wire up the edges into and out of the nested sections. */
12343 basic_block switch_bb
= single_succ (cur_region
->entry
);
12345 struct omp_region
*i
;
12346 for (i
= cur_region
->inner
; i
; i
= i
->next
)
12348 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
12349 make_edge (switch_bb
, i
->entry
, 0);
12350 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
12353 /* Make the loopback edge to the block with
12354 GIMPLE_OMP_SECTIONS_SWITCH. */
12355 make_edge (bb
, switch_bb
, 0);
12357 /* Make the edge from the switch to exit. */
12358 make_edge (switch_bb
, bb
->next_bb
, 0);
12363 case GIMPLE_OMP_TASK
:
12368 gcc_unreachable ();
12373 gcc_unreachable ();
12376 if (*region
!= cur_region
)
12378 *region
= cur_region
;
12380 *region_idx
= cur_region
->entry
->index
;
12388 static unsigned int
12389 diagnose_omp_structured_block_errors (void)
12391 struct walk_stmt_info wi
;
12392 gimple_seq body
= gimple_body (current_function_decl
);
12394 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
12396 memset (&wi
, 0, sizeof (wi
));
12397 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
12399 memset (&wi
, 0, sizeof (wi
));
12400 wi
.want_locations
= true;
12401 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
12403 gimple_set_body (current_function_decl
, body
);
12405 splay_tree_delete (all_labels
);
12413 const pass_data pass_data_diagnose_omp_blocks
=
12415 GIMPLE_PASS
, /* type */
12416 "*diagnose_omp_blocks", /* name */
12417 OPTGROUP_NONE
, /* optinfo_flags */
12418 TV_NONE
, /* tv_id */
12419 PROP_gimple_any
, /* properties_required */
12420 0, /* properties_provided */
12421 0, /* properties_destroyed */
12422 0, /* todo_flags_start */
12423 0, /* todo_flags_finish */
12426 class pass_diagnose_omp_blocks
: public gimple_opt_pass
12429 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12430 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
12433 /* opt_pass methods: */
12434 virtual bool gate (function
*)
12436 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
12438 virtual unsigned int execute (function
*)
12440 return diagnose_omp_structured_block_errors ();
12443 }; // class pass_diagnose_omp_blocks
12445 } // anon namespace
12448 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12450 return new pass_diagnose_omp_blocks (ctxt
);
12453 /* SIMD clone supporting code. */
12455 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12456 of arguments to reserve space for. */
12458 static struct cgraph_simd_clone
*
12459 simd_clone_struct_alloc (int nargs
)
12461 struct cgraph_simd_clone
*clone_info
;
12462 size_t len
= (sizeof (struct cgraph_simd_clone
)
12463 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
12464 clone_info
= (struct cgraph_simd_clone
*)
12465 ggc_internal_cleared_alloc (len
);
12469 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12472 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
12473 struct cgraph_simd_clone
*from
)
12475 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
12476 + ((from
->nargs
- from
->inbranch
)
12477 * sizeof (struct cgraph_simd_clone_arg
))));
12480 /* Return vector of parameter types of function FNDECL. This uses
12481 TYPE_ARG_TYPES if available, otherwise falls back to types of
12482 DECL_ARGUMENTS types. */
12485 simd_clone_vector_of_formal_parm_types (tree fndecl
)
12487 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
12488 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
12489 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
12492 FOR_EACH_VEC_ELT (args
, i
, arg
)
12493 args
[i
] = TREE_TYPE (args
[i
]);
12497 /* Given a simd function in NODE, extract the simd specific
12498 information from the OMP clauses passed in CLAUSES, and return
12499 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12500 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12501 otherwise set to FALSE. */
12503 static struct cgraph_simd_clone
*
12504 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
12505 bool *inbranch_specified
)
12507 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12510 *inbranch_specified
= false;
12512 n
= args
.length ();
12513 if (n
> 0 && args
.last () == void_type_node
)
12516 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12517 be cloned have a distinctive artificial label in addition to "omp
12521 && lookup_attribute ("cilk simd function",
12522 DECL_ATTRIBUTES (node
->decl
)));
12524 /* Allocate one more than needed just in case this is an in-branch
12525 clone which will require a mask argument. */
12526 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
12527 clone_info
->nargs
= n
;
12528 clone_info
->cilk_elemental
= cilk_clone
;
12535 clauses
= TREE_VALUE (clauses
);
12536 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
12539 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
12541 switch (OMP_CLAUSE_CODE (t
))
12543 case OMP_CLAUSE_INBRANCH
:
12544 clone_info
->inbranch
= 1;
12545 *inbranch_specified
= true;
12547 case OMP_CLAUSE_NOTINBRANCH
:
12548 clone_info
->inbranch
= 0;
12549 *inbranch_specified
= true;
12551 case OMP_CLAUSE_SIMDLEN
:
12552 clone_info
->simdlen
12553 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
12555 case OMP_CLAUSE_LINEAR
:
12557 tree decl
= OMP_CLAUSE_DECL (t
);
12558 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
12559 int argno
= TREE_INT_CST_LOW (decl
);
12560 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
12562 clone_info
->args
[argno
].arg_type
12563 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
12564 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12565 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
12566 && clone_info
->args
[argno
].linear_step
< n
);
12570 if (POINTER_TYPE_P (args
[argno
]))
12571 step
= fold_convert (ssizetype
, step
);
12572 if (!tree_fits_shwi_p (step
))
12574 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12575 "ignoring large linear step");
12579 else if (integer_zerop (step
))
12581 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12582 "ignoring zero linear step");
12588 clone_info
->args
[argno
].arg_type
12589 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
12590 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12595 case OMP_CLAUSE_UNIFORM
:
12597 tree decl
= OMP_CLAUSE_DECL (t
);
12598 int argno
= tree_to_uhwi (decl
);
12599 clone_info
->args
[argno
].arg_type
12600 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
12603 case OMP_CLAUSE_ALIGNED
:
12605 tree decl
= OMP_CLAUSE_DECL (t
);
12606 int argno
= tree_to_uhwi (decl
);
12607 clone_info
->args
[argno
].alignment
12608 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
12619 /* Given a SIMD clone in NODE, calculate the characteristic data
12620 type and return the coresponding type. The characteristic data
12621 type is computed as described in the Intel Vector ABI. */
12624 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
12625 struct cgraph_simd_clone
*clone_info
)
12627 tree type
= integer_type_node
;
12628 tree fndecl
= node
->decl
;
12630 /* a) For non-void function, the characteristic data type is the
12632 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
12633 type
= TREE_TYPE (TREE_TYPE (fndecl
));
12635 /* b) If the function has any non-uniform, non-linear parameters,
12636 then the characteristic data type is the type of the first
12640 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
12641 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
12642 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
12650 /* c) If the characteristic data type determined by a) or b) above
12651 is struct, union, or class type which is pass-by-value (except
12652 for the type that maps to the built-in complex data type), the
12653 characteristic data type is int. */
12654 if (RECORD_OR_UNION_TYPE_P (type
)
12655 && !aggregate_value_p (type
, NULL
)
12656 && TREE_CODE (type
) != COMPLEX_TYPE
)
12657 return integer_type_node
;
12659 /* d) If none of the above three classes is applicable, the
12660 characteristic data type is int. */
12664 /* e) For Intel Xeon Phi native and offload compilation, if the
12665 resulting characteristic data type is 8-bit or 16-bit integer
12666 data type, the characteristic data type is int. */
12667 /* Well, we don't handle Xeon Phi yet. */
12671 simd_clone_mangle (struct cgraph_node
*node
,
12672 struct cgraph_simd_clone
*clone_info
)
12674 char vecsize_mangle
= clone_info
->vecsize_mangle
;
12675 char mask
= clone_info
->inbranch
? 'M' : 'N';
12676 unsigned int simdlen
= clone_info
->simdlen
;
12680 gcc_assert (vecsize_mangle
&& simdlen
);
12682 pp_string (&pp
, "_ZGV");
12683 pp_character (&pp
, vecsize_mangle
);
12684 pp_character (&pp
, mask
);
12685 pp_decimal_int (&pp
, simdlen
);
12687 for (n
= 0; n
< clone_info
->nargs
; ++n
)
12689 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
12691 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
12692 pp_character (&pp
, 'u');
12693 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12695 gcc_assert (arg
.linear_step
!= 0);
12696 pp_character (&pp
, 'l');
12697 if (arg
.linear_step
> 1)
12698 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12699 else if (arg
.linear_step
< 0)
12701 pp_character (&pp
, 'n');
12702 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
12706 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
12708 pp_character (&pp
, 's');
12709 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12712 pp_character (&pp
, 'v');
12715 pp_character (&pp
, 'a');
12716 pp_decimal_int (&pp
, arg
.alignment
);
12720 pp_underscore (&pp
);
12721 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
12724 pp_string (&pp
, str
);
12725 str
= pp_formatted_text (&pp
);
12727 /* If there already is a SIMD clone with the same mangled name, don't
12728 add another one. This can happen e.g. for
12729 #pragma omp declare simd
12730 #pragma omp declare simd simdlen(8)
12731 int foo (int, int);
12732 if the simdlen is assumed to be 8 for the first one, etc. */
12733 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
12734 clone
= clone
->simdclone
->next_clone
)
12735 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
12739 return get_identifier (str
);
12742 /* Create a simd clone of OLD_NODE and return it. */
12744 static struct cgraph_node
*
12745 simd_clone_create (struct cgraph_node
*old_node
)
12747 struct cgraph_node
*new_node
;
12748 if (old_node
->definition
)
12750 if (!old_node
->has_gimple_body_p ())
12752 old_node
->get_body ();
12753 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
12759 tree old_decl
= old_node
->decl
;
12760 tree new_decl
= copy_node (old_node
->decl
);
12761 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
12762 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
12763 SET_DECL_RTL (new_decl
, NULL
);
12764 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
12765 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
12766 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
12767 symtab
->call_cgraph_insertion_hooks (new_node
);
12769 if (new_node
== NULL
)
12772 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
12774 /* The function cgraph_function_versioning () will force the new
12775 symbol local. Undo this, and inherit external visability from
12777 new_node
->local
.local
= old_node
->local
.local
;
12778 new_node
->externally_visible
= old_node
->externally_visible
;
12783 /* Adjust the return type of the given function to its appropriate
12784 vector counterpart. Returns a simd array to be used throughout the
12785 function as a return value. */
12788 simd_clone_adjust_return_type (struct cgraph_node
*node
)
12790 tree fndecl
= node
->decl
;
12791 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
12792 unsigned int veclen
;
12795 /* Adjust the function return type. */
12796 if (orig_rettype
== void_type_node
)
12798 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
12799 t
= TREE_TYPE (TREE_TYPE (fndecl
));
12800 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
12801 veclen
= node
->simdclone
->vecsize_int
;
12803 veclen
= node
->simdclone
->vecsize_float
;
12804 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
12805 if (veclen
> node
->simdclone
->simdlen
)
12806 veclen
= node
->simdclone
->simdlen
;
12807 if (POINTER_TYPE_P (t
))
12808 t
= pointer_sized_int_node
;
12809 if (veclen
== node
->simdclone
->simdlen
)
12810 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
12813 t
= build_vector_type (t
, veclen
);
12814 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
12816 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
12817 if (!node
->definition
)
12820 t
= DECL_RESULT (fndecl
);
12821 /* Adjust the DECL_RESULT. */
12822 gcc_assert (TREE_TYPE (t
) != void_type_node
);
12823 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
12826 tree atype
= build_array_type_nelts (orig_rettype
,
12827 node
->simdclone
->simdlen
);
12828 if (veclen
!= node
->simdclone
->simdlen
)
12829 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
12831 /* Set up a SIMD array to use as the return value. */
12832 tree retval
= create_tmp_var_raw (atype
, "retval");
12833 gimple_add_tmp_var (retval
);
12837 /* Each vector argument has a corresponding array to be used locally
12838 as part of the eventual loop. Create such temporary array and
12841 PREFIX is the prefix to be used for the temporary.
12843 TYPE is the inner element type.
12845 SIMDLEN is the number of elements. */
12848 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
12850 tree atype
= build_array_type_nelts (type
, simdlen
);
12851 tree avar
= create_tmp_var_raw (atype
, prefix
);
12852 gimple_add_tmp_var (avar
);
12856 /* Modify the function argument types to their corresponding vector
12857 counterparts if appropriate. Also, create one array for each simd
12858 argument to be used locally when using the function arguments as
12861 NODE is the function whose arguments are to be adjusted.
12863 Returns an adjustment vector that will be filled describing how the
12864 argument types will be adjusted. */
12866 static ipa_parm_adjustment_vec
12867 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
12870 ipa_parm_adjustment_vec adjustments
;
12872 if (node
->definition
)
12873 args
= ipa_get_vector_of_formal_parms (node
->decl
);
12875 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12876 adjustments
.create (args
.length ());
12877 unsigned i
, j
, veclen
;
12878 struct ipa_parm_adjustment adj
;
12879 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
12881 memset (&adj
, 0, sizeof (adj
));
12882 tree parm
= args
[i
];
12883 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
12884 adj
.base_index
= i
;
12887 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
12888 node
->simdclone
->args
[i
].orig_type
= parm_type
;
12890 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
12892 /* No adjustment necessary for scalar arguments. */
12893 adj
.op
= IPA_PARM_OP_COPY
;
12897 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
12898 veclen
= node
->simdclone
->vecsize_int
;
12900 veclen
= node
->simdclone
->vecsize_float
;
12901 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
12902 if (veclen
> node
->simdclone
->simdlen
)
12903 veclen
= node
->simdclone
->simdlen
;
12904 adj
.arg_prefix
= "simd";
12905 if (POINTER_TYPE_P (parm_type
))
12906 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12908 adj
.type
= build_vector_type (parm_type
, veclen
);
12909 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
12910 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12912 adjustments
.safe_push (adj
);
12915 memset (&adj
, 0, sizeof (adj
));
12916 adj
.op
= IPA_PARM_OP_NEW
;
12917 adj
.arg_prefix
= "simd";
12918 adj
.base_index
= i
;
12919 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
12923 if (node
->definition
)
12924 node
->simdclone
->args
[i
].simd_array
12925 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
12926 parm_type
, node
->simdclone
->simdlen
);
12928 adjustments
.safe_push (adj
);
12931 if (node
->simdclone
->inbranch
)
12934 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
12937 memset (&adj
, 0, sizeof (adj
));
12938 adj
.op
= IPA_PARM_OP_NEW
;
12939 adj
.arg_prefix
= "mask";
12941 adj
.base_index
= i
;
12942 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
12943 veclen
= node
->simdclone
->vecsize_int
;
12945 veclen
= node
->simdclone
->vecsize_float
;
12946 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
12947 if (veclen
> node
->simdclone
->simdlen
)
12948 veclen
= node
->simdclone
->simdlen
;
12949 if (POINTER_TYPE_P (base_type
))
12950 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12952 adj
.type
= build_vector_type (base_type
, veclen
);
12953 adjustments
.safe_push (adj
);
12955 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12956 adjustments
.safe_push (adj
);
12958 /* We have previously allocated one extra entry for the mask. Use
12960 struct cgraph_simd_clone
*sc
= node
->simdclone
;
12962 if (node
->definition
)
12964 sc
->args
[i
].orig_arg
12965 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
12966 sc
->args
[i
].simd_array
12967 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
12969 sc
->args
[i
].orig_type
= base_type
;
12970 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
12973 if (node
->definition
)
12974 ipa_modify_formal_parameters (node
->decl
, adjustments
);
12977 tree new_arg_types
= NULL_TREE
, new_reversed
;
12978 bool last_parm_void
= false;
12979 if (args
.length () > 0 && args
.last () == void_type_node
)
12980 last_parm_void
= true;
12982 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
12983 j
= adjustments
.length ();
12984 for (i
= 0; i
< j
; i
++)
12986 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
12988 if (adj
->op
== IPA_PARM_OP_COPY
)
12989 ptype
= args
[adj
->base_index
];
12992 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
12994 new_reversed
= nreverse (new_arg_types
);
12995 if (last_parm_void
)
12998 TREE_CHAIN (new_arg_types
) = void_list_node
;
13000 new_reversed
= void_list_node
;
13003 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
13004 TYPE_ARG_TYPES (new_type
) = new_reversed
;
13005 TREE_TYPE (node
->decl
) = new_type
;
13007 adjustments
.release ();
13010 return adjustments
;
13013 /* Initialize and copy the function arguments in NODE to their
13014 corresponding local simd arrays. Returns a fresh gimple_seq with
13015 the instruction sequence generated. */
13018 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
13019 ipa_parm_adjustment_vec adjustments
)
13021 gimple_seq seq
= NULL
;
13022 unsigned i
= 0, j
= 0, k
;
13024 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
13026 arg
= DECL_CHAIN (arg
), i
++, j
++)
13028 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
13031 node
->simdclone
->args
[i
].vector_arg
= arg
;
13033 tree array
= node
->simdclone
->args
[i
].simd_array
;
13034 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
13036 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
13037 tree ptr
= build_fold_addr_expr (array
);
13038 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
13039 build_int_cst (ptype
, 0));
13040 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
13041 gimplify_and_add (t
, &seq
);
13045 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
13046 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
13047 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
13049 tree ptr
= build_fold_addr_expr (array
);
13053 arg
= DECL_CHAIN (arg
);
13057 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
13058 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
13059 build_int_cst (ptype
, k
* elemsize
));
13060 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
13061 gimplify_and_add (t
, &seq
);
13068 /* Callback info for ipa_simd_modify_stmt_ops below. */
13070 struct modify_stmt_info
{
13071 ipa_parm_adjustment_vec adjustments
;
13073 /* True if the parent statement was modified by
13074 ipa_simd_modify_stmt_ops. */
13078 /* Callback for walk_gimple_op.
13080 Adjust operands from a given statement as specified in the
13081 adjustments vector in the callback data. */
13084 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
13086 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13087 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
13088 tree
*orig_tp
= tp
;
13089 if (TREE_CODE (*tp
) == ADDR_EXPR
)
13090 tp
= &TREE_OPERAND (*tp
, 0);
13091 struct ipa_parm_adjustment
*cand
= NULL
;
13092 if (TREE_CODE (*tp
) == PARM_DECL
)
13093 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
13097 *walk_subtrees
= 0;
13100 tree repl
= NULL_TREE
;
13102 repl
= unshare_expr (cand
->new_decl
);
13107 *walk_subtrees
= 0;
13108 bool modified
= info
->modified
;
13109 info
->modified
= false;
13110 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
13111 if (!info
->modified
)
13113 info
->modified
= modified
;
13116 info
->modified
= modified
;
13125 repl
= build_fold_addr_expr (repl
);
13127 if (is_gimple_debug (info
->stmt
))
13129 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
13130 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
13131 DECL_ARTIFICIAL (vexpr
) = 1;
13132 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
13133 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
13138 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
13139 repl
= gimple_assign_lhs (stmt
);
13141 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
13142 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13145 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
13147 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
13153 info
->modified
= true;
13157 /* Traverse the function body and perform all modifications as
13158 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13159 modified such that the replacement/reduction value will now be an
13160 offset into the corresponding simd_array.
13162 This function will replace all function argument uses with their
13163 corresponding simd array elements, and ajust the return values
13167 ipa_simd_modify_function_body (struct cgraph_node
*node
,
13168 ipa_parm_adjustment_vec adjustments
,
13169 tree retval_array
, tree iter
)
13172 unsigned int i
, j
, l
;
13174 /* Re-use the adjustments array, but this time use it to replace
13175 every function argument use to an offset into the corresponding
13177 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
13179 if (!node
->simdclone
->args
[i
].vector_arg
)
13182 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
13183 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
13184 adjustments
[j
].new_decl
13185 = build4 (ARRAY_REF
,
13187 node
->simdclone
->args
[i
].simd_array
,
13189 NULL_TREE
, NULL_TREE
);
13190 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
13191 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
13192 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
13195 l
= adjustments
.length ();
13196 for (i
= 1; i
< num_ssa_names
; i
++)
13198 tree name
= ssa_name (i
);
13200 && SSA_NAME_VAR (name
)
13201 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
13203 for (j
= 0; j
< l
; j
++)
13204 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
13205 && adjustments
[j
].new_decl
)
13208 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
13211 = copy_var_decl (adjustments
[j
].base
,
13212 DECL_NAME (adjustments
[j
].base
),
13213 TREE_TYPE (adjustments
[j
].base
));
13214 adjustments
[j
].new_ssa_base
= base_var
;
13217 base_var
= adjustments
[j
].new_ssa_base
;
13218 if (SSA_NAME_IS_DEFAULT_DEF (name
))
13220 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13221 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
13222 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
13223 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
13224 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13225 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
13226 gimple stmt
= gimple_build_assign (name
, new_decl
);
13227 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13230 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13235 struct modify_stmt_info info
;
13236 info
.adjustments
= adjustments
;
13238 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
13240 gimple_stmt_iterator gsi
;
13242 gsi
= gsi_start_bb (bb
);
13243 while (!gsi_end_p (gsi
))
13245 gimple stmt
= gsi_stmt (gsi
);
13247 struct walk_stmt_info wi
;
13249 memset (&wi
, 0, sizeof (wi
));
13250 info
.modified
= false;
13252 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
13254 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
13256 tree retval
= gimple_return_retval (return_stmt
);
13259 gsi_remove (&gsi
, true);
13263 /* Replace `return foo' with `retval_array[iter] = foo'. */
13264 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
13265 retval_array
, iter
, NULL
, NULL
);
13266 stmt
= gimple_build_assign (ref
, retval
);
13267 gsi_replace (&gsi
, stmt
, true);
13268 info
.modified
= true;
13273 update_stmt (stmt
);
13274 if (maybe_clean_eh_stmt (stmt
))
13275 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
13282 /* Adjust the argument types in NODE to their appropriate vector
13286 simd_clone_adjust (struct cgraph_node
*node
)
13288 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
13290 targetm
.simd_clone
.adjust (node
);
13292 tree retval
= simd_clone_adjust_return_type (node
);
13293 ipa_parm_adjustment_vec adjustments
13294 = simd_clone_adjust_argument_types (node
);
13296 push_gimplify_context ();
13298 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
13300 /* Adjust all uses of vector arguments accordingly. Adjust all
13301 return values accordingly. */
13302 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
13303 tree iter1
= make_ssa_name (iter
);
13304 tree iter2
= make_ssa_name (iter
);
13305 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
13307 /* Initialize the iteration variable. */
13308 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13309 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
13310 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
13311 /* Insert the SIMD array and iv initialization at function
13313 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
13315 pop_gimplify_context (NULL
);
13317 /* Create a new BB right before the original exit BB, to hold the
13318 iteration increment and the condition/branch. */
13319 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
13320 basic_block incr_bb
= create_empty_bb (orig_exit
);
13321 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
13322 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13323 flag. Set it now to be a FALLTHRU_EDGE. */
13324 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
13325 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
13326 for (unsigned i
= 0;
13327 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
13329 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
13330 redirect_edge_succ (e
, incr_bb
);
13332 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
13333 e
->probability
= REG_BR_PROB_BASE
;
13334 gsi
= gsi_last_bb (incr_bb
);
13335 gimple g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
13336 build_int_cst (unsigned_type_node
, 1));
13337 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13339 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13340 struct loop
*loop
= alloc_loop ();
13341 cfun
->has_force_vectorize_loops
= true;
13342 loop
->safelen
= node
->simdclone
->simdlen
;
13343 loop
->force_vectorize
= true;
13344 loop
->header
= body_bb
;
13346 /* Branch around the body if the mask applies. */
13347 if (node
->simdclone
->inbranch
)
13349 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
13351 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
13352 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
13353 tree aref
= build4 (ARRAY_REF
,
13354 TREE_TYPE (TREE_TYPE (mask_array
)),
13357 g
= gimple_build_assign (mask
, aref
);
13358 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13359 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
13360 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
13362 aref
= build1 (VIEW_CONVERT_EXPR
,
13363 build_nonstandard_integer_type (bitsize
, 0), mask
);
13364 mask
= make_ssa_name (TREE_TYPE (aref
));
13365 g
= gimple_build_assign (mask
, aref
);
13366 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13369 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
13371 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13372 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
13373 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
13376 /* Generate the condition. */
13377 g
= gimple_build_cond (LT_EXPR
,
13379 build_int_cst (unsigned_type_node
,
13380 node
->simdclone
->simdlen
),
13382 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13383 e
= split_block (incr_bb
, gsi_stmt (gsi
));
13384 basic_block latch_bb
= e
->dest
;
13385 basic_block new_exit_bb
;
13386 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
13387 loop
->latch
= latch_bb
;
13389 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
13391 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
13392 /* The successor of incr_bb is already pointing to latch_bb; just
13394 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13395 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
13397 gphi
*phi
= create_phi_node (iter1
, body_bb
);
13398 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
13399 edge latch_edge
= single_succ_edge (latch_bb
);
13400 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
13402 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13404 /* Generate the new return. */
13405 gsi
= gsi_last_bb (new_exit_bb
);
13407 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
13408 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
13409 retval
= TREE_OPERAND (retval
, 0);
13412 retval
= build1 (VIEW_CONVERT_EXPR
,
13413 TREE_TYPE (TREE_TYPE (node
->decl
)),
13415 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
13416 false, GSI_CONTINUE_LINKING
);
13418 g
= gimple_build_return (retval
);
13419 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13421 /* Handle aligned clauses by replacing default defs of the aligned
13422 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13423 lhs. Handle linear by adding PHIs. */
13424 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
13425 if (node
->simdclone
->args
[i
].alignment
13426 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13427 && (node
->simdclone
->args
[i
].alignment
13428 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
13429 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13432 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
13433 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13434 tree def
= ssa_default_def (cfun
, orig_arg
);
13435 if (def
&& !has_zero_uses (def
))
13437 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
13438 gimple_seq seq
= NULL
;
13439 bool need_cvt
= false;
13441 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
13443 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
13446 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
13447 gimple_call_set_lhs (g
, t
);
13448 gimple_seq_add_stmt_without_update (&seq
, g
);
13451 t
= make_ssa_name (orig_arg
);
13452 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
13453 gimple_seq_add_stmt_without_update (&seq
, g
);
13455 gsi_insert_seq_on_edge_immediate
13456 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
13458 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13459 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
13461 node
->create_edge (cgraph_node::get_create (fn
),
13462 call
, entry_bb
->count
, freq
);
13464 imm_use_iterator iter
;
13465 use_operand_p use_p
;
13467 tree repl
= gimple_get_lhs (g
);
13468 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13469 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
13472 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13473 SET_USE (use_p
, repl
);
13476 else if (node
->simdclone
->args
[i
].arg_type
13477 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
13479 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13480 tree def
= ssa_default_def (cfun
, orig_arg
);
13481 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13482 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
13483 if (def
&& !has_zero_uses (def
))
13485 iter1
= make_ssa_name (orig_arg
);
13486 iter2
= make_ssa_name (orig_arg
);
13487 phi
= create_phi_node (iter1
, body_bb
);
13488 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
13489 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13490 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13491 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
13492 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13493 ? TREE_TYPE (orig_arg
) : sizetype
;
13495 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
13496 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
13497 gsi
= gsi_last_bb (incr_bb
);
13498 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13500 imm_use_iterator iter
;
13501 use_operand_p use_p
;
13503 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13504 if (use_stmt
== phi
)
13507 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13508 SET_USE (use_p
, iter1
);
13512 calculate_dominance_info (CDI_DOMINATORS
);
13513 add_loop (loop
, loop
->header
->loop_father
);
13514 update_ssa (TODO_update_ssa
);
13519 /* If the function in NODE is tagged as an elemental SIMD function,
13520 create the appropriate SIMD clones. */
13523 expand_simd_clones (struct cgraph_node
*node
)
13525 tree attr
= lookup_attribute ("omp declare simd",
13526 DECL_ATTRIBUTES (node
->decl
));
13527 if (attr
== NULL_TREE
13528 || node
->global
.inlined_to
13529 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
13533 #pragma omp declare simd
13535 in C, there we don't know the argument types at all. */
13536 if (!node
->definition
13537 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
13542 /* Start with parsing the "omp declare simd" attribute(s). */
13543 bool inbranch_clause_specified
;
13544 struct cgraph_simd_clone
*clone_info
13545 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
13546 &inbranch_clause_specified
);
13547 if (clone_info
== NULL
)
13550 int orig_simdlen
= clone_info
->simdlen
;
13551 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
13552 /* The target can return 0 (no simd clones should be created),
13553 1 (just one ISA of simd clones should be created) or higher
13554 count of ISA variants. In that case, clone_info is initialized
13555 for the first ISA variant. */
13557 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
13562 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13563 also create one inbranch and one !inbranch clone of it. */
13564 for (int i
= 0; i
< count
* 2; i
++)
13566 struct cgraph_simd_clone
*clone
= clone_info
;
13567 if (inbranch_clause_specified
&& (i
& 1) != 0)
13572 clone
= simd_clone_struct_alloc (clone_info
->nargs
13574 simd_clone_struct_copy (clone
, clone_info
);
13575 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13576 and simd_clone_adjust_argument_types did to the first
13578 clone
->nargs
-= clone_info
->inbranch
;
13579 clone
->simdlen
= orig_simdlen
;
13580 /* And call the target hook again to get the right ISA. */
13581 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
13585 clone
->inbranch
= 1;
13588 /* simd_clone_mangle might fail if such a clone has been created
13590 tree id
= simd_clone_mangle (node
, clone
);
13591 if (id
== NULL_TREE
)
13594 /* Only when we are sure we want to create the clone actually
13595 clone the function (or definitions) or create another
13596 extern FUNCTION_DECL (for prototypes without definitions). */
13597 struct cgraph_node
*n
= simd_clone_create (node
);
13601 n
->simdclone
= clone
;
13602 clone
->origin
= node
;
13603 clone
->next_clone
= NULL
;
13604 if (node
->simd_clones
== NULL
)
13606 clone
->prev_clone
= n
;
13607 node
->simd_clones
= n
;
13611 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
13612 clone
->prev_clone
->simdclone
->next_clone
= n
;
13613 node
->simd_clones
->simdclone
->prev_clone
= n
;
13615 symtab
->change_decl_assembler_name (n
->decl
, id
);
13616 /* And finally adjust the return type, parameters and for
13617 definitions also function body. */
13618 if (node
->definition
)
13619 simd_clone_adjust (n
);
13622 simd_clone_adjust_return_type (n
);
13623 simd_clone_adjust_argument_types (n
);
13627 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
13630 /* Entry point for IPA simd clone creation pass. */
13632 static unsigned int
13633 ipa_omp_simd_clone (void)
13635 struct cgraph_node
*node
;
13636 FOR_EACH_FUNCTION (node
)
13637 expand_simd_clones (node
);
13643 const pass_data pass_data_omp_simd_clone
=
13645 SIMPLE_IPA_PASS
, /* type */
13646 "simdclone", /* name */
13647 OPTGROUP_NONE
, /* optinfo_flags */
13648 TV_NONE
, /* tv_id */
13649 ( PROP_ssa
| PROP_cfg
), /* properties_required */
13650 0, /* properties_provided */
13651 0, /* properties_destroyed */
13652 0, /* todo_flags_start */
13653 0, /* todo_flags_finish */
13656 class pass_omp_simd_clone
: public simple_ipa_opt_pass
13659 pass_omp_simd_clone(gcc::context
*ctxt
)
13660 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
13663 /* opt_pass methods: */
13664 virtual bool gate (function
*);
13665 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
13669 pass_omp_simd_clone::gate (function
*)
13671 return ((flag_openmp
|| flag_openmp_simd
13673 || (in_lto_p
&& !flag_wpa
))
13674 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
13677 } // anon namespace
13679 simple_ipa_opt_pass
*
13680 make_pass_omp_simd_clone (gcc::context
*ctxt
)
13682 return new pass_omp_simd_clone (ctxt
);
13685 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
13686 adds their addresses and sizes to constructor-vector V_CTOR. */
13688 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
13689 vec
<constructor_elt
, va_gc
> *v_ctor
)
13691 unsigned len
= vec_safe_length (v_decls
);
13692 for (unsigned i
= 0; i
< len
; i
++)
13694 tree it
= (*v_decls
)[i
];
13695 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
13697 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
13699 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
13700 fold_convert (const_ptr_type_node
,
13701 DECL_SIZE_UNIT (it
)));
13705 /* Create new symbols containing (address, size) pairs for global variables,
13706 marked with "omp declare target" attribute, as well as addresses for the
13707 functions, which are outlined offloading regions. */
13709 omp_finish_file (void)
13711 unsigned num_funcs
= vec_safe_length (offload_funcs
);
13712 unsigned num_vars
= vec_safe_length (offload_vars
);
13714 if (num_funcs
== 0 && num_vars
== 0)
13717 if (targetm_common
.have_named_sections
)
13719 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
13720 vec_alloc (v_f
, num_funcs
);
13721 vec_alloc (v_v
, num_vars
* 2);
13723 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
13724 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
13726 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13728 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13730 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13731 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13732 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
13733 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
13734 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
13735 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
13736 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13737 get_identifier (".offload_func_table"),
13739 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13740 get_identifier (".offload_var_table"),
13742 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
13743 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
13744 otherwise a joint table in a binary will contain padding between
13745 tables from multiple object files. */
13746 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
13747 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
13748 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
13749 DECL_INITIAL (funcs_decl
) = ctor_f
;
13750 DECL_INITIAL (vars_decl
) = ctor_v
;
13751 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
13752 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
13754 varpool_node::finalize_decl (vars_decl
);
13755 varpool_node::finalize_decl (funcs_decl
);
13759 for (unsigned i
= 0; i
< num_funcs
; i
++)
13761 tree it
= (*offload_funcs
)[i
];
13762 targetm
.record_offload_symbol (it
);
13764 for (unsigned i
= 0; i
< num_vars
; i
++)
13766 tree it
= (*offload_vars
)[i
];
13767 targetm
.record_offload_symbol (it
);
13772 #include "gt-omp-low.h"