1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
32 #include "double-int.h"
39 #include "fold-const.h"
40 #include "stringpool.h"
41 #include "stor-layout.h"
44 #include "hard-reg-set.h"
46 #include "dominance.h"
49 #include "basic-block.h"
50 #include "tree-ssa-alias.h"
51 #include "internal-fn.h"
52 #include "gimple-fold.h"
53 #include "gimple-expr.h"
57 #include "gimple-iterator.h"
58 #include "gimplify-me.h"
59 #include "gimple-walk.h"
60 #include "tree-iterator.h"
61 #include "tree-inline.h"
62 #include "langhooks.h"
63 #include "diagnostic-core.h"
64 #include "gimple-ssa.h"
66 #include "plugin-api.h"
70 #include "tree-phinodes.h"
71 #include "ssa-iterators.h"
72 #include "tree-ssanames.h"
73 #include "tree-into-ssa.h"
76 #include "statistics.h"
78 #include "fixed-value.h"
79 #include "insn-config.h"
90 #include "tree-pass.h"
92 #include "splay-tree.h"
93 #include "insn-codes.h"
97 #include "common/common-target.h"
99 #include "gimple-low.h"
100 #include "tree-cfgcleanup.h"
101 #include "pretty-print.h"
102 #include "alloc-pool.h"
103 #include "symbol-summary.h"
104 #include "ipa-prop.h"
105 #include "tree-nested.h"
109 #include "lto-section-names.h"
110 #include "gomp-constants.h"
113 /* Lowering of OMP parallel and workshare constructs proceeds in two
114 phases. The first phase scans the function looking for OMP statements
115 and then for variables that must be replaced to satisfy data sharing
116 clauses. The second phase expands code for the constructs, as well as
117 re-gimplifying things when variables have been replaced with complex
120 Final code generation is done by pass_expand_omp. The flowgraph is
121 scanned for regions which are then moved to a new
122 function, to be invoked by the thread library, or offloaded. */
124 /* OMP region information. Every parallel and workshare
125 directive is enclosed between two markers, the OMP_* directive
126 and a corresponding OMP_RETURN statement. */
130 /* The enclosing region. */
131 struct omp_region
*outer
;
133 /* First child region. */
134 struct omp_region
*inner
;
136 /* Next peer region. */
137 struct omp_region
*next
;
139 /* Block containing the omp directive as its last stmt. */
142 /* Block containing the OMP_RETURN as its last stmt. */
145 /* Block containing the OMP_CONTINUE as its last stmt. */
148 /* If this is a combined parallel+workshare region, this is a list
149 of additional arguments needed by the combined parallel+workshare
151 vec
<tree
, va_gc
> *ws_args
;
153 /* The code for the omp directive of this region. */
154 enum gimple_code type
;
156 /* Schedule kind, only used for OMP_FOR type regions. */
157 enum omp_clause_schedule_kind sched_kind
;
159 /* True if this is a combined parallel+workshare region. */
160 bool is_combined_parallel
;
163 /* Levels of parallelism as defined by OpenACC. Increasing numbers
164 correspond to deeper loop nesting levels. */
166 #define MASK_WORKER 2
167 #define MASK_VECTOR 4
169 /* Context structure. Used to store information about each parallel
170 directive in the code. */
172 typedef struct omp_context
174 /* This field must be at the beginning, as we do "inheritance": Some
175 callback functions for tree-inline.c (e.g., omp_copy_decl)
176 receive a copy_body_data pointer that is up-casted to an
177 omp_context pointer. */
180 /* The tree of contexts corresponding to the encountered constructs. */
181 struct omp_context
*outer
;
184 /* Map variables to fields in a structure that allows communication
185 between sending and receiving threads. */
186 splay_tree field_map
;
191 /* These are used just by task contexts, if task firstprivate fn is
192 needed. srecord_type is used to communicate from the thread
193 that encountered the task construct to task firstprivate fn,
194 record_type is allocated by GOMP_task, initialized by task firstprivate
195 fn and passed to the task body fn. */
196 splay_tree sfield_map
;
199 /* A chain of variables to add to the top-level block surrounding the
200 construct. In the case of a parallel, this is in the child function. */
203 /* A map of reduction pointer variables. For accelerators, each
204 reduction variable is replaced with an array. Each thread, in turn,
205 is assigned to a slot on that array. */
206 splay_tree reduction_map
;
208 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
209 barriers should jump to during omplower pass. */
212 /* What to do with variables with implicitly determined sharing
214 enum omp_clause_default_kind default_kind
;
216 /* Nesting depth of this context. Used to beautify error messages re
217 invalid gotos. The outermost ctx is depth 1, with depth 0 being
218 reserved for the main body of the function. */
221 /* True if this parallel directive is nested within another. */
224 /* True if this construct can be cancelled. */
227 /* For OpenACC loops, a mask of gang, worker and vector used at
228 levels below this one. */
230 /* For OpenACC loops, a mask of gang, worker and vector used at
231 this level and above. For parallel and kernels clauses, a mask
232 indicating which of num_gangs/num_workers/num_vectors was used. */
236 /* A structure holding the elements of:
237 for (V = N1; V cond N2; V += STEP) [...] */
239 struct omp_for_data_loop
241 tree v
, n1
, n2
, step
;
242 enum tree_code cond_code
;
245 /* A structure describing the main elements of a parallel loop. */
249 struct omp_for_data_loop loop
;
254 bool have_nowait
, have_ordered
;
255 enum omp_clause_schedule_kind sched_kind
;
256 struct omp_for_data_loop
*loops
;
260 static splay_tree all_contexts
;
261 static int taskreg_nesting_level
;
262 static int target_nesting_level
;
263 static struct omp_region
*root_omp_region
;
264 static bitmap task_shared_vars
;
265 static vec
<omp_context
*> taskreg_contexts
;
267 static void scan_omp (gimple_seq
*, omp_context
*);
268 static tree
scan_omp_1_op (tree
*, int *, void *);
270 #define WALK_SUBSTMTS \
274 case GIMPLE_EH_FILTER: \
275 case GIMPLE_TRANSACTION: \
276 /* The sub-statements for these should be walked. */ \
277 *handled_ops_p = false; \
280 /* Helper function to get the name of the array containing the partial
281 reductions for OpenACC reductions. */
283 oacc_get_reduction_array_id (tree node
)
285 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
286 int len
= strlen ("OACC") + strlen (id
);
287 char *temp_name
= XALLOCAVEC (char, len
+ 1);
288 snprintf (temp_name
, len
+ 1, "OACC%s", id
);
289 return IDENTIFIER_POINTER (get_identifier (temp_name
));
292 /* Determine the number of threads OpenACC threads used to determine the
293 size of the array of partial reductions. Currently, this is num_gangs
294 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
295 because it is independed of the device used. */
298 oacc_max_threads (omp_context
*ctx
)
300 tree nthreads
, vector_length
, gangs
, clauses
;
302 gangs
= fold_convert (sizetype
, integer_one_node
);
303 vector_length
= gangs
;
305 /* The reduction clause may be nested inside a loop directive.
306 Scan for the innermost vector_length clause. */
307 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
309 if (gimple_code (oc
->stmt
) != GIMPLE_OMP_TARGET
310 || (gimple_omp_target_kind (oc
->stmt
)
311 != GF_OMP_TARGET_KIND_OACC_PARALLEL
))
314 clauses
= gimple_omp_target_clauses (oc
->stmt
);
316 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
318 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
320 OMP_CLAUSE_VECTOR_LENGTH_EXPR
323 vector_length
= fold_convert (sizetype
, integer_one_node
);
325 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
327 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
328 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
330 gangs
= fold_convert (sizetype
, integer_one_node
);
335 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
340 /* Holds offload tables with decls. */
341 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
343 /* Convenience function for calling scan_omp_1_op on tree operands. */
346 scan_omp_op (tree
*tp
, omp_context
*ctx
)
348 struct walk_stmt_info wi
;
350 memset (&wi
, 0, sizeof (wi
));
352 wi
.want_locations
= true;
354 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
357 static void lower_omp (gimple_seq
*, omp_context
*);
358 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
359 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
361 /* Find an OMP clause of type KIND within CLAUSES. */
364 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
366 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
367 if (OMP_CLAUSE_CODE (clauses
) == kind
)
373 /* Return true if CTX is for an omp parallel. */
376 is_parallel_ctx (omp_context
*ctx
)
378 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
382 /* Return true if CTX is for an omp task. */
385 is_task_ctx (omp_context
*ctx
)
387 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
391 /* Return true if CTX is for an omp parallel or omp task. */
394 is_taskreg_ctx (omp_context
*ctx
)
396 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
397 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
401 /* Return true if REGION is a combined parallel+workshare region. */
404 is_combined_parallel (struct omp_region
*region
)
406 return region
->is_combined_parallel
;
410 /* Extract the header elements of parallel loop FOR_STMT and store
414 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
415 struct omp_for_data_loop
*loops
)
417 tree t
, var
, *collapse_iter
, *collapse_count
;
418 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
419 struct omp_for_data_loop
*loop
;
421 struct omp_for_data_loop dummy_loop
;
422 location_t loc
= gimple_location (for_stmt
);
423 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
424 bool distribute
= gimple_omp_for_kind (for_stmt
)
425 == GF_OMP_FOR_KIND_DISTRIBUTE
;
427 fd
->for_stmt
= for_stmt
;
429 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
430 if (fd
->collapse
> 1)
433 fd
->loops
= &fd
->loop
;
435 fd
->have_nowait
= distribute
|| simd
;
436 fd
->have_ordered
= false;
437 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
438 fd
->chunk_size
= NULL_TREE
;
439 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
440 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
441 collapse_iter
= NULL
;
442 collapse_count
= NULL
;
444 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
445 switch (OMP_CLAUSE_CODE (t
))
447 case OMP_CLAUSE_NOWAIT
:
448 fd
->have_nowait
= true;
450 case OMP_CLAUSE_ORDERED
:
451 fd
->have_ordered
= true;
453 case OMP_CLAUSE_SCHEDULE
:
454 gcc_assert (!distribute
);
455 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
456 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
458 case OMP_CLAUSE_DIST_SCHEDULE
:
459 gcc_assert (distribute
);
460 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
462 case OMP_CLAUSE_COLLAPSE
:
463 if (fd
->collapse
> 1)
465 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
466 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
473 /* FIXME: for now map schedule(auto) to schedule(static).
474 There should be analysis to determine whether all iterations
475 are approximately the same amount of work (then schedule(static)
476 is best) or if it varies (then schedule(dynamic,N) is better). */
477 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
479 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
480 gcc_assert (fd
->chunk_size
== NULL
);
482 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
483 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
484 gcc_assert (fd
->chunk_size
== NULL
);
485 else if (fd
->chunk_size
== NULL
)
487 /* We only need to compute a default chunk size for ordered
488 static loops and dynamic loops. */
489 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
491 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
492 ? integer_zero_node
: integer_one_node
;
495 for (i
= 0; i
< fd
->collapse
; i
++)
497 if (fd
->collapse
== 1)
499 else if (loops
!= NULL
)
504 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
505 gcc_assert (SSA_VAR_P (loop
->v
));
506 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
507 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
508 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
509 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
511 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
512 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
513 switch (loop
->cond_code
)
519 gcc_assert (gimple_omp_for_kind (for_stmt
)
520 == GF_OMP_FOR_KIND_CILKSIMD
521 || (gimple_omp_for_kind (for_stmt
)
522 == GF_OMP_FOR_KIND_CILKFOR
));
525 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
526 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
528 loop
->n2
= fold_build2_loc (loc
,
529 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
530 build_int_cst (TREE_TYPE (loop
->n2
), 1));
531 loop
->cond_code
= LT_EXPR
;
534 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
535 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
537 loop
->n2
= fold_build2_loc (loc
,
538 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
539 build_int_cst (TREE_TYPE (loop
->n2
), 1));
540 loop
->cond_code
= GT_EXPR
;
546 t
= gimple_omp_for_incr (for_stmt
, i
);
547 gcc_assert (TREE_OPERAND (t
, 0) == var
);
548 switch (TREE_CODE (t
))
551 loop
->step
= TREE_OPERAND (t
, 1);
553 case POINTER_PLUS_EXPR
:
554 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
557 loop
->step
= TREE_OPERAND (t
, 1);
558 loop
->step
= fold_build1_loc (loc
,
559 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
567 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
568 && !fd
->have_ordered
))
570 if (fd
->collapse
== 1)
571 iter_type
= TREE_TYPE (loop
->v
);
573 || TYPE_PRECISION (iter_type
)
574 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
576 = build_nonstandard_integer_type
577 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
579 else if (iter_type
!= long_long_unsigned_type_node
)
581 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
582 iter_type
= long_long_unsigned_type_node
;
583 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
584 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
585 >= TYPE_PRECISION (iter_type
))
589 if (loop
->cond_code
== LT_EXPR
)
590 n
= fold_build2_loc (loc
,
591 PLUS_EXPR
, TREE_TYPE (loop
->v
),
592 loop
->n2
, loop
->step
);
595 if (TREE_CODE (n
) != INTEGER_CST
596 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
597 iter_type
= long_long_unsigned_type_node
;
599 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
600 > TYPE_PRECISION (iter_type
))
604 if (loop
->cond_code
== LT_EXPR
)
607 n2
= fold_build2_loc (loc
,
608 PLUS_EXPR
, TREE_TYPE (loop
->v
),
609 loop
->n2
, loop
->step
);
613 n1
= fold_build2_loc (loc
,
614 MINUS_EXPR
, TREE_TYPE (loop
->v
),
615 loop
->n2
, loop
->step
);
618 if (TREE_CODE (n1
) != INTEGER_CST
619 || TREE_CODE (n2
) != INTEGER_CST
620 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
621 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
622 iter_type
= long_long_unsigned_type_node
;
626 if (collapse_count
&& *collapse_count
== NULL
)
628 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
629 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
630 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
631 if (t
&& integer_zerop (t
))
632 count
= build_zero_cst (long_long_unsigned_type_node
);
633 else if ((i
== 0 || count
!= NULL_TREE
)
634 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
635 && TREE_CONSTANT (loop
->n1
)
636 && TREE_CONSTANT (loop
->n2
)
637 && TREE_CODE (loop
->step
) == INTEGER_CST
)
639 tree itype
= TREE_TYPE (loop
->v
);
641 if (POINTER_TYPE_P (itype
))
642 itype
= signed_type_for (itype
);
643 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
644 t
= fold_build2_loc (loc
,
646 fold_convert_loc (loc
, itype
, loop
->step
), t
);
647 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
648 fold_convert_loc (loc
, itype
, loop
->n2
));
649 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
650 fold_convert_loc (loc
, itype
, loop
->n1
));
651 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
652 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
653 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
654 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
655 fold_convert_loc (loc
, itype
,
658 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
659 fold_convert_loc (loc
, itype
, loop
->step
));
660 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
661 if (count
!= NULL_TREE
)
662 count
= fold_build2_loc (loc
,
663 MULT_EXPR
, long_long_unsigned_type_node
,
667 if (TREE_CODE (count
) != INTEGER_CST
)
670 else if (count
&& !integer_zerop (count
))
677 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
678 || fd
->have_ordered
))
680 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
681 iter_type
= long_long_unsigned_type_node
;
683 iter_type
= long_integer_type_node
;
685 else if (collapse_iter
&& *collapse_iter
!= NULL
)
686 iter_type
= TREE_TYPE (*collapse_iter
);
687 fd
->iter_type
= iter_type
;
688 if (collapse_iter
&& *collapse_iter
== NULL
)
689 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
690 if (collapse_count
&& *collapse_count
== NULL
)
693 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
695 *collapse_count
= create_tmp_var (iter_type
, ".count");
698 if (fd
->collapse
> 1)
700 fd
->loop
.v
= *collapse_iter
;
701 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
702 fd
->loop
.n2
= *collapse_count
;
703 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
704 fd
->loop
.cond_code
= LT_EXPR
;
707 /* For OpenACC loops, force a chunk size of one, as this avoids the default
708 scheduling where several subsequent iterations are being executed by the
710 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
712 gcc_assert (fd
->chunk_size
== NULL_TREE
);
713 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
718 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
719 is the immediate dominator of PAR_ENTRY_BB, return true if there
720 are no data dependencies that would prevent expanding the parallel
721 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
723 When expanding a combined parallel+workshare region, the call to
724 the child function may need additional arguments in the case of
725 GIMPLE_OMP_FOR regions. In some cases, these arguments are
726 computed out of variables passed in from the parent to the child
727 via 'struct .omp_data_s'. For instance:
729 #pragma omp parallel for schedule (guided, i * 4)
734 # BLOCK 2 (PAR_ENTRY_BB)
736 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
738 # BLOCK 3 (WS_ENTRY_BB)
739 .omp_data_i = &.omp_data_o;
740 D.1667 = .omp_data_i->i;
742 #pragma omp for schedule (guided, D.1598)
744 When we outline the parallel region, the call to the child function
745 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
746 that value is computed *after* the call site. So, in principle we
747 cannot do the transformation.
749 To see whether the code in WS_ENTRY_BB blocks the combined
750 parallel+workshare call, we collect all the variables used in the
751 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
752 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
755 FIXME. If we had the SSA form built at this point, we could merely
756 hoist the code in block 3 into block 2 and be done with it. But at
757 this point we don't have dataflow information and though we could
758 hack something up here, it is really not worth the aggravation. */
761 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
763 struct omp_for_data fd
;
764 gimple ws_stmt
= last_stmt (ws_entry_bb
);
766 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
769 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
771 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
773 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
775 if (fd
.iter_type
!= long_integer_type_node
)
778 /* FIXME. We give up too easily here. If any of these arguments
779 are not constants, they will likely involve variables that have
780 been mapped into fields of .omp_data_s for sharing with the child
781 function. With appropriate data flow, it would be possible to
783 if (!is_gimple_min_invariant (fd
.loop
.n1
)
784 || !is_gimple_min_invariant (fd
.loop
.n2
)
785 || !is_gimple_min_invariant (fd
.loop
.step
)
786 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
793 /* Collect additional arguments needed to emit a combined
794 parallel+workshare call. WS_STMT is the workshare directive being
797 static vec
<tree
, va_gc
> *
798 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
801 location_t loc
= gimple_location (ws_stmt
);
802 vec
<tree
, va_gc
> *ws_args
;
804 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
806 struct omp_for_data fd
;
809 extract_omp_for_data (for_stmt
, &fd
, NULL
);
813 if (gimple_omp_for_combined_into_p (for_stmt
))
816 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
817 OMP_CLAUSE__LOOPTEMP_
);
819 n1
= OMP_CLAUSE_DECL (innerc
);
820 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
821 OMP_CLAUSE__LOOPTEMP_
);
823 n2
= OMP_CLAUSE_DECL (innerc
);
826 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
828 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
829 ws_args
->quick_push (t
);
831 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
832 ws_args
->quick_push (t
);
834 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
835 ws_args
->quick_push (t
);
839 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
840 ws_args
->quick_push (t
);
845 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
847 /* Number of sections is equal to the number of edges from the
848 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
849 the exit of the sections region. */
850 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
851 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
852 vec_alloc (ws_args
, 1);
853 ws_args
->quick_push (t
);
861 /* Discover whether REGION is a combined parallel+workshare region. */
864 determine_parallel_type (struct omp_region
*region
)
866 basic_block par_entry_bb
, par_exit_bb
;
867 basic_block ws_entry_bb
, ws_exit_bb
;
869 if (region
== NULL
|| region
->inner
== NULL
870 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
871 || region
->inner
->cont
== NULL
)
874 /* We only support parallel+for and parallel+sections. */
875 if (region
->type
!= GIMPLE_OMP_PARALLEL
876 || (region
->inner
->type
!= GIMPLE_OMP_FOR
877 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
880 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
881 WS_EXIT_BB -> PAR_EXIT_BB. */
882 par_entry_bb
= region
->entry
;
883 par_exit_bb
= region
->exit
;
884 ws_entry_bb
= region
->inner
->entry
;
885 ws_exit_bb
= region
->inner
->exit
;
887 if (single_succ (par_entry_bb
) == ws_entry_bb
888 && single_succ (ws_exit_bb
) == par_exit_bb
889 && workshare_safe_to_combine_p (ws_entry_bb
)
890 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
891 || (last_and_only_stmt (ws_entry_bb
)
892 && last_and_only_stmt (par_exit_bb
))))
894 gimple par_stmt
= last_stmt (par_entry_bb
);
895 gimple ws_stmt
= last_stmt (ws_entry_bb
);
897 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
899 /* If this is a combined parallel loop, we need to determine
900 whether or not to use the combined library calls. There
901 are two cases where we do not apply the transformation:
902 static loops and any kind of ordered loop. In the first
903 case, we already open code the loop so there is no need
904 to do anything else. In the latter case, the combined
905 parallel loop call would still need extra synchronization
906 to implement ordered semantics, so there would not be any
907 gain in using the combined call. */
908 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
909 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
911 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
912 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
914 region
->is_combined_parallel
= false;
915 region
->inner
->is_combined_parallel
= false;
920 region
->is_combined_parallel
= true;
921 region
->inner
->is_combined_parallel
= true;
922 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
927 /* Return true if EXPR is variable sized. */
930 is_variable_sized (const_tree expr
)
932 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
935 /* Return true if DECL is a reference type. */
938 is_reference (tree decl
)
940 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
943 /* Return the type of a decl. If the decl is reference type,
944 return its base type. */
946 get_base_type (tree decl
)
948 tree type
= TREE_TYPE (decl
);
949 if (is_reference (decl
))
950 type
= TREE_TYPE (type
);
954 /* Lookup variables. The "maybe" form
955 allows for the variable form to not have been entered, otherwise we
956 assert that the variable must have been entered. */
959 lookup_decl (tree var
, omp_context
*ctx
)
961 tree
*n
= ctx
->cb
.decl_map
->get (var
);
966 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
968 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
969 return n
? *n
: NULL_TREE
;
973 lookup_field (tree var
, omp_context
*ctx
)
976 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
977 return (tree
) n
->value
;
981 lookup_sfield (tree var
, omp_context
*ctx
)
984 n
= splay_tree_lookup (ctx
->sfield_map
985 ? ctx
->sfield_map
: ctx
->field_map
,
986 (splay_tree_key
) var
);
987 return (tree
) n
->value
;
991 maybe_lookup_field (tree var
, omp_context
*ctx
)
994 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
995 return n
? (tree
) n
->value
: NULL_TREE
;
999 lookup_oacc_reduction (const char *id
, omp_context
*ctx
)
1002 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
1003 return (tree
) n
->value
;
1007 maybe_lookup_oacc_reduction (tree var
, omp_context
*ctx
)
1009 splay_tree_node n
= NULL
;
1010 if (ctx
->reduction_map
)
1011 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
1012 return n
? (tree
) n
->value
: NULL_TREE
;
1015 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1016 the parallel context if DECL is to be shared. */
1019 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1021 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1024 /* We can only use copy-in/copy-out semantics for shared variables
1025 when we know the value is not accessible from an outer scope. */
1028 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1030 /* ??? Trivially accessible from anywhere. But why would we even
1031 be passing an address in this case? Should we simply assert
1032 this to be false, or should we have a cleanup pass that removes
1033 these from the list of mappings? */
1034 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1037 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1038 without analyzing the expression whether or not its location
1039 is accessible to anyone else. In the case of nested parallel
1040 regions it certainly may be. */
1041 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1044 /* Do not use copy-in/copy-out for variables that have their
1046 if (TREE_ADDRESSABLE (decl
))
1049 /* lower_send_shared_vars only uses copy-in, but not copy-out
1051 if (TREE_READONLY (decl
)
1052 || ((TREE_CODE (decl
) == RESULT_DECL
1053 || TREE_CODE (decl
) == PARM_DECL
)
1054 && DECL_BY_REFERENCE (decl
)))
1057 /* Disallow copy-in/out in nested parallel if
1058 decl is shared in outer parallel, otherwise
1059 each thread could store the shared variable
1060 in its own copy-in location, making the
1061 variable no longer really shared. */
1062 if (shared_ctx
->is_nested
)
1066 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1067 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1074 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1075 c
; c
= OMP_CLAUSE_CHAIN (c
))
1076 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1077 && OMP_CLAUSE_DECL (c
) == decl
)
1081 goto maybe_mark_addressable_and_ret
;
1085 /* For tasks avoid using copy-in/out. As tasks can be
1086 deferred or executed in different thread, when GOMP_task
1087 returns, the task hasn't necessarily terminated. */
1088 if (is_task_ctx (shared_ctx
))
1091 maybe_mark_addressable_and_ret
:
1092 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1093 if (is_gimple_reg (outer
))
1095 /* Taking address of OUTER in lower_send_shared_vars
1096 might need regimplification of everything that uses the
1098 if (!task_shared_vars
)
1099 task_shared_vars
= BITMAP_ALLOC (NULL
);
1100 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1101 TREE_ADDRESSABLE (outer
) = 1;
1110 /* Construct a new automatic decl similar to VAR. */
1113 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1115 tree copy
= copy_var_decl (var
, name
, type
);
1117 DECL_CONTEXT (copy
) = current_function_decl
;
1118 DECL_CHAIN (copy
) = ctx
->block_vars
;
1119 ctx
->block_vars
= copy
;
1125 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1127 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1130 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1133 omp_build_component_ref (tree obj
, tree field
)
1135 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1136 if (TREE_THIS_VOLATILE (field
))
1137 TREE_THIS_VOLATILE (ret
) |= 1;
1138 if (TREE_READONLY (field
))
1139 TREE_READONLY (ret
) |= 1;
1143 /* Build tree nodes to access the field for VAR on the receiver side. */
1146 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1148 tree x
, field
= lookup_field (var
, ctx
);
1150 /* If the receiver record type was remapped in the child function,
1151 remap the field into the new record type. */
1152 x
= maybe_lookup_field (field
, ctx
);
1156 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1157 x
= omp_build_component_ref (x
, field
);
1159 x
= build_simple_mem_ref (x
);
1164 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1165 of a parallel, this is a component reference; for workshare constructs
1166 this is some variable. */
1169 build_outer_var_ref (tree var
, omp_context
*ctx
)
1173 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1175 else if (is_variable_sized (var
))
1177 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1178 x
= build_outer_var_ref (x
, ctx
);
1179 x
= build_simple_mem_ref (x
);
1181 else if (is_taskreg_ctx (ctx
))
1183 bool by_ref
= use_pointer_for_field (var
, NULL
);
1184 x
= build_receiver_ref (var
, by_ref
, ctx
);
1186 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1187 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1189 /* #pragma omp simd isn't a worksharing construct, and can reference even
1190 private vars in its linear etc. clauses. */
1192 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1193 x
= lookup_decl (var
, ctx
->outer
);
1194 else if (ctx
->outer
)
1195 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1199 else if (ctx
->outer
)
1200 x
= lookup_decl (var
, ctx
->outer
);
1201 else if (is_reference (var
))
1202 /* This can happen with orphaned constructs. If var is reference, it is
1203 possible it is shared and as such valid. */
1208 if (is_reference (var
))
1209 x
= build_simple_mem_ref (x
);
1214 /* Build tree nodes to access the field for VAR on the sender side. */
1217 build_sender_ref (tree var
, omp_context
*ctx
)
1219 tree field
= lookup_sfield (var
, ctx
);
1220 return omp_build_component_ref (ctx
->sender_decl
, field
);
1223 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1226 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1228 tree field
, type
, sfield
= NULL_TREE
;
1230 gcc_assert ((mask
& 1) == 0
1231 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1232 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1233 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1234 gcc_assert ((mask
& 3) == 3
1235 || !is_gimple_omp_oacc (ctx
->stmt
));
1237 type
= TREE_TYPE (var
);
1240 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1241 type
= build_pointer_type (build_pointer_type (type
));
1244 type
= build_pointer_type (type
);
1245 else if ((mask
& 3) == 1 && is_reference (var
))
1246 type
= TREE_TYPE (type
);
1248 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1249 FIELD_DECL
, DECL_NAME (var
), type
);
1251 /* Remember what variable this field was created for. This does have a
1252 side effect of making dwarf2out ignore this member, so for helpful
1253 debugging we clear it later in delete_omp_context. */
1254 DECL_ABSTRACT_ORIGIN (field
) = var
;
1255 if (type
== TREE_TYPE (var
))
1257 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1258 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1259 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1262 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1264 if ((mask
& 3) == 3)
1266 insert_field_into_struct (ctx
->record_type
, field
);
1267 if (ctx
->srecord_type
)
1269 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1270 FIELD_DECL
, DECL_NAME (var
), type
);
1271 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1272 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1273 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1274 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1275 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1280 if (ctx
->srecord_type
== NULL_TREE
)
1284 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1285 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1286 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1288 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1289 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1290 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1291 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1292 splay_tree_insert (ctx
->sfield_map
,
1293 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1294 (splay_tree_value
) sfield
);
1298 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1299 : ctx
->srecord_type
, field
);
1303 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1304 (splay_tree_value
) field
);
1305 if ((mask
& 2) && ctx
->sfield_map
)
1306 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1307 (splay_tree_value
) sfield
);
1311 install_var_local (tree var
, omp_context
*ctx
)
1313 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1314 insert_decl_map (&ctx
->cb
, var
, new_var
);
1318 /* Adjust the replacement for DECL in CTX for the new context. This means
1319 copying the DECL_VALUE_EXPR, and fixing up the type. */
1322 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1324 tree new_decl
, size
;
1326 new_decl
= lookup_decl (decl
, ctx
);
1328 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1330 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1331 && DECL_HAS_VALUE_EXPR_P (decl
))
1333 tree ve
= DECL_VALUE_EXPR (decl
);
1334 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1335 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1336 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1339 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1341 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1342 if (size
== error_mark_node
)
1343 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1344 DECL_SIZE (new_decl
) = size
;
1346 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1347 if (size
== error_mark_node
)
1348 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1349 DECL_SIZE_UNIT (new_decl
) = size
;
1353 /* The callback for remap_decl. Search all containing contexts for a
1354 mapping of the variable; this avoids having to duplicate the splay
1355 tree ahead of time. We know a mapping doesn't already exist in the
1356 given context. Create new mappings to implement default semantics. */
1359 omp_copy_decl (tree var
, copy_body_data
*cb
)
1361 omp_context
*ctx
= (omp_context
*) cb
;
1364 if (TREE_CODE (var
) == LABEL_DECL
)
1366 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1367 DECL_CONTEXT (new_var
) = current_function_decl
;
1368 insert_decl_map (&ctx
->cb
, var
, new_var
);
1372 while (!is_taskreg_ctx (ctx
))
1377 new_var
= maybe_lookup_decl (var
, ctx
);
1382 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1385 return error_mark_node
;
1389 /* Debugging dumps for parallel regions. */
1390 void dump_omp_region (FILE *, struct omp_region
*, int);
1391 void debug_omp_region (struct omp_region
*);
1392 void debug_all_omp_regions (void);
1394 /* Dump the parallel region tree rooted at REGION. */
1397 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1399 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1400 gimple_code_name
[region
->type
]);
1403 dump_omp_region (file
, region
->inner
, indent
+ 4);
1407 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1408 region
->cont
->index
);
1412 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1413 region
->exit
->index
);
1415 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1418 dump_omp_region (file
, region
->next
, indent
);
1422 debug_omp_region (struct omp_region
*region
)
1424 dump_omp_region (stderr
, region
, 0);
1428 debug_all_omp_regions (void)
1430 dump_omp_region (stderr
, root_omp_region
, 0);
1434 /* Create a new parallel region starting at STMT inside region PARENT. */
1436 static struct omp_region
*
1437 new_omp_region (basic_block bb
, enum gimple_code type
,
1438 struct omp_region
*parent
)
1440 struct omp_region
*region
= XCNEW (struct omp_region
);
1442 region
->outer
= parent
;
1444 region
->type
= type
;
1448 /* This is a nested region. Add it to the list of inner
1449 regions in PARENT. */
1450 region
->next
= parent
->inner
;
1451 parent
->inner
= region
;
1455 /* This is a toplevel region. Add it to the list of toplevel
1456 regions in ROOT_OMP_REGION. */
1457 region
->next
= root_omp_region
;
1458 root_omp_region
= region
;
1464 /* Release the memory associated with the region tree rooted at REGION. */
1467 free_omp_region_1 (struct omp_region
*region
)
1469 struct omp_region
*i
, *n
;
1471 for (i
= region
->inner
; i
; i
= n
)
1474 free_omp_region_1 (i
);
1480 /* Release the memory for the entire omp region tree. */
1483 free_omp_regions (void)
1485 struct omp_region
*r
, *n
;
1486 for (r
= root_omp_region
; r
; r
= n
)
1489 free_omp_region_1 (r
);
1491 root_omp_region
= NULL
;
1495 /* Create a new context, with OUTER_CTX being the surrounding context. */
1497 static omp_context
*
1498 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1500 omp_context
*ctx
= XCNEW (omp_context
);
1502 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1503 (splay_tree_value
) ctx
);
1508 ctx
->outer
= outer_ctx
;
1509 ctx
->cb
= outer_ctx
->cb
;
1510 ctx
->cb
.block
= NULL
;
1511 ctx
->depth
= outer_ctx
->depth
+ 1;
1512 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1516 ctx
->cb
.src_fn
= current_function_decl
;
1517 ctx
->cb
.dst_fn
= current_function_decl
;
1518 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1519 gcc_checking_assert (ctx
->cb
.src_node
);
1520 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1521 ctx
->cb
.src_cfun
= cfun
;
1522 ctx
->cb
.copy_decl
= omp_copy_decl
;
1523 ctx
->cb
.eh_lp_nr
= 0;
1524 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1528 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1533 static gimple_seq
maybe_catch_exception (gimple_seq
);
1535 /* Finalize task copyfn. */
1538 finalize_task_copyfn (gomp_task
*task_stmt
)
1540 struct function
*child_cfun
;
1542 gimple_seq seq
= NULL
, new_seq
;
1545 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1546 if (child_fn
== NULL_TREE
)
1549 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1550 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1552 push_cfun (child_cfun
);
1553 bind
= gimplify_body (child_fn
, false);
1554 gimple_seq_add_stmt (&seq
, bind
);
1555 new_seq
= maybe_catch_exception (seq
);
1558 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1560 gimple_seq_add_stmt (&seq
, bind
);
1562 gimple_set_body (child_fn
, seq
);
1565 /* Inform the callgraph about the new function. */
1566 cgraph_node::add_new_function (child_fn
, false);
1569 /* Destroy a omp_context data structures. Called through the splay tree
1570 value delete callback. */
1573 delete_omp_context (splay_tree_value value
)
1575 omp_context
*ctx
= (omp_context
*) value
;
1577 delete ctx
->cb
.decl_map
;
1580 splay_tree_delete (ctx
->field_map
);
1581 if (ctx
->sfield_map
)
1582 splay_tree_delete (ctx
->sfield_map
);
1583 /* Reduction map is copied to nested contexts, so only delete it in the
1585 if (ctx
->reduction_map
1586 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
1587 && is_gimple_omp_offloaded (ctx
->stmt
)
1588 && is_gimple_omp_oacc (ctx
->stmt
))
1589 splay_tree_delete (ctx
->reduction_map
);
1591 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1592 it produces corrupt debug information. */
1593 if (ctx
->record_type
)
1596 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1597 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1599 if (ctx
->srecord_type
)
1602 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1603 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1606 if (is_task_ctx (ctx
))
1607 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1612 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1616 fixup_child_record_type (omp_context
*ctx
)
1618 tree f
, type
= ctx
->record_type
;
1620 /* ??? It isn't sufficient to just call remap_type here, because
1621 variably_modified_type_p doesn't work the way we expect for
1622 record types. Testing each field for whether it needs remapping
1623 and creating a new record by hand works, however. */
1624 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1625 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1629 tree name
, new_fields
= NULL
;
1631 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1632 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1633 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1634 TYPE_DECL
, name
, type
);
1635 TYPE_NAME (type
) = name
;
1637 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1639 tree new_f
= copy_node (f
);
1640 DECL_CONTEXT (new_f
) = type
;
1641 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1642 DECL_CHAIN (new_f
) = new_fields
;
1643 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1644 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1646 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1650 /* Arrange to be able to look up the receiver field
1651 given the sender field. */
1652 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1653 (splay_tree_value
) new_f
);
1655 TYPE_FIELDS (type
) = nreverse (new_fields
);
1659 TREE_TYPE (ctx
->receiver_decl
)
1660 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1663 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1664 specified by CLAUSES. */
1667 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1670 bool scan_array_reductions
= false;
1672 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1676 switch (OMP_CLAUSE_CODE (c
))
1678 case OMP_CLAUSE_PRIVATE
:
1679 decl
= OMP_CLAUSE_DECL (c
);
1680 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1682 else if (!is_variable_sized (decl
))
1683 install_var_local (decl
, ctx
);
1686 case OMP_CLAUSE_SHARED
:
1687 decl
= OMP_CLAUSE_DECL (c
);
1688 /* Ignore shared directives in teams construct. */
1689 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1691 /* Global variables don't need to be copied,
1692 the receiver side will use them directly. */
1693 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1694 if (is_global_var (odecl
))
1696 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1699 gcc_assert (is_taskreg_ctx (ctx
));
1700 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1701 || !is_variable_sized (decl
));
1702 /* Global variables don't need to be copied,
1703 the receiver side will use them directly. */
1704 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1706 by_ref
= use_pointer_for_field (decl
, ctx
);
1707 if (! TREE_READONLY (decl
)
1708 || TREE_ADDRESSABLE (decl
)
1710 || is_reference (decl
))
1712 install_var_field (decl
, by_ref
, 3, ctx
);
1713 install_var_local (decl
, ctx
);
1716 /* We don't need to copy const scalar vars back. */
1717 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1720 case OMP_CLAUSE_LASTPRIVATE
:
1721 /* Let the corresponding firstprivate clause create
1723 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1727 case OMP_CLAUSE_FIRSTPRIVATE
:
1728 if (is_gimple_omp_oacc (ctx
->stmt
))
1730 sorry ("clause not supported yet");
1734 case OMP_CLAUSE_REDUCTION
:
1735 case OMP_CLAUSE_LINEAR
:
1736 decl
= OMP_CLAUSE_DECL (c
);
1738 if (is_variable_sized (decl
))
1740 if (is_task_ctx (ctx
))
1741 install_var_field (decl
, false, 1, ctx
);
1744 else if (is_taskreg_ctx (ctx
))
1747 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1748 by_ref
= use_pointer_for_field (decl
, NULL
);
1750 if (is_task_ctx (ctx
)
1751 && (global
|| by_ref
|| is_reference (decl
)))
1753 install_var_field (decl
, false, 1, ctx
);
1755 install_var_field (decl
, by_ref
, 2, ctx
);
1758 install_var_field (decl
, by_ref
, 3, ctx
);
1760 install_var_local (decl
, ctx
);
1761 if (is_gimple_omp_oacc (ctx
->stmt
)
1762 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
1764 /* Create a decl for the reduction array. */
1765 tree var
= OMP_CLAUSE_DECL (c
);
1766 tree type
= get_base_type (var
);
1767 tree ptype
= build_pointer_type (type
);
1768 tree array
= create_tmp_var (ptype
,
1769 oacc_get_reduction_array_id (var
));
1770 omp_context
*c
= (ctx
->field_map
? ctx
: ctx
->outer
);
1771 install_var_field (array
, true, 3, c
);
1772 install_var_local (array
, c
);
1774 /* Insert it into the current context. */
1775 splay_tree_insert (ctx
->reduction_map
, (splay_tree_key
)
1776 oacc_get_reduction_array_id (var
),
1777 (splay_tree_value
) array
);
1778 splay_tree_insert (ctx
->reduction_map
,
1779 (splay_tree_key
) array
,
1780 (splay_tree_value
) array
);
1784 case OMP_CLAUSE__LOOPTEMP_
:
1785 gcc_assert (is_parallel_ctx (ctx
));
1786 decl
= OMP_CLAUSE_DECL (c
);
1787 install_var_field (decl
, false, 3, ctx
);
1788 install_var_local (decl
, ctx
);
1791 case OMP_CLAUSE_COPYPRIVATE
:
1792 case OMP_CLAUSE_COPYIN
:
1793 decl
= OMP_CLAUSE_DECL (c
);
1794 by_ref
= use_pointer_for_field (decl
, NULL
);
1795 install_var_field (decl
, by_ref
, 3, ctx
);
1798 case OMP_CLAUSE_DEFAULT
:
1799 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1802 case OMP_CLAUSE_FINAL
:
1804 case OMP_CLAUSE_NUM_THREADS
:
1805 case OMP_CLAUSE_NUM_TEAMS
:
1806 case OMP_CLAUSE_THREAD_LIMIT
:
1807 case OMP_CLAUSE_DEVICE
:
1808 case OMP_CLAUSE_SCHEDULE
:
1809 case OMP_CLAUSE_DIST_SCHEDULE
:
1810 case OMP_CLAUSE_DEPEND
:
1811 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1812 case OMP_CLAUSE_NUM_GANGS
:
1813 case OMP_CLAUSE_NUM_WORKERS
:
1814 case OMP_CLAUSE_VECTOR_LENGTH
:
1816 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1820 case OMP_CLAUSE_FROM
:
1821 case OMP_CLAUSE_MAP
:
1823 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1824 decl
= OMP_CLAUSE_DECL (c
);
1825 /* Global variables with "omp declare target" attribute
1826 don't need to be copied, the receiver side will use them
1828 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1830 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1831 && varpool_node::get_create (decl
)->offloadable
)
1833 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1834 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
1836 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1837 not offloaded; there is nothing to map for those. */
1838 if (!is_gimple_omp_offloaded (ctx
->stmt
)
1839 && !POINTER_TYPE_P (TREE_TYPE (decl
))
1840 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
1845 if (DECL_SIZE (decl
)
1846 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1848 tree decl2
= DECL_VALUE_EXPR (decl
);
1849 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1850 decl2
= TREE_OPERAND (decl2
, 0);
1851 gcc_assert (DECL_P (decl2
));
1852 install_var_field (decl2
, true, 3, ctx
);
1853 install_var_local (decl2
, ctx
);
1854 install_var_local (decl
, ctx
);
1858 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1859 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1860 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1861 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1862 install_var_field (decl
, true, 7, ctx
);
1864 install_var_field (decl
, true, 3, ctx
);
1865 if (is_gimple_omp_offloaded (ctx
->stmt
))
1866 install_var_local (decl
, ctx
);
1871 tree base
= get_base_address (decl
);
1872 tree nc
= OMP_CLAUSE_CHAIN (c
);
1875 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1876 && OMP_CLAUSE_DECL (nc
) == base
1877 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
1878 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1880 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1881 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1887 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1888 decl
= OMP_CLAUSE_DECL (c
);
1890 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1891 (splay_tree_key
) decl
));
1893 = build_decl (OMP_CLAUSE_LOCATION (c
),
1894 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1895 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1896 insert_field_into_struct (ctx
->record_type
, field
);
1897 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1898 (splay_tree_value
) field
);
1903 case OMP_CLAUSE_NOWAIT
:
1904 case OMP_CLAUSE_ORDERED
:
1905 case OMP_CLAUSE_COLLAPSE
:
1906 case OMP_CLAUSE_UNTIED
:
1907 case OMP_CLAUSE_MERGEABLE
:
1908 case OMP_CLAUSE_PROC_BIND
:
1909 case OMP_CLAUSE_SAFELEN
:
1910 case OMP_CLAUSE_ASYNC
:
1911 case OMP_CLAUSE_WAIT
:
1912 case OMP_CLAUSE_GANG
:
1913 case OMP_CLAUSE_WORKER
:
1914 case OMP_CLAUSE_VECTOR
:
1917 case OMP_CLAUSE_ALIGNED
:
1918 decl
= OMP_CLAUSE_DECL (c
);
1919 if (is_global_var (decl
)
1920 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1921 install_var_local (decl
, ctx
);
1924 case OMP_CLAUSE_DEVICE_RESIDENT
:
1925 case OMP_CLAUSE_USE_DEVICE
:
1926 case OMP_CLAUSE__CACHE_
:
1927 case OMP_CLAUSE_INDEPENDENT
:
1928 case OMP_CLAUSE_AUTO
:
1929 case OMP_CLAUSE_SEQ
:
1930 sorry ("Clause not supported yet");
1938 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1940 switch (OMP_CLAUSE_CODE (c
))
1942 case OMP_CLAUSE_LASTPRIVATE
:
1943 /* Let the corresponding firstprivate clause create
1945 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1946 scan_array_reductions
= true;
1947 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1951 case OMP_CLAUSE_FIRSTPRIVATE
:
1952 if (is_gimple_omp_oacc (ctx
->stmt
))
1954 sorry ("clause not supported yet");
1958 case OMP_CLAUSE_PRIVATE
:
1959 case OMP_CLAUSE_REDUCTION
:
1960 case OMP_CLAUSE_LINEAR
:
1961 decl
= OMP_CLAUSE_DECL (c
);
1962 if (is_variable_sized (decl
))
1963 install_var_local (decl
, ctx
);
1964 fixup_remapped_decl (decl
, ctx
,
1965 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1966 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1967 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1968 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1969 scan_array_reductions
= true;
1970 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1971 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1972 scan_array_reductions
= true;
1975 case OMP_CLAUSE_SHARED
:
1976 /* Ignore shared directives in teams construct. */
1977 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1979 decl
= OMP_CLAUSE_DECL (c
);
1980 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1981 fixup_remapped_decl (decl
, ctx
, false);
1984 case OMP_CLAUSE_MAP
:
1985 if (!is_gimple_omp_offloaded (ctx
->stmt
))
1987 decl
= OMP_CLAUSE_DECL (c
);
1989 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1990 && varpool_node::get_create (decl
)->offloadable
)
1994 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1995 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1996 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1998 tree new_decl
= lookup_decl (decl
, ctx
);
1999 TREE_TYPE (new_decl
)
2000 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2002 else if (DECL_SIZE (decl
)
2003 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2005 tree decl2
= DECL_VALUE_EXPR (decl
);
2006 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2007 decl2
= TREE_OPERAND (decl2
, 0);
2008 gcc_assert (DECL_P (decl2
));
2009 fixup_remapped_decl (decl2
, ctx
, false);
2010 fixup_remapped_decl (decl
, ctx
, true);
2013 fixup_remapped_decl (decl
, ctx
, false);
2017 case OMP_CLAUSE_COPYPRIVATE
:
2018 case OMP_CLAUSE_COPYIN
:
2019 case OMP_CLAUSE_DEFAULT
:
2021 case OMP_CLAUSE_NUM_THREADS
:
2022 case OMP_CLAUSE_NUM_TEAMS
:
2023 case OMP_CLAUSE_THREAD_LIMIT
:
2024 case OMP_CLAUSE_DEVICE
:
2025 case OMP_CLAUSE_SCHEDULE
:
2026 case OMP_CLAUSE_DIST_SCHEDULE
:
2027 case OMP_CLAUSE_NOWAIT
:
2028 case OMP_CLAUSE_ORDERED
:
2029 case OMP_CLAUSE_COLLAPSE
:
2030 case OMP_CLAUSE_UNTIED
:
2031 case OMP_CLAUSE_FINAL
:
2032 case OMP_CLAUSE_MERGEABLE
:
2033 case OMP_CLAUSE_PROC_BIND
:
2034 case OMP_CLAUSE_SAFELEN
:
2035 case OMP_CLAUSE_ALIGNED
:
2036 case OMP_CLAUSE_DEPEND
:
2037 case OMP_CLAUSE__LOOPTEMP_
:
2039 case OMP_CLAUSE_FROM
:
2040 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2041 case OMP_CLAUSE_ASYNC
:
2042 case OMP_CLAUSE_WAIT
:
2043 case OMP_CLAUSE_NUM_GANGS
:
2044 case OMP_CLAUSE_NUM_WORKERS
:
2045 case OMP_CLAUSE_VECTOR_LENGTH
:
2046 case OMP_CLAUSE_GANG
:
2047 case OMP_CLAUSE_WORKER
:
2048 case OMP_CLAUSE_VECTOR
:
2051 case OMP_CLAUSE_DEVICE_RESIDENT
:
2052 case OMP_CLAUSE_USE_DEVICE
:
2053 case OMP_CLAUSE__CACHE_
:
2054 case OMP_CLAUSE_INDEPENDENT
:
2055 case OMP_CLAUSE_AUTO
:
2056 case OMP_CLAUSE_SEQ
:
2057 sorry ("Clause not supported yet");
2065 gcc_checking_assert (!scan_array_reductions
2066 || !is_gimple_omp_oacc (ctx
->stmt
));
2067 if (scan_array_reductions
)
2068 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2069 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2070 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2072 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2073 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2075 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2076 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2077 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2078 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2079 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2080 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2083 /* Create a new name for omp child function. Returns an identifier. If
2084 IS_CILK_FOR is true then the suffix for the child function is
2088 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2091 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2092 return clone_function_name (current_function_decl
,
2093 task_copy
? "_omp_cpyfn" : "_omp_fn");
2096 /* Returns the type of the induction variable for the child function for
2097 _Cilk_for and the types for _high and _low variables based on TYPE. */
2100 cilk_for_check_loop_diff_type (tree type
)
2102 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2104 if (TYPE_UNSIGNED (type
))
2105 return uint32_type_node
;
2107 return integer_type_node
;
2111 if (TYPE_UNSIGNED (type
))
2112 return uint64_type_node
;
2114 return long_long_integer_type_node
;
2118 /* Build a decl for the omp child function. It'll not contain a body
2119 yet, just the bare decl. */
2122 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2124 tree decl
, type
, name
, t
;
2127 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2128 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2129 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2130 tree cilk_var_type
= NULL_TREE
;
2132 name
= create_omp_child_function_name (task_copy
,
2133 cilk_for_count
!= NULL_TREE
);
2135 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2136 ptr_type_node
, NULL_TREE
);
2137 else if (cilk_for_count
)
2139 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2140 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2141 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2142 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2145 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2147 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2149 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2152 ctx
->cb
.dst_fn
= decl
;
2154 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2156 TREE_STATIC (decl
) = 1;
2157 TREE_USED (decl
) = 1;
2158 DECL_ARTIFICIAL (decl
) = 1;
2159 DECL_IGNORED_P (decl
) = 0;
2160 TREE_PUBLIC (decl
) = 0;
2161 DECL_UNINLINABLE (decl
) = 1;
2162 DECL_EXTERNAL (decl
) = 0;
2163 DECL_CONTEXT (decl
) = NULL_TREE
;
2164 DECL_INITIAL (decl
) = make_node (BLOCK
);
2165 if (cgraph_node::get (current_function_decl
)->offloadable
)
2166 cgraph_node::get_create (decl
)->offloadable
= 1;
2170 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2171 if (is_gimple_omp_offloaded (octx
->stmt
))
2173 cgraph_node::get_create (decl
)->offloadable
= 1;
2174 #ifdef ENABLE_OFFLOADING
2175 g
->have_offload
= true;
2181 if (cgraph_node::get_create (decl
)->offloadable
2182 && !lookup_attribute ("omp declare target",
2183 DECL_ATTRIBUTES (current_function_decl
)))
2184 DECL_ATTRIBUTES (decl
)
2185 = tree_cons (get_identifier ("omp target entrypoint"),
2186 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2188 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2189 RESULT_DECL
, NULL_TREE
, void_type_node
);
2190 DECL_ARTIFICIAL (t
) = 1;
2191 DECL_IGNORED_P (t
) = 1;
2192 DECL_CONTEXT (t
) = decl
;
2193 DECL_RESULT (decl
) = t
;
2195 /* _Cilk_for's child function requires two extra parameters called
2196 __low and __high that are set the by Cilk runtime when it calls this
2200 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2201 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2202 DECL_ARTIFICIAL (t
) = 1;
2203 DECL_NAMELESS (t
) = 1;
2204 DECL_ARG_TYPE (t
) = ptr_type_node
;
2205 DECL_CONTEXT (t
) = current_function_decl
;
2207 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2208 DECL_ARGUMENTS (decl
) = t
;
2210 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2211 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2212 DECL_ARTIFICIAL (t
) = 1;
2213 DECL_NAMELESS (t
) = 1;
2214 DECL_ARG_TYPE (t
) = ptr_type_node
;
2215 DECL_CONTEXT (t
) = current_function_decl
;
2217 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2218 DECL_ARGUMENTS (decl
) = t
;
2221 tree data_name
= get_identifier (".omp_data_i");
2222 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2224 DECL_ARTIFICIAL (t
) = 1;
2225 DECL_NAMELESS (t
) = 1;
2226 DECL_ARG_TYPE (t
) = ptr_type_node
;
2227 DECL_CONTEXT (t
) = current_function_decl
;
2230 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2231 DECL_ARGUMENTS (decl
) = t
;
2233 ctx
->receiver_decl
= t
;
2236 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2237 PARM_DECL
, get_identifier (".omp_data_o"),
2239 DECL_ARTIFICIAL (t
) = 1;
2240 DECL_NAMELESS (t
) = 1;
2241 DECL_ARG_TYPE (t
) = ptr_type_node
;
2242 DECL_CONTEXT (t
) = current_function_decl
;
2244 TREE_ADDRESSABLE (t
) = 1;
2245 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2246 DECL_ARGUMENTS (decl
) = t
;
2249 /* Allocate memory for the function structure. The call to
2250 allocate_struct_function clobbers CFUN, so we need to restore
2252 push_struct_function (decl
);
2253 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2257 /* Callback for walk_gimple_seq. Check if combined parallel
2258 contains gimple_omp_for_combined_into_p OMP_FOR. */
2261 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2262 bool *handled_ops_p
,
2263 struct walk_stmt_info
*wi
)
2265 gimple stmt
= gsi_stmt (*gsi_p
);
2267 *handled_ops_p
= true;
2268 switch (gimple_code (stmt
))
2272 case GIMPLE_OMP_FOR
:
2273 if (gimple_omp_for_combined_into_p (stmt
)
2274 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2277 return integer_zero_node
;
2286 /* Scan an OpenMP parallel directive. */
2289 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2293 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2295 /* Ignore parallel directives with empty bodies, unless there
2296 are copyin clauses. */
2298 && empty_body_p (gimple_omp_body (stmt
))
2299 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2300 OMP_CLAUSE_COPYIN
) == NULL
)
2302 gsi_replace (gsi
, gimple_build_nop (), false);
2306 if (gimple_omp_parallel_combined_p (stmt
))
2308 struct walk_stmt_info wi
;
2310 memset (&wi
, 0, sizeof (wi
));
2312 walk_gimple_seq (gimple_omp_body (stmt
),
2313 find_combined_for
, NULL
, &wi
);
2316 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
) wi
.info
);
2317 struct omp_for_data fd
;
2318 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2319 /* We need two temporaries with fd.loop.v type (istart/iend)
2320 and then (fd.collapse - 1) temporaries with the same
2321 type for count2 ... countN-1 vars if not constant. */
2322 size_t count
= 2, i
;
2323 tree type
= fd
.iter_type
;
2325 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2326 count
+= fd
.collapse
- 1;
2327 for (i
= 0; i
< count
; i
++)
2329 tree temp
= create_tmp_var (type
);
2330 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2331 OMP_CLAUSE__LOOPTEMP_
);
2332 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2333 OMP_CLAUSE_DECL (c
) = temp
;
2334 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2335 gimple_omp_parallel_set_clauses (stmt
, c
);
2340 ctx
= new_omp_context (stmt
, outer_ctx
);
2341 taskreg_contexts
.safe_push (ctx
);
2342 if (taskreg_nesting_level
> 1)
2343 ctx
->is_nested
= true;
2344 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2345 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2346 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2347 name
= create_tmp_var_name (".omp_data_s");
2348 name
= build_decl (gimple_location (stmt
),
2349 TYPE_DECL
, name
, ctx
->record_type
);
2350 DECL_ARTIFICIAL (name
) = 1;
2351 DECL_NAMELESS (name
) = 1;
2352 TYPE_NAME (ctx
->record_type
) = name
;
2353 create_omp_child_function (ctx
, false);
2354 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2356 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2357 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2359 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2360 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2363 /* Scan an OpenMP task directive. */
2366 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2370 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2372 /* Ignore task directives with empty bodies. */
2374 && empty_body_p (gimple_omp_body (stmt
)))
2376 gsi_replace (gsi
, gimple_build_nop (), false);
2380 ctx
= new_omp_context (stmt
, outer_ctx
);
2381 taskreg_contexts
.safe_push (ctx
);
2382 if (taskreg_nesting_level
> 1)
2383 ctx
->is_nested
= true;
2384 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2385 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2386 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2387 name
= create_tmp_var_name (".omp_data_s");
2388 name
= build_decl (gimple_location (stmt
),
2389 TYPE_DECL
, name
, ctx
->record_type
);
2390 DECL_ARTIFICIAL (name
) = 1;
2391 DECL_NAMELESS (name
) = 1;
2392 TYPE_NAME (ctx
->record_type
) = name
;
2393 create_omp_child_function (ctx
, false);
2394 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2396 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2398 if (ctx
->srecord_type
)
2400 name
= create_tmp_var_name (".omp_data_a");
2401 name
= build_decl (gimple_location (stmt
),
2402 TYPE_DECL
, name
, ctx
->srecord_type
);
2403 DECL_ARTIFICIAL (name
) = 1;
2404 DECL_NAMELESS (name
) = 1;
2405 TYPE_NAME (ctx
->srecord_type
) = name
;
2406 create_omp_child_function (ctx
, true);
2409 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2411 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2413 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2414 t
= build_int_cst (long_integer_type_node
, 0);
2415 gimple_omp_task_set_arg_size (stmt
, t
);
2416 t
= build_int_cst (long_integer_type_node
, 1);
2417 gimple_omp_task_set_arg_align (stmt
, t
);
2422 /* If any decls have been made addressable during scan_omp,
2423 adjust their fields if needed, and layout record types
2424 of parallel/task constructs. */
2427 finish_taskreg_scan (omp_context
*ctx
)
2429 if (ctx
->record_type
== NULL_TREE
)
2432 /* If any task_shared_vars were needed, verify all
2433 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2434 statements if use_pointer_for_field hasn't changed
2435 because of that. If it did, update field types now. */
2436 if (task_shared_vars
)
2440 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2441 c
; c
= OMP_CLAUSE_CHAIN (c
))
2442 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2444 tree decl
= OMP_CLAUSE_DECL (c
);
2446 /* Global variables don't need to be copied,
2447 the receiver side will use them directly. */
2448 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2450 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2451 || !use_pointer_for_field (decl
, ctx
))
2453 tree field
= lookup_field (decl
, ctx
);
2454 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2455 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2457 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2458 TREE_THIS_VOLATILE (field
) = 0;
2459 DECL_USER_ALIGN (field
) = 0;
2460 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2461 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2462 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2463 if (ctx
->srecord_type
)
2465 tree sfield
= lookup_sfield (decl
, ctx
);
2466 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2467 TREE_THIS_VOLATILE (sfield
) = 0;
2468 DECL_USER_ALIGN (sfield
) = 0;
2469 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2470 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2471 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2476 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2478 layout_type (ctx
->record_type
);
2479 fixup_child_record_type (ctx
);
2483 location_t loc
= gimple_location (ctx
->stmt
);
2484 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2485 /* Move VLA fields to the end. */
2486 p
= &TYPE_FIELDS (ctx
->record_type
);
2488 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2489 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2492 *p
= TREE_CHAIN (*p
);
2493 TREE_CHAIN (*q
) = NULL_TREE
;
2494 q
= &TREE_CHAIN (*q
);
2497 p
= &DECL_CHAIN (*p
);
2499 layout_type (ctx
->record_type
);
2500 fixup_child_record_type (ctx
);
2501 if (ctx
->srecord_type
)
2502 layout_type (ctx
->srecord_type
);
2503 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2504 TYPE_SIZE_UNIT (ctx
->record_type
));
2505 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2506 t
= build_int_cst (long_integer_type_node
,
2507 TYPE_ALIGN_UNIT (ctx
->record_type
));
2508 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2513 static omp_context
*
2514 enclosing_target_ctx (omp_context
*ctx
)
2517 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2519 gcc_assert (ctx
!= NULL
);
2524 oacc_loop_or_target_p (gimple stmt
)
2526 enum gimple_code outer_type
= gimple_code (stmt
);
2527 return ((outer_type
== GIMPLE_OMP_TARGET
2528 && ((gimple_omp_target_kind (stmt
)
2529 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2530 || (gimple_omp_target_kind (stmt
)
2531 == GF_OMP_TARGET_KIND_OACC_KERNELS
)))
2532 || (outer_type
== GIMPLE_OMP_FOR
2533 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
));
2536 /* Scan a GIMPLE_OMP_FOR. */
2539 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2541 enum gimple_code outer_type
= GIMPLE_ERROR_MARK
;
2544 tree clauses
= gimple_omp_for_clauses (stmt
);
2547 outer_type
= gimple_code (outer_ctx
->stmt
);
2549 ctx
= new_omp_context (stmt
, outer_ctx
);
2551 if (is_gimple_omp_oacc (stmt
))
2553 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2554 ctx
->gwv_this
= outer_ctx
->gwv_this
;
2555 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2558 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_GANG
)
2560 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WORKER
)
2562 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR
)
2566 ctx
->gwv_this
|= val
;
2569 /* Skip; not nested inside a region. */
2572 if (!oacc_loop_or_target_p (outer_ctx
->stmt
))
2574 /* Skip; not nested inside an OpenACC region. */
2577 if (outer_type
== GIMPLE_OMP_FOR
)
2578 outer_ctx
->gwv_below
|= val
;
2579 if (OMP_CLAUSE_OPERAND (c
, 0) != NULL_TREE
)
2581 omp_context
*enclosing
= enclosing_target_ctx (outer_ctx
);
2582 if (gimple_omp_target_kind (enclosing
->stmt
)
2583 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2584 error_at (gimple_location (stmt
),
2585 "no arguments allowed to gang, worker and vector clauses inside parallel");
2590 scan_sharing_clauses (clauses
, ctx
);
2592 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2593 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2595 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2596 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2597 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2598 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2600 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2602 if (is_gimple_omp_oacc (stmt
))
2604 if (ctx
->gwv_this
& ctx
->gwv_below
)
2605 error_at (gimple_location (stmt
),
2606 "gang, worker and vector may occur only once in a loop nest");
2607 else if (ctx
->gwv_below
!= 0
2608 && ctx
->gwv_this
> ctx
->gwv_below
)
2609 error_at (gimple_location (stmt
),
2610 "gang, worker and vector must occur in this order in a loop nest");
2611 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2612 outer_ctx
->gwv_below
|= ctx
->gwv_below
;
2616 /* Scan an OpenMP sections directive. */
2619 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
2623 ctx
= new_omp_context (stmt
, outer_ctx
);
2624 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2625 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2628 /* Scan an OpenMP single directive. */
2631 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
2636 ctx
= new_omp_context (stmt
, outer_ctx
);
2637 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2638 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2639 name
= create_tmp_var_name (".omp_copy_s");
2640 name
= build_decl (gimple_location (stmt
),
2641 TYPE_DECL
, name
, ctx
->record_type
);
2642 TYPE_NAME (ctx
->record_type
) = name
;
2644 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2645 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2647 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2648 ctx
->record_type
= NULL
;
2650 layout_type (ctx
->record_type
);
2653 /* Scan a GIMPLE_OMP_TARGET. */
2656 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
2660 bool offloaded
= is_gimple_omp_offloaded (stmt
);
2661 tree clauses
= gimple_omp_target_clauses (stmt
);
2663 ctx
= new_omp_context (stmt
, outer_ctx
);
2664 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2665 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2666 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2667 name
= create_tmp_var_name (".omp_data_t");
2668 name
= build_decl (gimple_location (stmt
),
2669 TYPE_DECL
, name
, ctx
->record_type
);
2670 DECL_ARTIFICIAL (name
) = 1;
2671 DECL_NAMELESS (name
) = 1;
2672 TYPE_NAME (ctx
->record_type
) = name
;
2675 if (is_gimple_omp_oacc (stmt
))
2676 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
2679 create_omp_child_function (ctx
, false);
2680 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2683 if (is_gimple_omp_oacc (stmt
))
2685 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2687 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_GANGS
)
2688 ctx
->gwv_this
|= MASK_GANG
;
2689 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_WORKERS
)
2690 ctx
->gwv_this
|= MASK_WORKER
;
2691 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR_LENGTH
)
2692 ctx
->gwv_this
|= MASK_VECTOR
;
2696 scan_sharing_clauses (clauses
, ctx
);
2697 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2699 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2700 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2703 TYPE_FIELDS (ctx
->record_type
)
2704 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2705 #ifdef ENABLE_CHECKING
2707 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2708 for (field
= TYPE_FIELDS (ctx
->record_type
);
2710 field
= DECL_CHAIN (field
))
2711 gcc_assert (DECL_ALIGN (field
) == align
);
2713 layout_type (ctx
->record_type
);
2715 fixup_child_record_type (ctx
);
2719 /* Scan an OpenMP teams directive. */
2722 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
2724 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2725 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2726 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2729 /* Check nesting restrictions. */
2731 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2733 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2734 inside an OpenACC CTX. */
2735 if (!(is_gimple_omp (stmt
)
2736 && is_gimple_omp_oacc (stmt
)))
2738 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
2739 if (is_gimple_omp (ctx_
->stmt
)
2740 && is_gimple_omp_oacc (ctx_
->stmt
))
2742 error_at (gimple_location (stmt
),
2743 "non-OpenACC construct inside of OpenACC region");
2750 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2751 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2753 error_at (gimple_location (stmt
),
2754 "OpenMP constructs may not be nested inside simd region");
2757 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2759 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2760 || (gimple_omp_for_kind (stmt
)
2761 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2762 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2764 error_at (gimple_location (stmt
),
2765 "only distribute or parallel constructs are allowed to "
2766 "be closely nested inside teams construct");
2771 switch (gimple_code (stmt
))
2773 case GIMPLE_OMP_FOR
:
2774 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2776 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2778 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2780 error_at (gimple_location (stmt
),
2781 "distribute construct must be closely nested inside "
2789 if (is_gimple_call (stmt
)
2790 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2791 == BUILT_IN_GOMP_CANCEL
2792 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2793 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2795 const char *bad
= NULL
;
2796 const char *kind
= NULL
;
2799 error_at (gimple_location (stmt
), "orphaned %qs construct",
2800 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2801 == BUILT_IN_GOMP_CANCEL
2802 ? "#pragma omp cancel"
2803 : "#pragma omp cancellation point");
2806 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2807 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2811 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2812 bad
= "#pragma omp parallel";
2813 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2814 == BUILT_IN_GOMP_CANCEL
2815 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2816 ctx
->cancellable
= true;
2820 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2821 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2822 bad
= "#pragma omp for";
2823 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2824 == BUILT_IN_GOMP_CANCEL
2825 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2827 ctx
->cancellable
= true;
2828 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2830 warning_at (gimple_location (stmt
), 0,
2831 "%<#pragma omp cancel for%> inside "
2832 "%<nowait%> for construct");
2833 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2834 OMP_CLAUSE_ORDERED
))
2835 warning_at (gimple_location (stmt
), 0,
2836 "%<#pragma omp cancel for%> inside "
2837 "%<ordered%> for construct");
2842 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2843 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2844 bad
= "#pragma omp sections";
2845 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2846 == BUILT_IN_GOMP_CANCEL
2847 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2849 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2851 ctx
->cancellable
= true;
2852 if (find_omp_clause (gimple_omp_sections_clauses
2855 warning_at (gimple_location (stmt
), 0,
2856 "%<#pragma omp cancel sections%> inside "
2857 "%<nowait%> sections construct");
2861 gcc_assert (ctx
->outer
2862 && gimple_code (ctx
->outer
->stmt
)
2863 == GIMPLE_OMP_SECTIONS
);
2864 ctx
->outer
->cancellable
= true;
2865 if (find_omp_clause (gimple_omp_sections_clauses
2868 warning_at (gimple_location (stmt
), 0,
2869 "%<#pragma omp cancel sections%> inside "
2870 "%<nowait%> sections construct");
2876 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2877 bad
= "#pragma omp task";
2879 ctx
->cancellable
= true;
2883 error_at (gimple_location (stmt
), "invalid arguments");
2888 error_at (gimple_location (stmt
),
2889 "%<%s %s%> construct not closely nested inside of %qs",
2890 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2891 == BUILT_IN_GOMP_CANCEL
2892 ? "#pragma omp cancel"
2893 : "#pragma omp cancellation point", kind
, bad
);
2898 case GIMPLE_OMP_SECTIONS
:
2899 case GIMPLE_OMP_SINGLE
:
2900 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2901 switch (gimple_code (ctx
->stmt
))
2903 case GIMPLE_OMP_FOR
:
2904 case GIMPLE_OMP_SECTIONS
:
2905 case GIMPLE_OMP_SINGLE
:
2906 case GIMPLE_OMP_ORDERED
:
2907 case GIMPLE_OMP_MASTER
:
2908 case GIMPLE_OMP_TASK
:
2909 case GIMPLE_OMP_CRITICAL
:
2910 if (is_gimple_call (stmt
))
2912 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2913 != BUILT_IN_GOMP_BARRIER
)
2915 error_at (gimple_location (stmt
),
2916 "barrier region may not be closely nested inside "
2917 "of work-sharing, critical, ordered, master or "
2918 "explicit task region");
2921 error_at (gimple_location (stmt
),
2922 "work-sharing region may not be closely nested inside "
2923 "of work-sharing, critical, ordered, master or explicit "
2926 case GIMPLE_OMP_PARALLEL
:
2932 case GIMPLE_OMP_MASTER
:
2933 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2934 switch (gimple_code (ctx
->stmt
))
2936 case GIMPLE_OMP_FOR
:
2937 case GIMPLE_OMP_SECTIONS
:
2938 case GIMPLE_OMP_SINGLE
:
2939 case GIMPLE_OMP_TASK
:
2940 error_at (gimple_location (stmt
),
2941 "master region may not be closely nested inside "
2942 "of work-sharing or explicit task region");
2944 case GIMPLE_OMP_PARALLEL
:
2950 case GIMPLE_OMP_ORDERED
:
2951 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2952 switch (gimple_code (ctx
->stmt
))
2954 case GIMPLE_OMP_CRITICAL
:
2955 case GIMPLE_OMP_TASK
:
2956 error_at (gimple_location (stmt
),
2957 "ordered region may not be closely nested inside "
2958 "of critical or explicit task region");
2960 case GIMPLE_OMP_FOR
:
2961 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2962 OMP_CLAUSE_ORDERED
) == NULL
)
2964 error_at (gimple_location (stmt
),
2965 "ordered region must be closely nested inside "
2966 "a loop region with an ordered clause");
2970 case GIMPLE_OMP_PARALLEL
:
2971 error_at (gimple_location (stmt
),
2972 "ordered region must be closely nested inside "
2973 "a loop region with an ordered clause");
2979 case GIMPLE_OMP_CRITICAL
:
2982 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
2983 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2984 if (gomp_critical
*other_crit
2985 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
2986 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
2988 error_at (gimple_location (stmt
),
2989 "critical region may not be nested inside a critical "
2990 "region with the same name");
2995 case GIMPLE_OMP_TEAMS
:
2997 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2998 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3000 error_at (gimple_location (stmt
),
3001 "teams construct not closely nested inside of target "
3006 case GIMPLE_OMP_TARGET
:
3007 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3009 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3011 if (is_gimple_omp (stmt
)
3012 && is_gimple_omp_oacc (stmt
)
3013 && is_gimple_omp (ctx
->stmt
))
3015 error_at (gimple_location (stmt
),
3016 "OpenACC construct inside of non-OpenACC region");
3022 const char *stmt_name
, *ctx_stmt_name
;
3023 switch (gimple_omp_target_kind (stmt
))
3025 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3026 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3027 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3028 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3029 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3030 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3031 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3032 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
: stmt_name
= "enter/exit data"; break;
3033 default: gcc_unreachable ();
3035 switch (gimple_omp_target_kind (ctx
->stmt
))
3037 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3038 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3039 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: ctx_stmt_name
= "parallel"; break;
3040 case GF_OMP_TARGET_KIND_OACC_KERNELS
: ctx_stmt_name
= "kernels"; break;
3041 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3042 default: gcc_unreachable ();
3045 /* OpenACC/OpenMP mismatch? */
3046 if (is_gimple_omp_oacc (stmt
)
3047 != is_gimple_omp_oacc (ctx
->stmt
))
3049 error_at (gimple_location (stmt
),
3050 "%s %s construct inside of %s %s region",
3051 (is_gimple_omp_oacc (stmt
)
3052 ? "OpenACC" : "OpenMP"), stmt_name
,
3053 (is_gimple_omp_oacc (ctx
->stmt
)
3054 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3057 if (is_gimple_omp_offloaded (ctx
->stmt
))
3059 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3060 if (is_gimple_omp_oacc (ctx
->stmt
))
3062 error_at (gimple_location (stmt
),
3063 "%s construct inside of %s region",
3064 stmt_name
, ctx_stmt_name
);
3069 gcc_checking_assert (!is_gimple_omp_oacc (stmt
));
3070 warning_at (gimple_location (stmt
), 0,
3071 "%s construct inside of %s region",
3072 stmt_name
, ctx_stmt_name
);
3084 /* Helper function scan_omp.
3086 Callback for walk_tree or operators in walk_gimple_stmt used to
3087 scan for OMP directives in TP. */
3090 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3092 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3093 omp_context
*ctx
= (omp_context
*) wi
->info
;
3096 switch (TREE_CODE (t
))
3103 *tp
= remap_decl (t
, &ctx
->cb
);
3107 if (ctx
&& TYPE_P (t
))
3108 *tp
= remap_type (t
, &ctx
->cb
);
3109 else if (!DECL_P (t
))
3114 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3115 if (tem
!= TREE_TYPE (t
))
3117 if (TREE_CODE (t
) == INTEGER_CST
)
3118 *tp
= wide_int_to_tree (tem
, t
);
3120 TREE_TYPE (t
) = tem
;
3130 /* Return true if FNDECL is a setjmp or a longjmp. */
3133 setjmp_or_longjmp_p (const_tree fndecl
)
3135 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3136 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3137 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3140 tree declname
= DECL_NAME (fndecl
);
3143 const char *name
= IDENTIFIER_POINTER (declname
);
3144 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3148 /* Helper function for scan_omp.
3150 Callback for walk_gimple_stmt used to scan for OMP directives in
3151 the current statement in GSI. */
3154 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3155 struct walk_stmt_info
*wi
)
3157 gimple stmt
= gsi_stmt (*gsi
);
3158 omp_context
*ctx
= (omp_context
*) wi
->info
;
3160 if (gimple_has_location (stmt
))
3161 input_location
= gimple_location (stmt
);
3163 /* Check the nesting restrictions. */
3164 bool remove
= false;
3165 if (is_gimple_omp (stmt
))
3166 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3167 else if (is_gimple_call (stmt
))
3169 tree fndecl
= gimple_call_fndecl (stmt
);
3172 if (setjmp_or_longjmp_p (fndecl
)
3174 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3175 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3178 error_at (gimple_location (stmt
),
3179 "setjmp/longjmp inside simd construct");
3181 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3182 switch (DECL_FUNCTION_CODE (fndecl
))
3184 case BUILT_IN_GOMP_BARRIER
:
3185 case BUILT_IN_GOMP_CANCEL
:
3186 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3187 case BUILT_IN_GOMP_TASKYIELD
:
3188 case BUILT_IN_GOMP_TASKWAIT
:
3189 case BUILT_IN_GOMP_TASKGROUP_START
:
3190 case BUILT_IN_GOMP_TASKGROUP_END
:
3191 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3200 stmt
= gimple_build_nop ();
3201 gsi_replace (gsi
, stmt
, false);
3204 *handled_ops_p
= true;
3206 switch (gimple_code (stmt
))
3208 case GIMPLE_OMP_PARALLEL
:
3209 taskreg_nesting_level
++;
3210 scan_omp_parallel (gsi
, ctx
);
3211 taskreg_nesting_level
--;
3214 case GIMPLE_OMP_TASK
:
3215 taskreg_nesting_level
++;
3216 scan_omp_task (gsi
, ctx
);
3217 taskreg_nesting_level
--;
3220 case GIMPLE_OMP_FOR
:
3221 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3224 case GIMPLE_OMP_SECTIONS
:
3225 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3228 case GIMPLE_OMP_SINGLE
:
3229 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3232 case GIMPLE_OMP_SECTION
:
3233 case GIMPLE_OMP_MASTER
:
3234 case GIMPLE_OMP_TASKGROUP
:
3235 case GIMPLE_OMP_ORDERED
:
3236 case GIMPLE_OMP_CRITICAL
:
3237 ctx
= new_omp_context (stmt
, ctx
);
3238 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3241 case GIMPLE_OMP_TARGET
:
3242 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3245 case GIMPLE_OMP_TEAMS
:
3246 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3253 *handled_ops_p
= false;
3255 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3257 var
= DECL_CHAIN (var
))
3258 insert_decl_map (&ctx
->cb
, var
, var
);
3262 *handled_ops_p
= false;
3270 /* Scan all the statements starting at the current statement. CTX
3271 contains context information about the OMP directives and
3272 clauses found during the scan. */
3275 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3277 location_t saved_location
;
3278 struct walk_stmt_info wi
;
3280 memset (&wi
, 0, sizeof (wi
));
3282 wi
.want_locations
= true;
3284 saved_location
= input_location
;
3285 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3286 input_location
= saved_location
;
3289 /* Re-gimplification and code generation routines. */
3291 /* Build a call to GOMP_barrier. */
3294 build_omp_barrier (tree lhs
)
3296 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3297 : BUILT_IN_GOMP_BARRIER
);
3298 gcall
*g
= gimple_build_call (fndecl
, 0);
3300 gimple_call_set_lhs (g
, lhs
);
3304 /* If a context was created for STMT when it was scanned, return it. */
3306 static omp_context
*
3307 maybe_lookup_ctx (gimple stmt
)
3310 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3311 return n
? (omp_context
*) n
->value
: NULL
;
3315 /* Find the mapping for DECL in CTX or the immediately enclosing
3316 context that has a mapping for DECL.
3318 If CTX is a nested parallel directive, we may have to use the decl
3319 mappings created in CTX's parent context. Suppose that we have the
3320 following parallel nesting (variable UIDs showed for clarity):
3323 #omp parallel shared(iD.1562) -> outer parallel
3324 iD.1562 = iD.1562 + 1;
3326 #omp parallel shared (iD.1562) -> inner parallel
3327 iD.1562 = iD.1562 - 1;
3329 Each parallel structure will create a distinct .omp_data_s structure
3330 for copying iD.1562 in/out of the directive:
3332 outer parallel .omp_data_s.1.i -> iD.1562
3333 inner parallel .omp_data_s.2.i -> iD.1562
3335 A shared variable mapping will produce a copy-out operation before
3336 the parallel directive and a copy-in operation after it. So, in
3337 this case we would have:
3340 .omp_data_o.1.i = iD.1562;
3341 #omp parallel shared(iD.1562) -> outer parallel
3342 .omp_data_i.1 = &.omp_data_o.1
3343 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3345 .omp_data_o.2.i = iD.1562; -> **
3346 #omp parallel shared(iD.1562) -> inner parallel
3347 .omp_data_i.2 = &.omp_data_o.2
3348 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3351 ** This is a problem. The symbol iD.1562 cannot be referenced
3352 inside the body of the outer parallel region. But since we are
3353 emitting this copy operation while expanding the inner parallel
3354 directive, we need to access the CTX structure of the outer
3355 parallel directive to get the correct mapping:
3357 .omp_data_o.2.i = .omp_data_i.1->i
3359 Since there may be other workshare or parallel directives enclosing
3360 the parallel directive, it may be necessary to walk up the context
3361 parent chain. This is not a problem in general because nested
3362 parallelism happens only rarely. */
3365 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3370 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3371 t
= maybe_lookup_decl (decl
, up
);
3373 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3375 return t
? t
: decl
;
3379 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3380 in outer contexts. */
3383 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3388 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3389 t
= maybe_lookup_decl (decl
, up
);
3391 return t
? t
: decl
;
3395 /* Construct the initialization value for reduction CLAUSE. */
3398 omp_reduction_init (tree clause
, tree type
)
3400 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3401 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3408 case TRUTH_ORIF_EXPR
:
3409 case TRUTH_XOR_EXPR
:
3411 return build_zero_cst (type
);
3414 case TRUTH_AND_EXPR
:
3415 case TRUTH_ANDIF_EXPR
:
3417 return fold_convert_loc (loc
, type
, integer_one_node
);
3420 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3423 if (SCALAR_FLOAT_TYPE_P (type
))
3425 REAL_VALUE_TYPE max
, min
;
3426 if (HONOR_INFINITIES (type
))
3429 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3432 real_maxval (&min
, 1, TYPE_MODE (type
));
3433 return build_real (type
, min
);
3437 gcc_assert (INTEGRAL_TYPE_P (type
));
3438 return TYPE_MIN_VALUE (type
);
3442 if (SCALAR_FLOAT_TYPE_P (type
))
3444 REAL_VALUE_TYPE max
;
3445 if (HONOR_INFINITIES (type
))
3448 real_maxval (&max
, 0, TYPE_MODE (type
));
3449 return build_real (type
, max
);
3453 gcc_assert (INTEGRAL_TYPE_P (type
));
3454 return TYPE_MAX_VALUE (type
);
3462 /* Return alignment to be assumed for var in CLAUSE, which should be
3463 OMP_CLAUSE_ALIGNED. */
3466 omp_clause_aligned_alignment (tree clause
)
3468 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3469 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3471 /* Otherwise return implementation defined alignment. */
3472 unsigned int al
= 1;
3473 machine_mode mode
, vmode
;
3474 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3476 vs
= 1 << floor_log2 (vs
);
3477 static enum mode_class classes
[]
3478 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3479 for (int i
= 0; i
< 4; i
+= 2)
3480 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3482 mode
= GET_MODE_WIDER_MODE (mode
))
3484 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3485 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3488 && GET_MODE_SIZE (vmode
) < vs
3489 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3490 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3492 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3493 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3495 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3496 / GET_MODE_SIZE (mode
));
3497 if (TYPE_MODE (type
) != vmode
)
3499 if (TYPE_ALIGN_UNIT (type
) > al
)
3500 al
= TYPE_ALIGN_UNIT (type
);
3502 return build_int_cst (integer_type_node
, al
);
3505 /* Return maximum possible vectorization factor for the target. */
3512 || !flag_tree_loop_optimize
3513 || (!flag_tree_loop_vectorize
3514 && (global_options_set
.x_flag_tree_loop_vectorize
3515 || global_options_set
.x_flag_tree_vectorize
)))
3518 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3521 vs
= 1 << floor_log2 (vs
);
3524 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3525 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3526 return GET_MODE_NUNITS (vqimode
);
3530 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3534 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3535 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3539 max_vf
= omp_max_vf ();
3542 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3543 OMP_CLAUSE_SAFELEN
);
3544 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3546 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3548 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3552 idx
= create_tmp_var (unsigned_type_node
);
3553 lane
= create_tmp_var (unsigned_type_node
);
3559 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3560 tree avar
= create_tmp_var_raw (atype
);
3561 if (TREE_ADDRESSABLE (new_var
))
3562 TREE_ADDRESSABLE (avar
) = 1;
3563 DECL_ATTRIBUTES (avar
)
3564 = tree_cons (get_identifier ("omp simd array"), NULL
,
3565 DECL_ATTRIBUTES (avar
));
3566 gimple_add_tmp_var (avar
);
3567 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3568 NULL_TREE
, NULL_TREE
);
3569 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3570 NULL_TREE
, NULL_TREE
);
3571 if (DECL_P (new_var
))
3573 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3574 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3579 /* Helper function of lower_rec_input_clauses. For a reference
3580 in simd reduction, add an underlying variable it will reference. */
3583 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3585 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3586 if (TREE_CONSTANT (z
))
3588 const char *name
= NULL
;
3589 if (DECL_NAME (new_vard
))
3590 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3592 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3593 gimple_add_tmp_var (z
);
3594 TREE_ADDRESSABLE (z
) = 1;
3595 z
= build_fold_addr_expr_loc (loc
, z
);
3596 gimplify_assign (new_vard
, z
, ilist
);
3600 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3601 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3602 private variables. Initialization statements go in ILIST, while calls
3603 to destructors go in DLIST. */
3606 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3607 omp_context
*ctx
, struct omp_for_data
*fd
)
3609 tree c
, dtor
, copyin_seq
, x
, ptr
;
3610 bool copyin_by_ref
= false;
3611 bool lastprivate_firstprivate
= false;
3612 bool reduction_omp_orig_ref
= false;
3614 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3615 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3617 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3618 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3619 gimple_seq llist
[2] = { NULL
, NULL
};
3623 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3624 with data sharing clauses referencing variable sized vars. That
3625 is unnecessarily hard to support and very unlikely to result in
3626 vectorized code anyway. */
3628 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3629 switch (OMP_CLAUSE_CODE (c
))
3631 case OMP_CLAUSE_LINEAR
:
3632 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3635 case OMP_CLAUSE_REDUCTION
:
3636 case OMP_CLAUSE_PRIVATE
:
3637 case OMP_CLAUSE_FIRSTPRIVATE
:
3638 case OMP_CLAUSE_LASTPRIVATE
:
3639 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3646 /* Do all the fixed sized types in the first pass, and the variable sized
3647 types in the second pass. This makes sure that the scalar arguments to
3648 the variable sized types are processed before we use them in the
3649 variable sized operations. */
3650 for (pass
= 0; pass
< 2; ++pass
)
3652 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3654 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3657 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3661 case OMP_CLAUSE_PRIVATE
:
3662 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3665 case OMP_CLAUSE_SHARED
:
3666 /* Ignore shared directives in teams construct. */
3667 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3669 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3671 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3674 case OMP_CLAUSE_FIRSTPRIVATE
:
3675 case OMP_CLAUSE_COPYIN
:
3676 case OMP_CLAUSE_LINEAR
:
3678 case OMP_CLAUSE_REDUCTION
:
3679 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3680 reduction_omp_orig_ref
= true;
3682 case OMP_CLAUSE__LOOPTEMP_
:
3683 /* Handle _looptemp_ clauses only on parallel. */
3687 case OMP_CLAUSE_LASTPRIVATE
:
3688 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3690 lastprivate_firstprivate
= true;
3694 /* Even without corresponding firstprivate, if
3695 decl is Fortran allocatable, it needs outer var
3698 && lang_hooks
.decls
.omp_private_outer_ref
3699 (OMP_CLAUSE_DECL (c
)))
3700 lastprivate_firstprivate
= true;
3702 case OMP_CLAUSE_ALIGNED
:
3705 var
= OMP_CLAUSE_DECL (c
);
3706 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3707 && !is_global_var (var
))
3709 new_var
= maybe_lookup_decl (var
, ctx
);
3710 if (new_var
== NULL_TREE
)
3711 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3712 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3713 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3714 omp_clause_aligned_alignment (c
));
3715 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3716 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3717 gimplify_and_add (x
, ilist
);
3719 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3720 && is_global_var (var
))
3722 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3723 new_var
= lookup_decl (var
, ctx
);
3724 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3725 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3726 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3727 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3728 omp_clause_aligned_alignment (c
));
3729 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3730 x
= create_tmp_var (ptype
);
3731 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3732 gimplify_and_add (t
, ilist
);
3733 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3734 SET_DECL_VALUE_EXPR (new_var
, t
);
3735 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3742 new_var
= var
= OMP_CLAUSE_DECL (c
);
3743 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3744 new_var
= lookup_decl (var
, ctx
);
3746 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3751 else if (is_variable_sized (var
))
3753 /* For variable sized types, we need to allocate the
3754 actual storage here. Call alloca and store the
3755 result in the pointer decl that we created elsewhere. */
3759 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3764 ptr
= DECL_VALUE_EXPR (new_var
);
3765 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3766 ptr
= TREE_OPERAND (ptr
, 0);
3767 gcc_assert (DECL_P (ptr
));
3768 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3770 /* void *tmp = __builtin_alloca */
3771 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3772 stmt
= gimple_build_call (atmp
, 1, x
);
3773 tmp
= create_tmp_var_raw (ptr_type_node
);
3774 gimple_add_tmp_var (tmp
);
3775 gimple_call_set_lhs (stmt
, tmp
);
3777 gimple_seq_add_stmt (ilist
, stmt
);
3779 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3780 gimplify_assign (ptr
, x
, ilist
);
3783 else if (is_reference (var
))
3785 /* For references that are being privatized for Fortran,
3786 allocate new backing storage for the new pointer
3787 variable. This allows us to avoid changing all the
3788 code that expects a pointer to something that expects
3789 a direct variable. */
3793 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3794 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3796 x
= build_receiver_ref (var
, false, ctx
);
3797 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3799 else if (TREE_CONSTANT (x
))
3801 /* For reduction in SIMD loop, defer adding the
3802 initialization of the reference, because if we decide
3803 to use SIMD array for it, the initilization could cause
3805 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3809 const char *name
= NULL
;
3810 if (DECL_NAME (var
))
3811 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3813 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3815 gimple_add_tmp_var (x
);
3816 TREE_ADDRESSABLE (x
) = 1;
3817 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3822 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3823 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3828 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3829 gimplify_assign (new_var
, x
, ilist
);
3832 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3834 else if (c_kind
== OMP_CLAUSE_REDUCTION
3835 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3843 switch (OMP_CLAUSE_CODE (c
))
3845 case OMP_CLAUSE_SHARED
:
3846 /* Ignore shared directives in teams construct. */
3847 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3849 /* Shared global vars are just accessed directly. */
3850 if (is_global_var (new_var
))
3852 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3853 needs to be delayed until after fixup_child_record_type so
3854 that we get the correct type during the dereference. */
3855 by_ref
= use_pointer_for_field (var
, ctx
);
3856 x
= build_receiver_ref (var
, by_ref
, ctx
);
3857 SET_DECL_VALUE_EXPR (new_var
, x
);
3858 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3860 /* ??? If VAR is not passed by reference, and the variable
3861 hasn't been initialized yet, then we'll get a warning for
3862 the store into the omp_data_s structure. Ideally, we'd be
3863 able to notice this and not store anything at all, but
3864 we're generating code too early. Suppress the warning. */
3866 TREE_NO_WARNING (var
) = 1;
3869 case OMP_CLAUSE_LASTPRIVATE
:
3870 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3874 case OMP_CLAUSE_PRIVATE
:
3875 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3876 x
= build_outer_var_ref (var
, ctx
);
3877 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3879 if (is_task_ctx (ctx
))
3880 x
= build_receiver_ref (var
, false, ctx
);
3882 x
= build_outer_var_ref (var
, ctx
);
3888 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3891 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3892 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3893 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3894 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3895 idx
, lane
, ivar
, lvar
))
3898 x
= lang_hooks
.decls
.omp_clause_default_ctor
3899 (c
, unshare_expr (ivar
), x
);
3901 gimplify_and_add (x
, &llist
[0]);
3904 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3907 gimple_seq tseq
= NULL
;
3910 gimplify_stmt (&dtor
, &tseq
);
3911 gimple_seq_add_seq (&llist
[1], tseq
);
3918 gimplify_and_add (nx
, ilist
);
3922 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3925 gimple_seq tseq
= NULL
;
3928 gimplify_stmt (&dtor
, &tseq
);
3929 gimple_seq_add_seq (dlist
, tseq
);
3933 case OMP_CLAUSE_LINEAR
:
3934 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3935 goto do_firstprivate
;
3936 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3939 x
= build_outer_var_ref (var
, ctx
);
3942 case OMP_CLAUSE_FIRSTPRIVATE
:
3943 if (is_task_ctx (ctx
))
3945 if (is_reference (var
) || is_variable_sized (var
))
3947 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3949 || use_pointer_for_field (var
, NULL
))
3951 x
= build_receiver_ref (var
, false, ctx
);
3952 SET_DECL_VALUE_EXPR (new_var
, x
);
3953 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3958 x
= build_outer_var_ref (var
, ctx
);
3961 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3962 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3964 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3965 tree stept
= TREE_TYPE (t
);
3966 tree ct
= find_omp_clause (clauses
,
3967 OMP_CLAUSE__LOOPTEMP_
);
3969 tree l
= OMP_CLAUSE_DECL (ct
);
3970 tree n1
= fd
->loop
.n1
;
3971 tree step
= fd
->loop
.step
;
3972 tree itype
= TREE_TYPE (l
);
3973 if (POINTER_TYPE_P (itype
))
3974 itype
= signed_type_for (itype
);
3975 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3976 if (TYPE_UNSIGNED (itype
)
3977 && fd
->loop
.cond_code
== GT_EXPR
)
3978 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3979 fold_build1 (NEGATE_EXPR
, itype
, l
),
3980 fold_build1 (NEGATE_EXPR
,
3983 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3984 t
= fold_build2 (MULT_EXPR
, stept
,
3985 fold_convert (stept
, l
), t
);
3987 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3989 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3991 gimplify_and_add (x
, ilist
);
3995 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3996 x
= fold_build2 (POINTER_PLUS_EXPR
,
3997 TREE_TYPE (x
), x
, t
);
3999 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
4002 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
4003 || TREE_ADDRESSABLE (new_var
))
4004 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4005 idx
, lane
, ivar
, lvar
))
4007 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
4009 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
4010 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
4011 gimplify_and_add (x
, ilist
);
4012 gimple_stmt_iterator gsi
4013 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4015 = gimple_build_assign (unshare_expr (lvar
), iv
);
4016 gsi_insert_before_without_update (&gsi
, g
,
4018 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4019 enum tree_code code
= PLUS_EXPR
;
4020 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4021 code
= POINTER_PLUS_EXPR
;
4022 g
= gimple_build_assign (iv
, code
, iv
, t
);
4023 gsi_insert_before_without_update (&gsi
, g
,
4027 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4028 (c
, unshare_expr (ivar
), x
);
4029 gimplify_and_add (x
, &llist
[0]);
4030 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4033 gimple_seq tseq
= NULL
;
4036 gimplify_stmt (&dtor
, &tseq
);
4037 gimple_seq_add_seq (&llist
[1], tseq
);
4042 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
4043 gimplify_and_add (x
, ilist
);
4046 case OMP_CLAUSE__LOOPTEMP_
:
4047 gcc_assert (is_parallel_ctx (ctx
));
4048 x
= build_outer_var_ref (var
, ctx
);
4049 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4050 gimplify_and_add (x
, ilist
);
4053 case OMP_CLAUSE_COPYIN
:
4054 by_ref
= use_pointer_for_field (var
, NULL
);
4055 x
= build_receiver_ref (var
, by_ref
, ctx
);
4056 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4057 append_to_statement_list (x
, ©in_seq
);
4058 copyin_by_ref
|= by_ref
;
4061 case OMP_CLAUSE_REDUCTION
:
4062 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4064 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4066 x
= build_outer_var_ref (var
, ctx
);
4068 if (is_reference (var
)
4069 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4071 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4072 SET_DECL_VALUE_EXPR (placeholder
, x
);
4073 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4074 tree new_vard
= new_var
;
4075 if (is_reference (var
))
4077 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4078 new_vard
= TREE_OPERAND (new_var
, 0);
4079 gcc_assert (DECL_P (new_vard
));
4082 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4083 idx
, lane
, ivar
, lvar
))
4085 if (new_vard
== new_var
)
4087 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4088 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4092 SET_DECL_VALUE_EXPR (new_vard
,
4093 build_fold_addr_expr (ivar
));
4094 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4096 x
= lang_hooks
.decls
.omp_clause_default_ctor
4097 (c
, unshare_expr (ivar
),
4098 build_outer_var_ref (var
, ctx
));
4100 gimplify_and_add (x
, &llist
[0]);
4101 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4103 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4104 lower_omp (&tseq
, ctx
);
4105 gimple_seq_add_seq (&llist
[0], tseq
);
4107 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4108 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4109 lower_omp (&tseq
, ctx
);
4110 gimple_seq_add_seq (&llist
[1], tseq
);
4111 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4112 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4113 if (new_vard
== new_var
)
4114 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4116 SET_DECL_VALUE_EXPR (new_vard
,
4117 build_fold_addr_expr (lvar
));
4118 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4123 gimplify_stmt (&dtor
, &tseq
);
4124 gimple_seq_add_seq (&llist
[1], tseq
);
4128 /* If this is a reference to constant size reduction var
4129 with placeholder, we haven't emitted the initializer
4130 for it because it is undesirable if SIMD arrays are used.
4131 But if they aren't used, we need to emit the deferred
4132 initialization now. */
4133 else if (is_reference (var
) && is_simd
)
4134 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4135 x
= lang_hooks
.decls
.omp_clause_default_ctor
4136 (c
, unshare_expr (new_var
),
4137 build_outer_var_ref (var
, ctx
));
4139 gimplify_and_add (x
, ilist
);
4140 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4142 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4143 lower_omp (&tseq
, ctx
);
4144 gimple_seq_add_seq (ilist
, tseq
);
4146 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4149 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4150 lower_omp (&tseq
, ctx
);
4151 gimple_seq_add_seq (dlist
, tseq
);
4152 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4154 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4159 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4160 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4161 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4163 /* reduction(-:var) sums up the partial results, so it
4164 acts identically to reduction(+:var). */
4165 if (code
== MINUS_EXPR
)
4168 tree new_vard
= new_var
;
4169 if (is_simd
&& is_reference (var
))
4171 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4172 new_vard
= TREE_OPERAND (new_var
, 0);
4173 gcc_assert (DECL_P (new_vard
));
4176 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4177 idx
, lane
, ivar
, lvar
))
4179 tree ref
= build_outer_var_ref (var
, ctx
);
4181 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
4183 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
4184 ref
= build_outer_var_ref (var
, ctx
);
4185 gimplify_assign (ref
, x
, &llist
[1]);
4187 if (new_vard
!= new_var
)
4189 SET_DECL_VALUE_EXPR (new_vard
,
4190 build_fold_addr_expr (lvar
));
4191 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4196 if (is_reference (var
) && is_simd
)
4197 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4198 gimplify_assign (new_var
, x
, ilist
);
4201 tree ref
= build_outer_var_ref (var
, ctx
);
4203 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4204 ref
= build_outer_var_ref (var
, ctx
);
4205 gimplify_assign (ref
, x
, dlist
);
4219 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
4220 /* Don't want uninit warnings on simduid, it is always uninitialized,
4221 but we use it not for the value, but for the DECL_UID only. */
4222 TREE_NO_WARNING (uid
) = 1;
4224 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
4225 gimple_call_set_lhs (g
, lane
);
4226 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4227 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
4228 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
4229 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
4230 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4231 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4232 g
= gimple_build_assign (lane
, INTEGER_CST
,
4233 build_int_cst (unsigned_type_node
, 0));
4234 gimple_seq_add_stmt (ilist
, g
);
4235 for (int i
= 0; i
< 2; i
++)
4238 tree vf
= create_tmp_var (unsigned_type_node
);
4239 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
4240 gimple_call_set_lhs (g
, vf
);
4241 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
4242 gimple_seq_add_stmt (seq
, g
);
4243 tree t
= build_int_cst (unsigned_type_node
, 0);
4244 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
4245 gimple_seq_add_stmt (seq
, g
);
4246 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4247 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
4248 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4249 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
4250 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
4251 gimple_seq_add_seq (seq
, llist
[i
]);
4252 t
= build_int_cst (unsigned_type_node
, 1);
4253 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
4254 gimple_seq_add_stmt (seq
, g
);
4255 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
4256 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
4257 gimple_seq_add_stmt (seq
, g
);
4258 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
4262 /* The copyin sequence is not to be executed by the main thread, since
4263 that would result in self-copies. Perhaps not visible to scalars,
4264 but it certainly is to C++ operator=. */
4267 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
4269 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
4270 build_int_cst (TREE_TYPE (x
), 0));
4271 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
4272 gimplify_and_add (x
, ilist
);
4275 /* If any copyin variable is passed by reference, we must ensure the
4276 master thread doesn't modify it before it is copied over in all
4277 threads. Similarly for variables in both firstprivate and
4278 lastprivate clauses we need to ensure the lastprivate copying
4279 happens after firstprivate copying in all threads. And similarly
4280 for UDRs if initializer expression refers to omp_orig. */
4281 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
4283 /* Don't add any barrier for #pragma omp simd or
4284 #pragma omp distribute. */
4285 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
4286 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
4287 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
4290 /* If max_vf is non-zero, then we can use only a vectorization factor
4291 up to the max_vf we chose. So stick it into the safelen clause. */
4294 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4295 OMP_CLAUSE_SAFELEN
);
4297 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
4298 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4301 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
4302 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
4304 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4305 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4311 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4312 both parallel and workshare constructs. PREDICATE may be NULL if it's
4316 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
4319 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
4320 bool par_clauses
= false;
4321 tree simduid
= NULL
, lastlane
= NULL
;
4323 /* Early exit if there are no lastprivate or linear clauses. */
4324 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
4325 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
4326 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
4327 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
4329 if (clauses
== NULL
)
4331 /* If this was a workshare clause, see if it had been combined
4332 with its parallel. In that case, look for the clauses on the
4333 parallel statement itself. */
4334 if (is_parallel_ctx (ctx
))
4338 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4341 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4342 OMP_CLAUSE_LASTPRIVATE
);
4343 if (clauses
== NULL
)
4351 tree label_true
, arm1
, arm2
;
4353 label
= create_artificial_label (UNKNOWN_LOCATION
);
4354 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
4355 arm1
= TREE_OPERAND (predicate
, 0);
4356 arm2
= TREE_OPERAND (predicate
, 1);
4357 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4358 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4359 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
4361 gimple_seq_add_stmt (stmt_list
, stmt
);
4362 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
4365 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4366 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4368 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
4370 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
4373 for (c
= clauses
; c
;)
4376 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4378 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4379 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4380 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
4382 var
= OMP_CLAUSE_DECL (c
);
4383 new_var
= lookup_decl (var
, ctx
);
4385 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
4387 tree val
= DECL_VALUE_EXPR (new_var
);
4388 if (TREE_CODE (val
) == ARRAY_REF
4389 && VAR_P (TREE_OPERAND (val
, 0))
4390 && lookup_attribute ("omp simd array",
4391 DECL_ATTRIBUTES (TREE_OPERAND (val
,
4394 if (lastlane
== NULL
)
4396 lastlane
= create_tmp_var (unsigned_type_node
);
4398 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4400 TREE_OPERAND (val
, 1));
4401 gimple_call_set_lhs (g
, lastlane
);
4402 gimple_seq_add_stmt (stmt_list
, g
);
4404 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4405 TREE_OPERAND (val
, 0), lastlane
,
4406 NULL_TREE
, NULL_TREE
);
4410 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4411 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4413 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4414 gimple_seq_add_seq (stmt_list
,
4415 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4416 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4418 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4419 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4421 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4422 gimple_seq_add_seq (stmt_list
,
4423 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4424 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4427 x
= build_outer_var_ref (var
, ctx
);
4428 if (is_reference (var
))
4429 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4430 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4431 gimplify_and_add (x
, stmt_list
);
4433 c
= OMP_CLAUSE_CHAIN (c
);
4434 if (c
== NULL
&& !par_clauses
)
4436 /* If this was a workshare clause, see if it had been combined
4437 with its parallel. In that case, continue looking for the
4438 clauses also on the parallel statement itself. */
4439 if (is_parallel_ctx (ctx
))
4443 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4446 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4447 OMP_CLAUSE_LASTPRIVATE
);
4453 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4457 oacc_lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
,
4458 tree tid
, tree var
, tree new_var
)
4460 /* The atomic add at the end of the sum creates unnecessary
4461 write contention on accelerators. To work around this,
4462 create an array to store the partial reductions. Later, in
4463 lower_omp_for (for openacc), the values of array will be
4466 tree t
= NULL_TREE
, array
, x
;
4467 tree type
= get_base_type (var
);
4470 /* Now insert the partial reductions into the array. */
4472 /* Find the reduction array. */
4474 tree ptype
= build_pointer_type (type
);
4476 t
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
), ctx
);
4477 t
= build_receiver_ref (t
, false, ctx
->outer
);
4479 array
= create_tmp_var (ptype
);
4480 gimplify_assign (array
, t
, stmt_seqp
);
4482 tree ptr
= create_tmp_var (TREE_TYPE (array
));
4484 /* Find the reduction array. */
4486 /* testing a unary conversion. */
4487 tree offset
= create_tmp_var (sizetype
);
4488 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
4490 t
= create_tmp_var (sizetype
);
4491 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
4493 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, t
);
4494 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4496 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4497 of adding sizeof(var) to the array? */
4498 ptr
= create_tmp_var (ptype
);
4499 stmt
= gimple_build_assign (unshare_expr (ptr
), POINTER_PLUS_EXPR
, array
,
4501 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4503 /* Move the local sum to gfc$sum[i]. */
4504 x
= unshare_expr (build_simple_mem_ref (ptr
));
4505 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
4508 /* Generate code to implement the REDUCTION clauses. */
4511 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4513 gimple_seq sub_seq
= NULL
;
4515 tree x
, c
, tid
= NULL_TREE
;
4518 /* SIMD reductions are handled in lower_rec_input_clauses. */
4519 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4520 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4523 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4524 update in that case, otherwise use a lock. */
4525 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4526 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4528 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4530 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4540 /* Initialize thread info for OpenACC. */
4541 if (is_gimple_omp_oacc (ctx
->stmt
))
4543 /* Get the current thread id. */
4544 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
4545 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)));
4546 gimple stmt
= gimple_build_call (call
, 0);
4547 gimple_call_set_lhs (stmt
, tid
);
4548 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4551 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4553 tree var
, ref
, new_var
;
4554 enum tree_code code
;
4555 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4557 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4560 var
= OMP_CLAUSE_DECL (c
);
4561 new_var
= lookup_decl (var
, ctx
);
4562 if (is_reference (var
))
4563 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4564 ref
= build_outer_var_ref (var
, ctx
);
4565 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4567 /* reduction(-:var) sums up the partial results, so it acts
4568 identically to reduction(+:var). */
4569 if (code
== MINUS_EXPR
)
4572 if (is_gimple_omp_oacc (ctx
->stmt
))
4574 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
));
4576 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
4578 else if (count
== 1)
4580 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4582 addr
= save_expr (addr
);
4583 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4584 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4585 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4586 gimplify_and_add (x
, stmt_seqp
);
4589 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4591 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4593 if (is_reference (var
)
4594 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4596 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4597 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4598 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4599 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4600 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4601 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4602 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4606 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4607 ref
= build_outer_var_ref (var
, ctx
);
4608 gimplify_assign (ref
, x
, &sub_seq
);
4612 if (is_gimple_omp_oacc (ctx
->stmt
))
4615 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4617 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4619 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4621 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4623 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4627 /* Generate code to implement the COPYPRIVATE clauses. */
4630 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4635 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4637 tree var
, new_var
, ref
, x
;
4639 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4641 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4644 var
= OMP_CLAUSE_DECL (c
);
4645 by_ref
= use_pointer_for_field (var
, NULL
);
4647 ref
= build_sender_ref (var
, ctx
);
4648 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4651 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4652 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4654 gimplify_assign (ref
, x
, slist
);
4656 ref
= build_receiver_ref (var
, false, ctx
);
4659 ref
= fold_convert_loc (clause_loc
,
4660 build_pointer_type (TREE_TYPE (new_var
)),
4662 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4664 if (is_reference (var
))
4666 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4667 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4668 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4670 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4671 gimplify_and_add (x
, rlist
);
4676 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4677 and REDUCTION from the sender (aka parent) side. */
4680 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4685 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4687 tree val
, ref
, x
, var
;
4688 bool by_ref
, do_in
= false, do_out
= false;
4689 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4691 switch (OMP_CLAUSE_CODE (c
))
4693 case OMP_CLAUSE_PRIVATE
:
4694 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4697 case OMP_CLAUSE_FIRSTPRIVATE
:
4698 case OMP_CLAUSE_COPYIN
:
4699 case OMP_CLAUSE_LASTPRIVATE
:
4700 case OMP_CLAUSE_REDUCTION
:
4701 case OMP_CLAUSE__LOOPTEMP_
:
4707 val
= OMP_CLAUSE_DECL (c
);
4708 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4710 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4711 && is_global_var (var
))
4713 if (is_variable_sized (val
))
4715 by_ref
= use_pointer_for_field (val
, NULL
);
4717 switch (OMP_CLAUSE_CODE (c
))
4719 case OMP_CLAUSE_PRIVATE
:
4720 case OMP_CLAUSE_FIRSTPRIVATE
:
4721 case OMP_CLAUSE_COPYIN
:
4722 case OMP_CLAUSE__LOOPTEMP_
:
4726 case OMP_CLAUSE_LASTPRIVATE
:
4727 if (by_ref
|| is_reference (val
))
4729 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4736 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4741 case OMP_CLAUSE_REDUCTION
:
4743 do_out
= !(by_ref
|| is_reference (val
));
4752 ref
= build_sender_ref (val
, ctx
);
4753 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4754 gimplify_assign (ref
, x
, ilist
);
4755 if (is_task_ctx (ctx
))
4756 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4761 ref
= build_sender_ref (val
, ctx
);
4762 gimplify_assign (var
, ref
, olist
);
4767 /* Generate code to implement SHARED from the sender (aka parent)
4768 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4769 list things that got automatically shared. */
4772 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4774 tree var
, ovar
, nvar
, f
, x
, record_type
;
4776 if (ctx
->record_type
== NULL
)
4779 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4780 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4782 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4783 nvar
= maybe_lookup_decl (ovar
, ctx
);
4784 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4787 /* If CTX is a nested parallel directive. Find the immediately
4788 enclosing parallel or workshare construct that contains a
4789 mapping for OVAR. */
4790 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4792 if (use_pointer_for_field (ovar
, ctx
))
4794 x
= build_sender_ref (ovar
, ctx
);
4795 var
= build_fold_addr_expr (var
);
4796 gimplify_assign (x
, var
, ilist
);
4800 x
= build_sender_ref (ovar
, ctx
);
4801 gimplify_assign (x
, var
, ilist
);
4803 if (!TREE_READONLY (var
)
4804 /* We don't need to receive a new reference to a result
4805 or parm decl. In fact we may not store to it as we will
4806 invalidate any pending RSO and generate wrong gimple
4808 && !((TREE_CODE (var
) == RESULT_DECL
4809 || TREE_CODE (var
) == PARM_DECL
)
4810 && DECL_BY_REFERENCE (var
)))
4812 x
= build_sender_ref (ovar
, ctx
);
4813 gimplify_assign (var
, x
, olist
);
4820 /* A convenience function to build an empty GIMPLE_COND with just the
4824 gimple_build_cond_empty (tree cond
)
4826 enum tree_code pred_code
;
4829 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4830 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4834 /* Build the function calls to GOMP_parallel_start etc to actually
4835 generate the parallel operation. REGION is the parallel region
4836 being expanded. BB is the block where to insert the code. WS_ARGS
4837 will be set if this is a call to a combined parallel+workshare
4838 construct, it contains the list of additional arguments needed by
4839 the workshare construct. */
4842 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4843 gomp_parallel
*entry_stmt
,
4844 vec
<tree
, va_gc
> *ws_args
)
4846 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4847 gimple_stmt_iterator gsi
;
4849 enum built_in_function start_ix
;
4851 location_t clause_loc
;
4852 vec
<tree
, va_gc
> *args
;
4854 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4856 /* Determine what flavor of GOMP_parallel we will be
4858 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4859 if (is_combined_parallel (region
))
4861 switch (region
->inner
->type
)
4863 case GIMPLE_OMP_FOR
:
4864 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4865 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4866 + (region
->inner
->sched_kind
4867 == OMP_CLAUSE_SCHEDULE_RUNTIME
4868 ? 3 : region
->inner
->sched_kind
));
4869 start_ix
= (enum built_in_function
)start_ix2
;
4871 case GIMPLE_OMP_SECTIONS
:
4872 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4879 /* By default, the value of NUM_THREADS is zero (selected at run time)
4880 and there is no conditional. */
4882 val
= build_int_cst (unsigned_type_node
, 0);
4883 flags
= build_int_cst (unsigned_type_node
, 0);
4885 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4887 cond
= OMP_CLAUSE_IF_EXPR (c
);
4889 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4892 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4893 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4896 clause_loc
= gimple_location (entry_stmt
);
4898 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4900 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4902 /* Ensure 'val' is of the correct type. */
4903 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4905 /* If we found the clause 'if (cond)', build either
4906 (cond != 0) or (cond ? val : 1u). */
4909 cond
= gimple_boolify (cond
);
4911 if (integer_zerop (val
))
4912 val
= fold_build2_loc (clause_loc
,
4913 EQ_EXPR
, unsigned_type_node
, cond
,
4914 build_int_cst (TREE_TYPE (cond
), 0));
4917 basic_block cond_bb
, then_bb
, else_bb
;
4918 edge e
, e_then
, e_else
;
4919 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4921 tmp_var
= create_tmp_var (TREE_TYPE (val
));
4922 if (gimple_in_ssa_p (cfun
))
4924 tmp_then
= make_ssa_name (tmp_var
);
4925 tmp_else
= make_ssa_name (tmp_var
);
4926 tmp_join
= make_ssa_name (tmp_var
);
4935 e
= split_block (bb
, NULL
);
4940 then_bb
= create_empty_bb (cond_bb
);
4941 else_bb
= create_empty_bb (then_bb
);
4942 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4943 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4945 stmt
= gimple_build_cond_empty (cond
);
4946 gsi
= gsi_start_bb (cond_bb
);
4947 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4949 gsi
= gsi_start_bb (then_bb
);
4950 stmt
= gimple_build_assign (tmp_then
, val
);
4951 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4953 gsi
= gsi_start_bb (else_bb
);
4954 stmt
= gimple_build_assign
4955 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4956 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4958 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4959 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4960 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4961 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4962 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4963 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4965 if (gimple_in_ssa_p (cfun
))
4967 gphi
*phi
= create_phi_node (tmp_join
, bb
);
4968 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4969 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4975 gsi
= gsi_start_bb (bb
);
4976 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4977 false, GSI_CONTINUE_LINKING
);
4980 gsi
= gsi_last_bb (bb
);
4981 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4983 t1
= null_pointer_node
;
4985 t1
= build_fold_addr_expr (t
);
4986 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4988 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4989 args
->quick_push (t2
);
4990 args
->quick_push (t1
);
4991 args
->quick_push (val
);
4993 args
->splice (*ws_args
);
4994 args
->quick_push (flags
);
4996 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4997 builtin_decl_explicit (start_ix
), args
);
4999 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5000 false, GSI_CONTINUE_LINKING
);
5003 /* Insert a function call whose name is FUNC_NAME with the information from
5004 ENTRY_STMT into the basic_block BB. */
5007 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
5008 vec
<tree
, va_gc
> *ws_args
)
5011 gimple_stmt_iterator gsi
;
5012 vec
<tree
, va_gc
> *args
;
5014 gcc_assert (vec_safe_length (ws_args
) == 2);
5015 tree func_name
= (*ws_args
)[0];
5016 tree grain
= (*ws_args
)[1];
5018 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
5019 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
5020 gcc_assert (count
!= NULL_TREE
);
5021 count
= OMP_CLAUSE_OPERAND (count
, 0);
5023 gsi
= gsi_last_bb (bb
);
5024 t
= gimple_omp_parallel_data_arg (entry_stmt
);
5026 t1
= null_pointer_node
;
5028 t1
= build_fold_addr_expr (t
);
5029 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5031 vec_alloc (args
, 4);
5032 args
->quick_push (t2
);
5033 args
->quick_push (t1
);
5034 args
->quick_push (count
);
5035 args
->quick_push (grain
);
5036 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
5038 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
5039 GSI_CONTINUE_LINKING
);
5042 /* Build the function call to GOMP_task to actually
5043 generate the task operation. BB is the block where to insert the code. */
5046 expand_task_call (basic_block bb
, gomp_task
*entry_stmt
)
5048 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
5049 gimple_stmt_iterator gsi
;
5050 location_t loc
= gimple_location (entry_stmt
);
5052 clauses
= gimple_omp_task_clauses (entry_stmt
);
5054 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
5056 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
5058 cond
= boolean_true_node
;
5060 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
5061 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
5062 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
5063 flags
= build_int_cst (unsigned_type_node
,
5064 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
5066 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
5069 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
5070 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
5071 build_int_cst (unsigned_type_node
, 2),
5072 build_int_cst (unsigned_type_node
, 0));
5073 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
5076 depend
= OMP_CLAUSE_DECL (depend
);
5078 depend
= build_int_cst (ptr_type_node
, 0);
5080 gsi
= gsi_last_bb (bb
);
5081 t
= gimple_omp_task_data_arg (entry_stmt
);
5083 t2
= null_pointer_node
;
5085 t2
= build_fold_addr_expr_loc (loc
, t
);
5086 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
5087 t
= gimple_omp_task_copy_fn (entry_stmt
);
5089 t3
= null_pointer_node
;
5091 t3
= build_fold_addr_expr_loc (loc
, t
);
5093 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
5095 gimple_omp_task_arg_size (entry_stmt
),
5096 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
5099 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5100 false, GSI_CONTINUE_LINKING
);
5104 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5105 catch handler and return it. This prevents programs from violating the
5106 structured block semantics with throws. */
5109 maybe_catch_exception (gimple_seq body
)
5114 if (!flag_exceptions
)
5117 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
5118 decl
= lang_hooks
.eh_protect_cleanup_actions ();
5120 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
5122 g
= gimple_build_eh_must_not_throw (decl
);
5123 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
5126 return gimple_seq_alloc_with_stmt (g
);
5129 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5132 vec2chain (vec
<tree
, va_gc
> *v
)
5134 tree chain
= NULL_TREE
, t
;
5137 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
5139 DECL_CHAIN (t
) = chain
;
5147 /* Remove barriers in REGION->EXIT's block. Note that this is only
5148 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5149 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5150 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5154 remove_exit_barrier (struct omp_region
*region
)
5156 gimple_stmt_iterator gsi
;
5157 basic_block exit_bb
;
5161 int any_addressable_vars
= -1;
5163 exit_bb
= region
->exit
;
5165 /* If the parallel region doesn't return, we don't have REGION->EXIT
5170 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5171 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5172 statements that can appear in between are extremely limited -- no
5173 memory operations at all. Here, we allow nothing at all, so the
5174 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5175 gsi
= gsi_last_bb (exit_bb
);
5176 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5178 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
5181 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
5183 gsi
= gsi_last_bb (e
->src
);
5184 if (gsi_end_p (gsi
))
5186 stmt
= gsi_stmt (gsi
);
5187 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
5188 && !gimple_omp_return_nowait_p (stmt
))
5190 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5191 in many cases. If there could be tasks queued, the barrier
5192 might be needed to let the tasks run before some local
5193 variable of the parallel that the task uses as shared
5194 runs out of scope. The task can be spawned either
5195 from within current function (this would be easy to check)
5196 or from some function it calls and gets passed an address
5197 of such a variable. */
5198 if (any_addressable_vars
< 0)
5200 gomp_parallel
*parallel_stmt
5201 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
5202 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
5203 tree local_decls
, block
, decl
;
5206 any_addressable_vars
= 0;
5207 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
5208 if (TREE_ADDRESSABLE (decl
))
5210 any_addressable_vars
= 1;
5213 for (block
= gimple_block (stmt
);
5214 !any_addressable_vars
5216 && TREE_CODE (block
) == BLOCK
;
5217 block
= BLOCK_SUPERCONTEXT (block
))
5219 for (local_decls
= BLOCK_VARS (block
);
5221 local_decls
= DECL_CHAIN (local_decls
))
5222 if (TREE_ADDRESSABLE (local_decls
))
5224 any_addressable_vars
= 1;
5227 if (block
== gimple_block (parallel_stmt
))
5231 if (!any_addressable_vars
)
5232 gimple_omp_return_set_nowait (stmt
);
5238 remove_exit_barriers (struct omp_region
*region
)
5240 if (region
->type
== GIMPLE_OMP_PARALLEL
)
5241 remove_exit_barrier (region
);
5245 region
= region
->inner
;
5246 remove_exit_barriers (region
);
5247 while (region
->next
)
5249 region
= region
->next
;
5250 remove_exit_barriers (region
);
5255 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5256 calls. These can't be declared as const functions, but
5257 within one parallel body they are constant, so they can be
5258 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5259 which are declared const. Similarly for task body, except
5260 that in untied task omp_get_thread_num () can change at any task
5261 scheduling point. */
5264 optimize_omp_library_calls (gimple entry_stmt
)
5267 gimple_stmt_iterator gsi
;
5268 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5269 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
5270 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5271 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
5272 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
5273 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
5274 OMP_CLAUSE_UNTIED
) != NULL
);
5276 FOR_EACH_BB_FN (bb
, cfun
)
5277 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5279 gimple call
= gsi_stmt (gsi
);
5282 if (is_gimple_call (call
)
5283 && (decl
= gimple_call_fndecl (call
))
5284 && DECL_EXTERNAL (decl
)
5285 && TREE_PUBLIC (decl
)
5286 && DECL_INITIAL (decl
) == NULL
)
5290 if (DECL_NAME (decl
) == thr_num_id
)
5292 /* In #pragma omp task untied omp_get_thread_num () can change
5293 during the execution of the task region. */
5296 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5298 else if (DECL_NAME (decl
) == num_thr_id
)
5299 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5303 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
5304 || gimple_call_num_args (call
) != 0)
5307 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
5310 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
5311 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
5312 TREE_TYPE (TREE_TYPE (built_in
))))
5315 gimple_call_set_fndecl (call
, built_in
);
5320 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5324 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
5328 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5329 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
5332 if (TREE_CODE (t
) == ADDR_EXPR
)
5333 recompute_tree_invariant_for_addr_expr (t
);
5335 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
5339 /* Prepend TO = FROM assignment before *GSI_P. */
5342 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
5344 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
5345 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
5346 true, GSI_SAME_STMT
);
5347 gimple stmt
= gimple_build_assign (to
, from
);
5348 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
5349 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
5350 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
5352 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
5353 gimple_regimplify_operands (stmt
, &gsi
);
5357 /* Expand the OpenMP parallel or task directive starting at REGION. */
5360 expand_omp_taskreg (struct omp_region
*region
)
5362 basic_block entry_bb
, exit_bb
, new_bb
;
5363 struct function
*child_cfun
;
5364 tree child_fn
, block
, t
;
5365 gimple_stmt_iterator gsi
;
5366 gimple entry_stmt
, stmt
;
5368 vec
<tree
, va_gc
> *ws_args
;
5370 entry_stmt
= last_stmt (region
->entry
);
5371 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
5372 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
5374 entry_bb
= region
->entry
;
5375 exit_bb
= region
->exit
;
5379 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
5380 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
5381 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
5384 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5385 and the inner statement contains the name of the built-in function
5387 ws_args
= region
->inner
->ws_args
;
5388 else if (is_combined_parallel (region
))
5389 ws_args
= region
->ws_args
;
5393 if (child_cfun
->cfg
)
5395 /* Due to inlining, it may happen that we have already outlined
5396 the region, in which case all we need to do is make the
5397 sub-graph unreachable and emit the parallel call. */
5398 edge entry_succ_e
, exit_succ_e
;
5400 entry_succ_e
= single_succ_edge (entry_bb
);
5402 gsi
= gsi_last_bb (entry_bb
);
5403 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
5404 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
5405 gsi_remove (&gsi
, true);
5410 exit_succ_e
= single_succ_edge (exit_bb
);
5411 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
5413 remove_edge_and_dominated_blocks (entry_succ_e
);
5417 unsigned srcidx
, dstidx
, num
;
5419 /* If the parallel region needs data sent from the parent
5420 function, then the very first statement (except possible
5421 tree profile counter updates) of the parallel body
5422 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5423 &.OMP_DATA_O is passed as an argument to the child function,
5424 we need to replace it with the argument as seen by the child
5427 In most cases, this will end up being the identity assignment
5428 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5429 a function call that has been inlined, the original PARM_DECL
5430 .OMP_DATA_I may have been converted into a different local
5431 variable. In which case, we need to keep the assignment. */
5432 if (gimple_omp_taskreg_data_arg (entry_stmt
))
5434 basic_block entry_succ_bb
= single_succ (entry_bb
);
5436 gimple parcopy_stmt
= NULL
;
5438 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
5442 gcc_assert (!gsi_end_p (gsi
));
5443 stmt
= gsi_stmt (gsi
);
5444 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
5447 if (gimple_num_ops (stmt
) == 2)
5449 tree arg
= gimple_assign_rhs1 (stmt
);
5451 /* We're ignore the subcode because we're
5452 effectively doing a STRIP_NOPS. */
5454 if (TREE_CODE (arg
) == ADDR_EXPR
5455 && TREE_OPERAND (arg
, 0)
5456 == gimple_omp_taskreg_data_arg (entry_stmt
))
5458 parcopy_stmt
= stmt
;
5464 gcc_assert (parcopy_stmt
!= NULL
);
5465 arg
= DECL_ARGUMENTS (child_fn
);
5467 if (!gimple_in_ssa_p (cfun
))
5469 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5470 gsi_remove (&gsi
, true);
5473 /* ?? Is setting the subcode really necessary ?? */
5474 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5475 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5480 /* If we are in ssa form, we must load the value from the default
5481 definition of the argument. That should not be defined now,
5482 since the argument is not used uninitialized. */
5483 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5484 narg
= make_ssa_name (arg
, gimple_build_nop ());
5485 set_ssa_default_def (cfun
, arg
, narg
);
5486 /* ?? Is setting the subcode really necessary ?? */
5487 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5488 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5489 update_stmt (parcopy_stmt
);
5493 /* Declare local variables needed in CHILD_CFUN. */
5494 block
= DECL_INITIAL (child_fn
);
5495 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5496 /* The gimplifier could record temporaries in parallel/task block
5497 rather than in containing function's local_decls chain,
5498 which would mean cgraph missed finalizing them. Do it now. */
5499 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5500 if (TREE_CODE (t
) == VAR_DECL
5502 && !DECL_EXTERNAL (t
))
5503 varpool_node::finalize_decl (t
);
5504 DECL_SAVED_TREE (child_fn
) = NULL
;
5505 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5506 gimple_set_body (child_fn
, NULL
);
5507 TREE_USED (block
) = 1;
5509 /* Reset DECL_CONTEXT on function arguments. */
5510 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5511 DECL_CONTEXT (t
) = child_fn
;
5513 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5514 so that it can be moved to the child function. */
5515 gsi
= gsi_last_bb (entry_bb
);
5516 stmt
= gsi_stmt (gsi
);
5517 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5518 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5519 e
= split_block (entry_bb
, stmt
);
5520 gsi_remove (&gsi
, true);
5522 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5524 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5527 gsi
= gsi_last_bb (exit_bb
);
5528 gcc_assert (!gsi_end_p (gsi
)
5529 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5530 stmt
= gimple_build_return (NULL
);
5531 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5532 gsi_remove (&gsi
, true);
5535 /* Move the parallel region into CHILD_CFUN. */
5537 if (gimple_in_ssa_p (cfun
))
5539 init_tree_ssa (child_cfun
);
5540 init_ssa_operands (child_cfun
);
5541 child_cfun
->gimple_df
->in_ssa_p
= true;
5545 block
= gimple_block (entry_stmt
);
5547 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5549 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5550 /* When the OMP expansion process cannot guarantee an up-to-date
5551 loop tree arrange for the child function to fixup loops. */
5552 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5553 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5555 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5556 num
= vec_safe_length (child_cfun
->local_decls
);
5557 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5559 t
= (*child_cfun
->local_decls
)[srcidx
];
5560 if (DECL_CONTEXT (t
) == cfun
->decl
)
5562 if (srcidx
!= dstidx
)
5563 (*child_cfun
->local_decls
)[dstidx
] = t
;
5567 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5569 /* Inform the callgraph about the new function. */
5570 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5571 cgraph_node::add_new_function (child_fn
, true);
5573 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5574 fixed in a following pass. */
5575 push_cfun (child_cfun
);
5577 optimize_omp_library_calls (entry_stmt
);
5578 cgraph_edge::rebuild_edges ();
5580 /* Some EH regions might become dead, see PR34608. If
5581 pass_cleanup_cfg isn't the first pass to happen with the
5582 new child, these dead EH edges might cause problems.
5583 Clean them up now. */
5584 if (flag_exceptions
)
5587 bool changed
= false;
5589 FOR_EACH_BB_FN (bb
, cfun
)
5590 changed
|= gimple_purge_dead_eh_edges (bb
);
5592 cleanup_tree_cfg ();
5594 if (gimple_in_ssa_p (cfun
))
5595 update_ssa (TODO_update_ssa
);
5599 /* Emit a library call to launch the children threads. */
5601 expand_cilk_for_call (new_bb
,
5602 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5603 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5604 expand_parallel_call (region
, new_bb
,
5605 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5607 expand_task_call (new_bb
, as_a
<gomp_task
*> (entry_stmt
));
5608 if (gimple_in_ssa_p (cfun
))
5609 update_ssa (TODO_update_ssa_only_virtuals
);
5613 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5614 of the combined collapse > 1 loop constructs, generate code like:
5615 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5620 count3 = (adj + N32 - N31) / STEP3;
5621 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5626 count2 = (adj + N22 - N21) / STEP2;
5627 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5632 count1 = (adj + N12 - N11) / STEP1;
5633 count = count1 * count2 * count3;
5634 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5636 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5637 of the combined loop constructs, just initialize COUNTS array
5638 from the _looptemp_ clauses. */
5640 /* NOTE: It *could* be better to moosh all of the BBs together,
5641 creating one larger BB with all the computation and the unexpected
5642 jump at the end. I.e.
5644 bool zero3, zero2, zero1, zero;
5647 count3 = (N32 - N31) /[cl] STEP3;
5649 count2 = (N22 - N21) /[cl] STEP2;
5651 count1 = (N12 - N11) /[cl] STEP1;
5652 zero = zero3 || zero2 || zero1;
5653 count = count1 * count2 * count3;
5654 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5656 After all, we expect the zero=false, and thus we expect to have to
5657 evaluate all of the comparison expressions, so short-circuiting
5658 oughtn't be a win. Since the condition isn't protecting a
5659 denominator, we're not concerned about divide-by-zero, so we can
5660 fully evaluate count even if a numerator turned out to be wrong.
5662 It seems like putting this all together would create much better
5663 scheduling opportunities, and less pressure on the chip's branch
5667 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5668 basic_block
&entry_bb
, tree
*counts
,
5669 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5670 basic_block
&l2_dom_bb
)
5672 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5676 /* Collapsed loops need work for expansion into SSA form. */
5677 gcc_assert (!gimple_in_ssa_p (cfun
));
5679 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5680 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5682 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5683 isn't supposed to be handled, as the inner loop doesn't
5685 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5686 OMP_CLAUSE__LOOPTEMP_
);
5687 gcc_assert (innerc
);
5688 for (i
= 0; i
< fd
->collapse
; i
++)
5690 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5691 OMP_CLAUSE__LOOPTEMP_
);
5692 gcc_assert (innerc
);
5694 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5696 counts
[0] = NULL_TREE
;
5701 for (i
= 0; i
< fd
->collapse
; i
++)
5703 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5705 if (SSA_VAR_P (fd
->loop
.n2
)
5706 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5707 fold_convert (itype
, fd
->loops
[i
].n1
),
5708 fold_convert (itype
, fd
->loops
[i
].n2
)))
5709 == NULL_TREE
|| !integer_onep (t
)))
5713 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5714 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5715 true, GSI_SAME_STMT
);
5716 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5717 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5718 true, GSI_SAME_STMT
);
5719 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5720 NULL_TREE
, NULL_TREE
);
5721 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
5722 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
5723 expand_omp_regimplify_p
, NULL
, NULL
)
5724 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
5725 expand_omp_regimplify_p
, NULL
, NULL
))
5727 *gsi
= gsi_for_stmt (cond_stmt
);
5728 gimple_regimplify_operands (cond_stmt
, gsi
);
5730 e
= split_block (entry_bb
, cond_stmt
);
5731 if (zero_iter_bb
== NULL
)
5733 gassign
*assign_stmt
;
5734 first_zero_iter
= i
;
5735 zero_iter_bb
= create_empty_bb (entry_bb
);
5736 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5737 *gsi
= gsi_after_labels (zero_iter_bb
);
5738 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
5739 build_zero_cst (type
));
5740 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
5741 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5744 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5745 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5746 e
->flags
= EDGE_TRUE_VALUE
;
5747 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5748 if (l2_dom_bb
== NULL
)
5749 l2_dom_bb
= entry_bb
;
5751 *gsi
= gsi_last_bb (entry_bb
);
5754 if (POINTER_TYPE_P (itype
))
5755 itype
= signed_type_for (itype
);
5756 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5758 t
= fold_build2 (PLUS_EXPR
, itype
,
5759 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5760 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5761 fold_convert (itype
, fd
->loops
[i
].n2
));
5762 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5763 fold_convert (itype
, fd
->loops
[i
].n1
));
5764 /* ?? We could probably use CEIL_DIV_EXPR instead of
5765 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5766 generate the same code in the end because generically we
5767 don't know that the values involved must be negative for
5769 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5770 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5771 fold_build1 (NEGATE_EXPR
, itype
, t
),
5772 fold_build1 (NEGATE_EXPR
, itype
,
5773 fold_convert (itype
,
5774 fd
->loops
[i
].step
)));
5776 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5777 fold_convert (itype
, fd
->loops
[i
].step
));
5778 t
= fold_convert (type
, t
);
5779 if (TREE_CODE (t
) == INTEGER_CST
)
5783 counts
[i
] = create_tmp_reg (type
, ".count");
5784 expand_omp_build_assign (gsi
, counts
[i
], t
);
5786 if (SSA_VAR_P (fd
->loop
.n2
))
5791 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5792 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5798 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5800 V3 = N31 + (T % count3) * STEP3;
5802 V2 = N21 + (T % count2) * STEP2;
5804 V1 = N11 + T * STEP1;
5805 if this loop doesn't have an inner loop construct combined with it.
5806 If it does have an inner loop construct combined with it and the
5807 iteration count isn't known constant, store values from counts array
5808 into its _looptemp_ temporaries instead. */
5811 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5812 tree
*counts
, gimple inner_stmt
, tree startvar
)
5815 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5817 /* If fd->loop.n2 is constant, then no propagation of the counts
5818 is needed, they are constant. */
5819 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5822 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5823 ? gimple_omp_parallel_clauses (inner_stmt
)
5824 : gimple_omp_for_clauses (inner_stmt
);
5825 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5826 isn't supposed to be handled, as the inner loop doesn't
5828 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5829 gcc_assert (innerc
);
5830 for (i
= 0; i
< fd
->collapse
; i
++)
5832 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5833 OMP_CLAUSE__LOOPTEMP_
);
5834 gcc_assert (innerc
);
5837 tree tem
= OMP_CLAUSE_DECL (innerc
);
5838 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5839 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5840 false, GSI_CONTINUE_LINKING
);
5841 gassign
*stmt
= gimple_build_assign (tem
, t
);
5842 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5848 tree type
= TREE_TYPE (fd
->loop
.v
);
5849 tree tem
= create_tmp_reg (type
, ".tem");
5850 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
5851 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5853 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5855 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5857 if (POINTER_TYPE_P (vtype
))
5858 itype
= signed_type_for (vtype
);
5860 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5863 t
= fold_convert (itype
, t
);
5864 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5865 fold_convert (itype
, fd
->loops
[i
].step
));
5866 if (POINTER_TYPE_P (vtype
))
5867 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5869 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5870 t
= force_gimple_operand_gsi (gsi
, t
,
5871 DECL_P (fd
->loops
[i
].v
)
5872 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5874 GSI_CONTINUE_LINKING
);
5875 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5876 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5879 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5880 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5881 false, GSI_CONTINUE_LINKING
);
5882 stmt
= gimple_build_assign (tem
, t
);
5883 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5889 /* Helper function for expand_omp_for_*. Generate code like:
5892 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5896 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5903 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5904 basic_block body_bb
)
5906 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5908 gimple_stmt_iterator gsi
;
5914 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5916 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5918 bb
= create_empty_bb (last_bb
);
5919 add_bb_to_loop (bb
, last_bb
->loop_father
);
5920 gsi
= gsi_start_bb (bb
);
5922 if (i
< fd
->collapse
- 1)
5924 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5925 e
->probability
= REG_BR_PROB_BASE
/ 8;
5927 t
= fd
->loops
[i
+ 1].n1
;
5928 t
= force_gimple_operand_gsi (&gsi
, t
,
5929 DECL_P (fd
->loops
[i
+ 1].v
)
5930 && TREE_ADDRESSABLE (fd
->loops
[i
5933 GSI_CONTINUE_LINKING
);
5934 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5935 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5940 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5942 if (POINTER_TYPE_P (vtype
))
5943 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5945 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5946 t
= force_gimple_operand_gsi (&gsi
, t
,
5947 DECL_P (fd
->loops
[i
].v
)
5948 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5949 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5950 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5951 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5955 t
= fd
->loops
[i
].n2
;
5956 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5957 false, GSI_CONTINUE_LINKING
);
5958 tree v
= fd
->loops
[i
].v
;
5959 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5960 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5961 false, GSI_CONTINUE_LINKING
);
5962 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5963 stmt
= gimple_build_cond_empty (t
);
5964 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5965 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5966 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5969 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5977 /* A subroutine of expand_omp_for. Generate code for a parallel
5978 loop with any schedule. Given parameters:
5980 for (V = N1; V cond N2; V += STEP) BODY;
5982 where COND is "<" or ">", we generate pseudocode
5984 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5985 if (more) goto L0; else goto L3;
5992 if (V cond iend) goto L1; else goto L2;
5994 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5997 If this is a combined omp parallel loop, instead of the call to
5998 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5999 If this is gimple_omp_for_combined_p loop, then instead of assigning
6000 V and iend in L0 we assign the first two _looptemp_ clause decls of the
6001 inner GIMPLE_OMP_FOR and V += STEP; and
6002 if (V cond iend) goto L1; else goto L2; are removed.
6004 For collapsed loops, given parameters:
6006 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6007 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6008 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6011 we generate pseudocode
6013 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6018 count3 = (adj + N32 - N31) / STEP3;
6019 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6024 count2 = (adj + N22 - N21) / STEP2;
6025 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6030 count1 = (adj + N12 - N11) / STEP1;
6031 count = count1 * count2 * count3;
6036 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6037 if (more) goto L0; else goto L3;
6041 V3 = N31 + (T % count3) * STEP3;
6043 V2 = N21 + (T % count2) * STEP2;
6045 V1 = N11 + T * STEP1;
6050 if (V < iend) goto L10; else goto L2;
6053 if (V3 cond3 N32) goto L1; else goto L11;
6057 if (V2 cond2 N22) goto L1; else goto L12;
6063 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6069 expand_omp_for_generic (struct omp_region
*region
,
6070 struct omp_for_data
*fd
,
6071 enum built_in_function start_fn
,
6072 enum built_in_function next_fn
,
6075 tree type
, istart0
, iend0
, iend
;
6076 tree t
, vmain
, vback
, bias
= NULL_TREE
;
6077 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
6078 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
6079 gimple_stmt_iterator gsi
;
6080 gassign
*assign_stmt
;
6081 bool in_combined_parallel
= is_combined_parallel (region
);
6082 bool broken_loop
= region
->cont
== NULL
;
6084 tree
*counts
= NULL
;
6087 gcc_assert (!broken_loop
|| !in_combined_parallel
);
6088 gcc_assert (fd
->iter_type
== long_integer_type_node
6089 || !in_combined_parallel
);
6091 type
= TREE_TYPE (fd
->loop
.v
);
6092 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
6093 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
6094 TREE_ADDRESSABLE (istart0
) = 1;
6095 TREE_ADDRESSABLE (iend0
) = 1;
6097 /* See if we need to bias by LLONG_MIN. */
6098 if (fd
->iter_type
== long_long_unsigned_type_node
6099 && TREE_CODE (type
) == INTEGER_TYPE
6100 && !TYPE_UNSIGNED (type
))
6104 if (fd
->loop
.cond_code
== LT_EXPR
)
6107 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6111 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6114 if (TREE_CODE (n1
) != INTEGER_CST
6115 || TREE_CODE (n2
) != INTEGER_CST
6116 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
6117 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
6120 entry_bb
= region
->entry
;
6121 cont_bb
= region
->cont
;
6123 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6124 gcc_assert (broken_loop
6125 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6126 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6127 l1_bb
= single_succ (l0_bb
);
6130 l2_bb
= create_empty_bb (cont_bb
);
6131 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
6132 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6136 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6137 exit_bb
= region
->exit
;
6139 gsi
= gsi_last_bb (entry_bb
);
6141 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6142 if (fd
->collapse
> 1)
6144 int first_zero_iter
= -1;
6145 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
6147 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6148 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6149 zero_iter_bb
, first_zero_iter
,
6154 /* Some counts[i] vars might be uninitialized if
6155 some loop has zero iterations. But the body shouldn't
6156 be executed in that case, so just avoid uninit warnings. */
6157 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
6158 if (SSA_VAR_P (counts
[i
]))
6159 TREE_NO_WARNING (counts
[i
]) = 1;
6161 e
= split_block (entry_bb
, gsi_stmt (gsi
));
6163 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
6164 gsi
= gsi_last_bb (entry_bb
);
6165 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
6166 get_immediate_dominator (CDI_DOMINATORS
,
6170 if (in_combined_parallel
)
6172 /* In a combined parallel loop, emit a call to
6173 GOMP_loop_foo_next. */
6174 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6175 build_fold_addr_expr (istart0
),
6176 build_fold_addr_expr (iend0
));
6180 tree t0
, t1
, t2
, t3
, t4
;
6181 /* If this is not a combined parallel loop, emit a call to
6182 GOMP_loop_foo_start in ENTRY_BB. */
6183 t4
= build_fold_addr_expr (iend0
);
6184 t3
= build_fold_addr_expr (istart0
);
6185 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
6188 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6190 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6191 OMP_CLAUSE__LOOPTEMP_
);
6192 gcc_assert (innerc
);
6193 t0
= OMP_CLAUSE_DECL (innerc
);
6194 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6195 OMP_CLAUSE__LOOPTEMP_
);
6196 gcc_assert (innerc
);
6197 t1
= OMP_CLAUSE_DECL (innerc
);
6199 if (POINTER_TYPE_P (TREE_TYPE (t0
))
6200 && TYPE_PRECISION (TREE_TYPE (t0
))
6201 != TYPE_PRECISION (fd
->iter_type
))
6203 /* Avoid casting pointers to integer of a different size. */
6204 tree itype
= signed_type_for (type
);
6205 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
6206 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
6210 t1
= fold_convert (fd
->iter_type
, t1
);
6211 t0
= fold_convert (fd
->iter_type
, t0
);
6215 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
6216 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
6218 if (fd
->iter_type
== long_integer_type_node
)
6222 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6223 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6224 6, t0
, t1
, t2
, t
, t3
, t4
);
6227 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6228 5, t0
, t1
, t2
, t3
, t4
);
6236 /* The GOMP_loop_ull_*start functions have additional boolean
6237 argument, true for < loops and false for > loops.
6238 In Fortran, the C bool type can be different from
6239 boolean_type_node. */
6240 bfn_decl
= builtin_decl_explicit (start_fn
);
6241 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
6242 t5
= build_int_cst (c_bool_type
,
6243 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
6246 tree bfn_decl
= builtin_decl_explicit (start_fn
);
6247 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6248 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
6251 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6252 6, t5
, t0
, t1
, t2
, t3
, t4
);
6255 if (TREE_TYPE (t
) != boolean_type_node
)
6256 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6257 t
, build_int_cst (TREE_TYPE (t
), 0));
6258 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6259 true, GSI_SAME_STMT
);
6260 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6262 /* Remove the GIMPLE_OMP_FOR statement. */
6263 gsi_remove (&gsi
, true);
6265 /* Iteration setup for sequential loop goes in L0_BB. */
6266 tree startvar
= fd
->loop
.v
;
6267 tree endvar
= NULL_TREE
;
6269 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6271 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
6272 && gimple_omp_for_kind (inner_stmt
)
6273 == GF_OMP_FOR_KIND_SIMD
);
6274 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
6275 OMP_CLAUSE__LOOPTEMP_
);
6276 gcc_assert (innerc
);
6277 startvar
= OMP_CLAUSE_DECL (innerc
);
6278 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6279 OMP_CLAUSE__LOOPTEMP_
);
6280 gcc_assert (innerc
);
6281 endvar
= OMP_CLAUSE_DECL (innerc
);
6284 gsi
= gsi_start_bb (l0_bb
);
6287 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6288 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6289 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6290 t
= fold_convert (TREE_TYPE (startvar
), t
);
6291 t
= force_gimple_operand_gsi (&gsi
, t
,
6293 && TREE_ADDRESSABLE (startvar
),
6294 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6295 assign_stmt
= gimple_build_assign (startvar
, t
);
6296 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6300 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6301 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6302 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6303 t
= fold_convert (TREE_TYPE (startvar
), t
);
6304 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6305 false, GSI_CONTINUE_LINKING
);
6308 assign_stmt
= gimple_build_assign (endvar
, iend
);
6309 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6310 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
6311 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
6313 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
6314 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6316 if (fd
->collapse
> 1)
6317 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6321 /* Code to control the increment and predicate for the sequential
6322 loop goes in the CONT_BB. */
6323 gsi
= gsi_last_bb (cont_bb
);
6324 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6325 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6326 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6327 vback
= gimple_omp_continue_control_def (cont_stmt
);
6329 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6331 if (POINTER_TYPE_P (type
))
6332 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
6334 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
6335 t
= force_gimple_operand_gsi (&gsi
, t
,
6337 && TREE_ADDRESSABLE (vback
),
6338 NULL_TREE
, true, GSI_SAME_STMT
);
6339 assign_stmt
= gimple_build_assign (vback
, t
);
6340 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6342 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6343 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
6345 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6346 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6349 /* Remove GIMPLE_OMP_CONTINUE. */
6350 gsi_remove (&gsi
, true);
6352 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6353 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
6355 /* Emit code to get the next parallel iteration in L2_BB. */
6356 gsi
= gsi_start_bb (l2_bb
);
6358 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6359 build_fold_addr_expr (istart0
),
6360 build_fold_addr_expr (iend0
));
6361 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6362 false, GSI_CONTINUE_LINKING
);
6363 if (TREE_TYPE (t
) != boolean_type_node
)
6364 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6365 t
, build_int_cst (TREE_TYPE (t
), 0));
6366 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6367 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
6370 /* Add the loop cleanup function. */
6371 gsi
= gsi_last_bb (exit_bb
);
6372 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6373 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
6374 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6375 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
6377 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
6378 gcall
*call_stmt
= gimple_build_call (t
, 0);
6379 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6380 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
6381 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
6382 gsi_remove (&gsi
, true);
6384 /* Connect the new blocks. */
6385 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
6386 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
6392 e
= find_edge (cont_bb
, l3_bb
);
6393 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
6395 phis
= phi_nodes (l3_bb
);
6396 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6398 gimple phi
= gsi_stmt (gsi
);
6399 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
6400 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
6404 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
6405 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
6406 e
= find_edge (cont_bb
, l1_bb
);
6407 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6412 else if (fd
->collapse
> 1)
6415 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6418 e
->flags
= EDGE_TRUE_VALUE
;
6421 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6422 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
6426 e
= find_edge (cont_bb
, l2_bb
);
6427 e
->flags
= EDGE_FALLTHRU
;
6429 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6431 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
6432 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
6433 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
6434 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
6435 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
6436 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
6437 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
6438 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
6440 struct loop
*outer_loop
= alloc_loop ();
6441 outer_loop
->header
= l0_bb
;
6442 outer_loop
->latch
= l2_bb
;
6443 add_loop (outer_loop
, l0_bb
->loop_father
);
6445 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6447 struct loop
*loop
= alloc_loop ();
6448 loop
->header
= l1_bb
;
6449 /* The loop may have multiple latches. */
6450 add_loop (loop
, outer_loop
);
6456 /* A subroutine of expand_omp_for. Generate code for a parallel
6457 loop with static schedule and no specified chunk size. Given
6460 for (V = N1; V cond N2; V += STEP) BODY;
6462 where COND is "<" or ">", we generate pseudocode
6464 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6469 if ((__typeof (V)) -1 > 0 && cond is >)
6470 n = -(adj + N2 - N1) / -STEP;
6472 n = (adj + N2 - N1) / STEP;
6475 if (threadid < tt) goto L3; else goto L4;
6480 s0 = q * threadid + tt;
6483 if (s0 >= e0) goto L2; else goto L0;
6489 if (V cond e) goto L1;
6494 expand_omp_for_static_nochunk (struct omp_region
*region
,
6495 struct omp_for_data
*fd
,
6498 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6499 tree type
, itype
, vmain
, vback
;
6500 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6501 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6503 gimple_stmt_iterator gsi
;
6505 bool broken_loop
= region
->cont
== NULL
;
6506 tree
*counts
= NULL
;
6509 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6510 != GF_OMP_FOR_KIND_OACC_LOOP
)
6513 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6514 if (POINTER_TYPE_P (type
))
6515 itype
= signed_type_for (type
);
6517 entry_bb
= region
->entry
;
6518 cont_bb
= region
->cont
;
6519 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6520 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6521 gcc_assert (broken_loop
6522 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6523 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6524 body_bb
= single_succ (seq_start_bb
);
6527 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6528 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6530 exit_bb
= region
->exit
;
6532 /* Iteration space partitioning goes in ENTRY_BB. */
6533 gsi
= gsi_last_bb (entry_bb
);
6534 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6536 if (fd
->collapse
> 1)
6538 int first_zero_iter
= -1;
6539 basic_block l2_dom_bb
= NULL
;
6541 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6542 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6543 fin_bb
, first_zero_iter
,
6547 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6548 t
= integer_one_node
;
6550 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6551 fold_convert (type
, fd
->loop
.n1
),
6552 fold_convert (type
, fd
->loop
.n2
));
6553 if (fd
->collapse
== 1
6554 && TYPE_UNSIGNED (type
)
6555 && (t
== NULL_TREE
|| !integer_onep (t
)))
6557 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6558 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6559 true, GSI_SAME_STMT
);
6560 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6561 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6562 true, GSI_SAME_STMT
);
6563 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6564 NULL_TREE
, NULL_TREE
);
6565 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6566 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6567 expand_omp_regimplify_p
, NULL
, NULL
)
6568 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6569 expand_omp_regimplify_p
, NULL
, NULL
))
6571 gsi
= gsi_for_stmt (cond_stmt
);
6572 gimple_regimplify_operands (cond_stmt
, &gsi
);
6574 ep
= split_block (entry_bb
, cond_stmt
);
6575 ep
->flags
= EDGE_TRUE_VALUE
;
6576 entry_bb
= ep
->dest
;
6577 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6578 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6579 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6580 if (gimple_in_ssa_p (cfun
))
6582 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6583 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6584 !gsi_end_p (gpi
); gsi_next (&gpi
))
6586 gphi
*phi
= gpi
.phi ();
6587 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6588 ep
, UNKNOWN_LOCATION
);
6591 gsi
= gsi_last_bb (entry_bb
);
6594 switch (gimple_omp_for_kind (fd
->for_stmt
))
6596 case GF_OMP_FOR_KIND_FOR
:
6597 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6598 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6600 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6601 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6602 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6604 case GF_OMP_FOR_KIND_OACC_LOOP
:
6605 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6606 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
6611 nthreads
= build_call_expr (nthreads
, 0);
6612 nthreads
= fold_convert (itype
, nthreads
);
6613 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
6614 true, GSI_SAME_STMT
);
6615 threadid
= build_call_expr (threadid
, 0);
6616 threadid
= fold_convert (itype
, threadid
);
6617 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
6618 true, GSI_SAME_STMT
);
6622 step
= fd
->loop
.step
;
6623 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6625 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6626 OMP_CLAUSE__LOOPTEMP_
);
6627 gcc_assert (innerc
);
6628 n1
= OMP_CLAUSE_DECL (innerc
);
6629 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6630 OMP_CLAUSE__LOOPTEMP_
);
6631 gcc_assert (innerc
);
6632 n2
= OMP_CLAUSE_DECL (innerc
);
6634 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6635 true, NULL_TREE
, true, GSI_SAME_STMT
);
6636 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6637 true, NULL_TREE
, true, GSI_SAME_STMT
);
6638 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6639 true, NULL_TREE
, true, GSI_SAME_STMT
);
6641 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6642 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6643 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6644 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6645 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6646 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6647 fold_build1 (NEGATE_EXPR
, itype
, t
),
6648 fold_build1 (NEGATE_EXPR
, itype
, step
));
6650 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6651 t
= fold_convert (itype
, t
);
6652 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6654 q
= create_tmp_reg (itype
, "q");
6655 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6656 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6657 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6659 tt
= create_tmp_reg (itype
, "tt");
6660 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6661 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6662 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6664 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6665 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6666 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6668 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
6669 gsi
= gsi_last_bb (second_bb
);
6670 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6672 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6674 gassign
*assign_stmt
6675 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
6676 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6678 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
6679 gsi
= gsi_last_bb (third_bb
);
6680 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6682 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6683 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6684 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6686 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6687 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6689 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6690 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6692 /* Remove the GIMPLE_OMP_FOR statement. */
6693 gsi_remove (&gsi
, true);
6695 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6696 gsi
= gsi_start_bb (seq_start_bb
);
6698 tree startvar
= fd
->loop
.v
;
6699 tree endvar
= NULL_TREE
;
6701 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6703 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6704 ? gimple_omp_parallel_clauses (inner_stmt
)
6705 : gimple_omp_for_clauses (inner_stmt
);
6706 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6707 gcc_assert (innerc
);
6708 startvar
= OMP_CLAUSE_DECL (innerc
);
6709 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6710 OMP_CLAUSE__LOOPTEMP_
);
6711 gcc_assert (innerc
);
6712 endvar
= OMP_CLAUSE_DECL (innerc
);
6714 t
= fold_convert (itype
, s0
);
6715 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6716 if (POINTER_TYPE_P (type
))
6717 t
= fold_build_pointer_plus (n1
, t
);
6719 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6720 t
= fold_convert (TREE_TYPE (startvar
), t
);
6721 t
= force_gimple_operand_gsi (&gsi
, t
,
6723 && TREE_ADDRESSABLE (startvar
),
6724 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6725 assign_stmt
= gimple_build_assign (startvar
, t
);
6726 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6728 t
= fold_convert (itype
, e0
);
6729 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6730 if (POINTER_TYPE_P (type
))
6731 t
= fold_build_pointer_plus (n1
, t
);
6733 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6734 t
= fold_convert (TREE_TYPE (startvar
), t
);
6735 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6736 false, GSI_CONTINUE_LINKING
);
6739 assign_stmt
= gimple_build_assign (endvar
, e
);
6740 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6741 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6742 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6744 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
6745 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6747 if (fd
->collapse
> 1)
6748 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6752 /* The code controlling the sequential loop replaces the
6753 GIMPLE_OMP_CONTINUE. */
6754 gsi
= gsi_last_bb (cont_bb
);
6755 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6756 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6757 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6758 vback
= gimple_omp_continue_control_def (cont_stmt
);
6760 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6762 if (POINTER_TYPE_P (type
))
6763 t
= fold_build_pointer_plus (vmain
, step
);
6765 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6766 t
= force_gimple_operand_gsi (&gsi
, t
,
6768 && TREE_ADDRESSABLE (vback
),
6769 NULL_TREE
, true, GSI_SAME_STMT
);
6770 assign_stmt
= gimple_build_assign (vback
, t
);
6771 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6773 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6774 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6776 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6779 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6780 gsi_remove (&gsi
, true);
6782 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6783 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6786 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6787 gsi
= gsi_last_bb (exit_bb
);
6788 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6790 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6791 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
6792 gcc_checking_assert (t
== NULL_TREE
);
6794 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6796 gsi_remove (&gsi
, true);
6798 /* Connect all the blocks. */
6799 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6800 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6801 ep
= find_edge (entry_bb
, second_bb
);
6802 ep
->flags
= EDGE_TRUE_VALUE
;
6803 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6804 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6805 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6809 ep
= find_edge (cont_bb
, body_bb
);
6810 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6815 else if (fd
->collapse
> 1)
6818 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6821 ep
->flags
= EDGE_TRUE_VALUE
;
6822 find_edge (cont_bb
, fin_bb
)->flags
6823 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6826 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6827 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6828 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6830 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6831 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6832 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6833 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6835 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6837 struct loop
*loop
= alloc_loop ();
6838 loop
->header
= body_bb
;
6839 if (collapse_bb
== NULL
)
6840 loop
->latch
= cont_bb
;
6841 add_loop (loop
, body_bb
->loop_father
);
6846 /* A subroutine of expand_omp_for. Generate code for a parallel
6847 loop with static schedule and a specified chunk size. Given
6850 for (V = N1; V cond N2; V += STEP) BODY;
6852 where COND is "<" or ">", we generate pseudocode
6854 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6859 if ((__typeof (V)) -1 > 0 && cond is >)
6860 n = -(adj + N2 - N1) / -STEP;
6862 n = (adj + N2 - N1) / STEP;
6864 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6865 here so that V is defined
6866 if the loop is not entered
6868 s0 = (trip * nthreads + threadid) * CHUNK;
6869 e0 = min(s0 + CHUNK, n);
6870 if (s0 < n) goto L1; else goto L4;
6877 if (V cond e) goto L2; else goto L3;
6885 expand_omp_for_static_chunk (struct omp_region
*region
,
6886 struct omp_for_data
*fd
, gimple inner_stmt
)
6888 tree n
, s0
, e0
, e
, t
;
6889 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6890 tree type
, itype
, vmain
, vback
, vextra
;
6891 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6892 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6893 gimple_stmt_iterator gsi
;
6895 bool broken_loop
= region
->cont
== NULL
;
6896 tree
*counts
= NULL
;
6899 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6900 != GF_OMP_FOR_KIND_OACC_LOOP
)
6903 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6904 if (POINTER_TYPE_P (type
))
6905 itype
= signed_type_for (type
);
6907 entry_bb
= region
->entry
;
6908 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6910 iter_part_bb
= se
->dest
;
6911 cont_bb
= region
->cont
;
6912 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6913 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6914 gcc_assert (broken_loop
6915 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6916 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6917 body_bb
= single_succ (seq_start_bb
);
6920 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6921 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6922 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6924 exit_bb
= region
->exit
;
6926 /* Trip and adjustment setup goes in ENTRY_BB. */
6927 gsi
= gsi_last_bb (entry_bb
);
6928 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6930 if (fd
->collapse
> 1)
6932 int first_zero_iter
= -1;
6933 basic_block l2_dom_bb
= NULL
;
6935 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6936 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6937 fin_bb
, first_zero_iter
,
6941 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6942 t
= integer_one_node
;
6944 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6945 fold_convert (type
, fd
->loop
.n1
),
6946 fold_convert (type
, fd
->loop
.n2
));
6947 if (fd
->collapse
== 1
6948 && TYPE_UNSIGNED (type
)
6949 && (t
== NULL_TREE
|| !integer_onep (t
)))
6951 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6952 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6953 true, GSI_SAME_STMT
);
6954 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6955 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6956 true, GSI_SAME_STMT
);
6957 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6958 NULL_TREE
, NULL_TREE
);
6959 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6960 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6961 expand_omp_regimplify_p
, NULL
, NULL
)
6962 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6963 expand_omp_regimplify_p
, NULL
, NULL
))
6965 gsi
= gsi_for_stmt (cond_stmt
);
6966 gimple_regimplify_operands (cond_stmt
, &gsi
);
6968 se
= split_block (entry_bb
, cond_stmt
);
6969 se
->flags
= EDGE_TRUE_VALUE
;
6970 entry_bb
= se
->dest
;
6971 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6972 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6973 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6974 if (gimple_in_ssa_p (cfun
))
6976 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6977 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6978 !gsi_end_p (gpi
); gsi_next (&gpi
))
6980 gphi
*phi
= gpi
.phi ();
6981 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6982 se
, UNKNOWN_LOCATION
);
6985 gsi
= gsi_last_bb (entry_bb
);
6988 switch (gimple_omp_for_kind (fd
->for_stmt
))
6990 case GF_OMP_FOR_KIND_FOR
:
6991 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6992 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6994 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6995 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6996 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6998 case GF_OMP_FOR_KIND_OACC_LOOP
:
6999 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
7000 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
7005 nthreads
= build_call_expr (nthreads
, 0);
7006 nthreads
= fold_convert (itype
, nthreads
);
7007 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
7008 true, GSI_SAME_STMT
);
7009 threadid
= build_call_expr (threadid
, 0);
7010 threadid
= fold_convert (itype
, threadid
);
7011 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
7012 true, GSI_SAME_STMT
);
7016 step
= fd
->loop
.step
;
7017 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7019 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7020 OMP_CLAUSE__LOOPTEMP_
);
7021 gcc_assert (innerc
);
7022 n1
= OMP_CLAUSE_DECL (innerc
);
7023 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7024 OMP_CLAUSE__LOOPTEMP_
);
7025 gcc_assert (innerc
);
7026 n2
= OMP_CLAUSE_DECL (innerc
);
7028 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
7029 true, NULL_TREE
, true, GSI_SAME_STMT
);
7030 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
7031 true, NULL_TREE
, true, GSI_SAME_STMT
);
7032 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
7033 true, NULL_TREE
, true, GSI_SAME_STMT
);
7035 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
7036 true, NULL_TREE
, true, GSI_SAME_STMT
);
7038 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
7039 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
7040 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
7041 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
7042 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
7043 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7044 fold_build1 (NEGATE_EXPR
, itype
, t
),
7045 fold_build1 (NEGATE_EXPR
, itype
, step
));
7047 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
7048 t
= fold_convert (itype
, t
);
7049 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7050 true, GSI_SAME_STMT
);
7052 trip_var
= create_tmp_reg (itype
, ".trip");
7053 if (gimple_in_ssa_p (cfun
))
7055 trip_init
= make_ssa_name (trip_var
);
7056 trip_main
= make_ssa_name (trip_var
);
7057 trip_back
= make_ssa_name (trip_var
);
7061 trip_init
= trip_var
;
7062 trip_main
= trip_var
;
7063 trip_back
= trip_var
;
7066 gassign
*assign_stmt
7067 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
7068 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7070 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
7071 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7072 if (POINTER_TYPE_P (type
))
7073 t
= fold_build_pointer_plus (n1
, t
);
7075 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7076 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7077 true, GSI_SAME_STMT
);
7079 /* Remove the GIMPLE_OMP_FOR. */
7080 gsi_remove (&gsi
, true);
7082 /* Iteration space partitioning goes in ITER_PART_BB. */
7083 gsi
= gsi_last_bb (iter_part_bb
);
7085 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
7086 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
7087 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
7088 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7089 false, GSI_CONTINUE_LINKING
);
7091 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
7092 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
7093 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7094 false, GSI_CONTINUE_LINKING
);
7096 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
7097 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
7099 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7100 gsi
= gsi_start_bb (seq_start_bb
);
7102 tree startvar
= fd
->loop
.v
;
7103 tree endvar
= NULL_TREE
;
7105 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7107 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
7108 ? gimple_omp_parallel_clauses (inner_stmt
)
7109 : gimple_omp_for_clauses (inner_stmt
);
7110 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7111 gcc_assert (innerc
);
7112 startvar
= OMP_CLAUSE_DECL (innerc
);
7113 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7114 OMP_CLAUSE__LOOPTEMP_
);
7115 gcc_assert (innerc
);
7116 endvar
= OMP_CLAUSE_DECL (innerc
);
7119 t
= fold_convert (itype
, s0
);
7120 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7121 if (POINTER_TYPE_P (type
))
7122 t
= fold_build_pointer_plus (n1
, t
);
7124 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7125 t
= fold_convert (TREE_TYPE (startvar
), t
);
7126 t
= force_gimple_operand_gsi (&gsi
, t
,
7128 && TREE_ADDRESSABLE (startvar
),
7129 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7130 assign_stmt
= gimple_build_assign (startvar
, t
);
7131 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7133 t
= fold_convert (itype
, e0
);
7134 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7135 if (POINTER_TYPE_P (type
))
7136 t
= fold_build_pointer_plus (n1
, t
);
7138 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7139 t
= fold_convert (TREE_TYPE (startvar
), t
);
7140 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7141 false, GSI_CONTINUE_LINKING
);
7144 assign_stmt
= gimple_build_assign (endvar
, e
);
7145 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7146 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
7147 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
7149 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
7150 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7152 if (fd
->collapse
> 1)
7153 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
7157 /* The code controlling the sequential loop goes in CONT_BB,
7158 replacing the GIMPLE_OMP_CONTINUE. */
7159 gsi
= gsi_last_bb (cont_bb
);
7160 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
7161 vmain
= gimple_omp_continue_control_use (cont_stmt
);
7162 vback
= gimple_omp_continue_control_def (cont_stmt
);
7164 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7166 if (POINTER_TYPE_P (type
))
7167 t
= fold_build_pointer_plus (vmain
, step
);
7169 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
7170 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
7171 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7172 true, GSI_SAME_STMT
);
7173 assign_stmt
= gimple_build_assign (vback
, t
);
7174 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7176 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
7177 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
7179 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
7182 /* Remove GIMPLE_OMP_CONTINUE. */
7183 gsi_remove (&gsi
, true);
7185 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
7186 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
7188 /* Trip update code goes into TRIP_UPDATE_BB. */
7189 gsi
= gsi_start_bb (trip_update_bb
);
7191 t
= build_int_cst (itype
, 1);
7192 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
7193 assign_stmt
= gimple_build_assign (trip_back
, t
);
7194 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7197 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7198 gsi
= gsi_last_bb (exit_bb
);
7199 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
7201 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
7202 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
7203 gcc_checking_assert (t
== NULL_TREE
);
7205 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
7207 gsi_remove (&gsi
, true);
7209 /* Connect the new blocks. */
7210 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
7211 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
7215 se
= find_edge (cont_bb
, body_bb
);
7216 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7221 else if (fd
->collapse
> 1)
7224 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
7227 se
->flags
= EDGE_TRUE_VALUE
;
7228 find_edge (cont_bb
, trip_update_bb
)->flags
7229 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
7231 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
7234 if (gimple_in_ssa_p (cfun
))
7242 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
7244 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7245 remove arguments of the phi nodes in fin_bb. We need to create
7246 appropriate phi nodes in iter_part_bb instead. */
7247 se
= single_pred_edge (fin_bb
);
7248 re
= single_succ_edge (trip_update_bb
);
7249 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
7250 ene
= single_succ_edge (entry_bb
);
7252 psi
= gsi_start_phis (fin_bb
);
7253 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
7254 gsi_next (&psi
), ++i
)
7257 source_location locus
;
7260 t
= gimple_phi_result (phi
);
7261 gcc_assert (t
== redirect_edge_var_map_result (vm
));
7262 nphi
= create_phi_node (t
, iter_part_bb
);
7264 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
7265 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
7267 /* A special case -- fd->loop.v is not yet computed in
7268 iter_part_bb, we need to use vextra instead. */
7269 if (t
== fd
->loop
.v
)
7271 add_phi_arg (nphi
, t
, ene
, locus
);
7272 locus
= redirect_edge_var_map_location (vm
);
7273 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
7275 gcc_assert (gsi_end_p (psi
) && i
== head
->length ());
7276 redirect_edge_var_map_clear (re
);
7279 psi
= gsi_start_phis (fin_bb
);
7280 if (gsi_end_p (psi
))
7282 remove_phi_node (&psi
, false);
7285 /* Make phi node for trip. */
7286 phi
= create_phi_node (trip_main
, iter_part_bb
);
7287 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
7289 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
7294 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
7295 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
7296 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
7297 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
7298 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
7299 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
7300 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
7301 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
7302 recompute_dominator (CDI_DOMINATORS
, body_bb
));
7306 struct loop
*trip_loop
= alloc_loop ();
7307 trip_loop
->header
= iter_part_bb
;
7308 trip_loop
->latch
= trip_update_bb
;
7309 add_loop (trip_loop
, iter_part_bb
->loop_father
);
7311 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7313 struct loop
*loop
= alloc_loop ();
7314 loop
->header
= body_bb
;
7315 if (collapse_bb
== NULL
)
7316 loop
->latch
= cont_bb
;
7317 add_loop (loop
, trip_loop
);
7322 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7324 for (V = N1; V cond N2; V += STEP) BODY;
7326 where COND is "<" or ">" or "!=", we generate pseudocode
7328 for (ind_var = low; ind_var < high; ind_var++)
7330 V = n1 + (ind_var * STEP)
7335 In the above pseudocode, low and high are function parameters of the
7336 child function. In the function below, we are inserting a temp.
7337 variable that will be making a call to two OMP functions that will not be
7338 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7339 with _Cilk_for). These functions are replaced with low and high
7340 by the function that handles taskreg. */
7344 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
7346 bool broken_loop
= region
->cont
== NULL
;
7347 basic_block entry_bb
= region
->entry
;
7348 basic_block cont_bb
= region
->cont
;
7350 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7351 gcc_assert (broken_loop
7352 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7353 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7354 basic_block l1_bb
, l2_bb
;
7358 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7359 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7360 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7361 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7365 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7366 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7367 l2_bb
= single_succ (l1_bb
);
7369 basic_block exit_bb
= region
->exit
;
7370 basic_block l2_dom_bb
= NULL
;
7372 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
7374 /* Below statements until the "tree high_val = ..." are pseudo statements
7375 used to pass information to be used by expand_omp_taskreg.
7376 low_val and high_val will be replaced by the __low and __high
7377 parameter from the child function.
7379 The call_exprs part is a place-holder, it is mainly used
7380 to distinctly identify to the top-level part that this is
7381 where we should put low and high (reasoning given in header
7385 = gimple_omp_parallel_child_fn (
7386 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
7387 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
7388 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
7390 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
7392 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
7395 gcc_assert (low_val
&& high_val
);
7397 tree type
= TREE_TYPE (low_val
);
7398 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
7399 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7401 /* Not needed in SSA form right now. */
7402 gcc_assert (!gimple_in_ssa_p (cfun
));
7403 if (l2_dom_bb
== NULL
)
7409 gimple stmt
= gimple_build_assign (ind_var
, n1
);
7411 /* Replace the GIMPLE_OMP_FOR statement. */
7412 gsi_replace (&gsi
, stmt
, true);
7416 /* Code to control the increment goes in the CONT_BB. */
7417 gsi
= gsi_last_bb (cont_bb
);
7418 stmt
= gsi_stmt (gsi
);
7419 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7420 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
7421 build_one_cst (type
));
7423 /* Replace GIMPLE_OMP_CONTINUE. */
7424 gsi_replace (&gsi
, stmt
, true);
7427 /* Emit the condition in L1_BB. */
7428 gsi
= gsi_after_labels (l1_bb
);
7429 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
7430 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
7432 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
7433 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7434 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7436 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7437 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
7438 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
7439 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7441 /* The condition is always '<' since the runtime will fill in the low
7443 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
7444 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7446 /* Remove GIMPLE_OMP_RETURN. */
7447 gsi
= gsi_last_bb (exit_bb
);
7448 gsi_remove (&gsi
, true);
7450 /* Connect the new blocks. */
7451 remove_edge (FALLTHRU_EDGE (entry_bb
));
7456 remove_edge (BRANCH_EDGE (entry_bb
));
7457 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7459 e
= BRANCH_EDGE (l1_bb
);
7460 ne
= FALLTHRU_EDGE (l1_bb
);
7461 e
->flags
= EDGE_TRUE_VALUE
;
7465 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7467 ne
= single_succ_edge (l1_bb
);
7468 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7471 ne
->flags
= EDGE_FALSE_VALUE
;
7472 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7473 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7475 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7476 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7477 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7481 struct loop
*loop
= alloc_loop ();
7482 loop
->header
= l1_bb
;
7483 loop
->latch
= cont_bb
;
7484 add_loop (loop
, l1_bb
->loop_father
);
7485 loop
->safelen
= INT_MAX
;
7488 /* Pick the correct library function based on the precision of the
7489 induction variable type. */
7490 tree lib_fun
= NULL_TREE
;
7491 if (TYPE_PRECISION (type
) == 32)
7492 lib_fun
= cilk_for_32_fndecl
;
7493 else if (TYPE_PRECISION (type
) == 64)
7494 lib_fun
= cilk_for_64_fndecl
;
7498 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7500 /* WS_ARGS contains the library function flavor to call:
7501 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7502 user-defined grain value. If the user does not define one, then zero
7503 is passed in by the parser. */
7504 vec_alloc (region
->ws_args
, 2);
7505 region
->ws_args
->quick_push (lib_fun
);
7506 region
->ws_args
->quick_push (fd
->chunk_size
);
7509 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7510 loop. Given parameters:
7512 for (V = N1; V cond N2; V += STEP) BODY;
7514 where COND is "<" or ">", we generate pseudocode
7522 if (V cond N2) goto L0; else goto L2;
7525 For collapsed loops, given parameters:
7527 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7528 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7529 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7532 we generate pseudocode
7538 count3 = (adj + N32 - N31) / STEP3;
7543 count2 = (adj + N22 - N21) / STEP2;
7548 count1 = (adj + N12 - N11) / STEP1;
7549 count = count1 * count2 * count3;
7559 V2 += (V3 cond3 N32) ? 0 : STEP2;
7560 V3 = (V3 cond3 N32) ? V3 : N31;
7561 V1 += (V2 cond2 N22) ? 0 : STEP1;
7562 V2 = (V2 cond2 N22) ? V2 : N21;
7564 if (V < count) goto L0; else goto L2;
7570 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7573 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7574 gimple_stmt_iterator gsi
;
7577 bool broken_loop
= region
->cont
== NULL
;
7579 tree
*counts
= NULL
;
7581 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7582 OMP_CLAUSE_SAFELEN
);
7583 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7584 OMP_CLAUSE__SIMDUID_
);
7587 type
= TREE_TYPE (fd
->loop
.v
);
7588 entry_bb
= region
->entry
;
7589 cont_bb
= region
->cont
;
7590 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7591 gcc_assert (broken_loop
7592 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7593 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7596 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7597 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7598 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7599 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7603 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7604 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7605 l2_bb
= single_succ (l1_bb
);
7607 exit_bb
= region
->exit
;
7610 gsi
= gsi_last_bb (entry_bb
);
7612 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7613 /* Not needed in SSA form right now. */
7614 gcc_assert (!gimple_in_ssa_p (cfun
));
7615 if (fd
->collapse
> 1)
7617 int first_zero_iter
= -1;
7618 basic_block zero_iter_bb
= l2_bb
;
7620 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7621 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7622 zero_iter_bb
, first_zero_iter
,
7625 if (l2_dom_bb
== NULL
)
7630 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7632 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7633 OMP_CLAUSE__LOOPTEMP_
);
7634 gcc_assert (innerc
);
7635 n1
= OMP_CLAUSE_DECL (innerc
);
7636 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7637 OMP_CLAUSE__LOOPTEMP_
);
7638 gcc_assert (innerc
);
7639 n2
= OMP_CLAUSE_DECL (innerc
);
7640 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7641 fold_convert (type
, n1
));
7642 if (fd
->collapse
> 1)
7645 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7651 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7652 fold_convert (type
, fd
->loop
.n1
));
7653 if (fd
->collapse
> 1)
7654 for (i
= 0; i
< fd
->collapse
; i
++)
7656 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7657 if (POINTER_TYPE_P (itype
))
7658 itype
= signed_type_for (itype
);
7659 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7660 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7664 /* Remove the GIMPLE_OMP_FOR statement. */
7665 gsi_remove (&gsi
, true);
7669 /* Code to control the increment goes in the CONT_BB. */
7670 gsi
= gsi_last_bb (cont_bb
);
7671 stmt
= gsi_stmt (gsi
);
7672 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7674 if (POINTER_TYPE_P (type
))
7675 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7677 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7678 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7680 if (fd
->collapse
> 1)
7682 i
= fd
->collapse
- 1;
7683 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7685 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7686 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7690 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7692 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7695 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7697 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7699 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7700 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7701 if (POINTER_TYPE_P (itype2
))
7702 itype2
= signed_type_for (itype2
);
7703 t
= build3 (COND_EXPR
, itype2
,
7704 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7706 fold_convert (itype
, fd
->loops
[i
].n2
)),
7707 build_int_cst (itype2
, 0),
7708 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7709 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7710 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7712 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7713 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7715 t
= build3 (COND_EXPR
, itype
,
7716 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7718 fold_convert (itype
, fd
->loops
[i
].n2
)),
7720 fold_convert (itype
, fd
->loops
[i
].n1
));
7721 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7725 /* Remove GIMPLE_OMP_CONTINUE. */
7726 gsi_remove (&gsi
, true);
7729 /* Emit the condition in L1_BB. */
7730 gsi
= gsi_start_bb (l1_bb
);
7732 t
= fold_convert (type
, n2
);
7733 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7734 false, GSI_CONTINUE_LINKING
);
7735 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7736 cond_stmt
= gimple_build_cond_empty (t
);
7737 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
7738 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7740 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7743 gsi
= gsi_for_stmt (cond_stmt
);
7744 gimple_regimplify_operands (cond_stmt
, &gsi
);
7747 /* Remove GIMPLE_OMP_RETURN. */
7748 gsi
= gsi_last_bb (exit_bb
);
7749 gsi_remove (&gsi
, true);
7751 /* Connect the new blocks. */
7752 remove_edge (FALLTHRU_EDGE (entry_bb
));
7756 remove_edge (BRANCH_EDGE (entry_bb
));
7757 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7759 e
= BRANCH_EDGE (l1_bb
);
7760 ne
= FALLTHRU_EDGE (l1_bb
);
7761 e
->flags
= EDGE_TRUE_VALUE
;
7765 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7767 ne
= single_succ_edge (l1_bb
);
7768 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7771 ne
->flags
= EDGE_FALSE_VALUE
;
7772 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7773 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7775 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7776 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7777 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7781 struct loop
*loop
= alloc_loop ();
7782 loop
->header
= l1_bb
;
7783 loop
->latch
= cont_bb
;
7784 add_loop (loop
, l1_bb
->loop_father
);
7785 if (safelen
== NULL_TREE
)
7786 loop
->safelen
= INT_MAX
;
7789 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7790 if (TREE_CODE (safelen
) != INTEGER_CST
)
7792 else if (!tree_fits_uhwi_p (safelen
)
7793 || tree_to_uhwi (safelen
) > INT_MAX
)
7794 loop
->safelen
= INT_MAX
;
7796 loop
->safelen
= tree_to_uhwi (safelen
);
7797 if (loop
->safelen
== 1)
7802 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7803 cfun
->has_simduid_loops
= true;
7805 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7807 if ((flag_tree_loop_vectorize
7808 || (!global_options_set
.x_flag_tree_loop_vectorize
7809 && !global_options_set
.x_flag_tree_vectorize
))
7810 && flag_tree_loop_optimize
7811 && loop
->safelen
> 1)
7813 loop
->force_vectorize
= true;
7814 cfun
->has_force_vectorize_loops
= true;
7820 /* Expand the OMP loop defined by REGION. */
7823 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7825 struct omp_for_data fd
;
7826 struct omp_for_data_loop
*loops
;
7829 = (struct omp_for_data_loop
*)
7830 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7831 * sizeof (struct omp_for_data_loop
));
7832 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
7834 region
->sched_kind
= fd
.sched_kind
;
7836 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7837 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7838 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7841 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7842 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7843 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7846 /* If there isn't a continue then this is a degerate case where
7847 the introduction of abnormal edges during lowering will prevent
7848 original loops from being detected. Fix that up. */
7849 loops_state_set (LOOPS_NEED_FIXUP
);
7851 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7852 expand_omp_simd (region
, &fd
);
7853 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7854 expand_cilk_for (region
, &fd
);
7855 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7856 && !fd
.have_ordered
)
7858 if (fd
.chunk_size
== NULL
)
7859 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7861 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7865 int fn_index
, start_ix
, next_ix
;
7867 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7868 == GF_OMP_FOR_KIND_FOR
);
7869 if (fd
.chunk_size
== NULL
7870 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7871 fd
.chunk_size
= integer_zero_node
;
7872 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7873 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7874 ? 3 : fd
.sched_kind
;
7875 fn_index
+= fd
.have_ordered
* 4;
7876 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7877 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7878 if (fd
.iter_type
== long_long_unsigned_type_node
)
7880 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7881 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7882 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7883 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7885 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7886 (enum built_in_function
) next_ix
, inner_stmt
);
7889 if (gimple_in_ssa_p (cfun
))
7890 update_ssa (TODO_update_ssa_only_virtuals
);
7894 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7896 v = GOMP_sections_start (n);
7913 v = GOMP_sections_next ();
7918 If this is a combined parallel sections, replace the call to
7919 GOMP_sections_start with call to GOMP_sections_next. */
7922 expand_omp_sections (struct omp_region
*region
)
7924 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7926 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7927 gimple_stmt_iterator si
, switch_si
;
7928 gomp_sections
*sections_stmt
;
7930 gomp_continue
*cont
;
7933 struct omp_region
*inner
;
7935 bool exit_reachable
= region
->cont
!= NULL
;
7937 gcc_assert (region
->exit
!= NULL
);
7938 entry_bb
= region
->entry
;
7939 l0_bb
= single_succ (entry_bb
);
7940 l1_bb
= region
->cont
;
7941 l2_bb
= region
->exit
;
7942 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7943 l2
= gimple_block_label (l2_bb
);
7946 /* This can happen if there are reductions. */
7947 len
= EDGE_COUNT (l0_bb
->succs
);
7948 gcc_assert (len
> 0);
7949 e
= EDGE_SUCC (l0_bb
, len
- 1);
7950 si
= gsi_last_bb (e
->dest
);
7953 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7954 l2
= gimple_block_label (e
->dest
);
7956 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7958 si
= gsi_last_bb (e
->dest
);
7960 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7962 l2
= gimple_block_label (e
->dest
);
7968 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7970 default_bb
= create_empty_bb (l0_bb
);
7972 /* We will build a switch() with enough cases for all the
7973 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7974 and a default case to abort if something goes wrong. */
7975 len
= EDGE_COUNT (l0_bb
->succs
);
7977 /* Use vec::quick_push on label_vec throughout, since we know the size
7979 auto_vec
<tree
> label_vec (len
);
7981 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7982 GIMPLE_OMP_SECTIONS statement. */
7983 si
= gsi_last_bb (entry_bb
);
7984 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
7985 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7986 vin
= gimple_omp_sections_control (sections_stmt
);
7987 if (!is_combined_parallel (region
))
7989 /* If we are not inside a combined parallel+sections region,
7990 call GOMP_sections_start. */
7991 t
= build_int_cst (unsigned_type_node
, len
- 1);
7992 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7993 stmt
= gimple_build_call (u
, 1, t
);
7997 /* Otherwise, call GOMP_sections_next. */
7998 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7999 stmt
= gimple_build_call (u
, 0);
8001 gimple_call_set_lhs (stmt
, vin
);
8002 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8003 gsi_remove (&si
, true);
8005 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8007 switch_si
= gsi_last_bb (l0_bb
);
8008 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
8011 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
8012 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
8013 vmain
= gimple_omp_continue_control_use (cont
);
8014 vnext
= gimple_omp_continue_control_def (cont
);
8022 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
8023 label_vec
.quick_push (t
);
8026 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8027 for (inner
= region
->inner
, casei
= 1;
8029 inner
= inner
->next
, i
++, casei
++)
8031 basic_block s_entry_bb
, s_exit_bb
;
8033 /* Skip optional reduction region. */
8034 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
8041 s_entry_bb
= inner
->entry
;
8042 s_exit_bb
= inner
->exit
;
8044 t
= gimple_block_label (s_entry_bb
);
8045 u
= build_int_cst (unsigned_type_node
, casei
);
8046 u
= build_case_label (u
, NULL
, t
);
8047 label_vec
.quick_push (u
);
8049 si
= gsi_last_bb (s_entry_bb
);
8050 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
8051 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
8052 gsi_remove (&si
, true);
8053 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
8055 if (s_exit_bb
== NULL
)
8058 si
= gsi_last_bb (s_exit_bb
);
8059 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8060 gsi_remove (&si
, true);
8062 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
8065 /* Error handling code goes in DEFAULT_BB. */
8066 t
= gimple_block_label (default_bb
);
8067 u
= build_case_label (NULL
, NULL
, t
);
8068 make_edge (l0_bb
, default_bb
, 0);
8069 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
8071 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
8072 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
8073 gsi_remove (&switch_si
, true);
8075 si
= gsi_start_bb (default_bb
);
8076 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
8077 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
8083 /* Code to get the next section goes in L1_BB. */
8084 si
= gsi_last_bb (l1_bb
);
8085 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
8087 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8088 stmt
= gimple_build_call (bfn_decl
, 0);
8089 gimple_call_set_lhs (stmt
, vnext
);
8090 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8091 gsi_remove (&si
, true);
8093 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
8096 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8097 si
= gsi_last_bb (l2_bb
);
8098 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
8099 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
8100 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
8101 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
8103 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
8104 stmt
= gimple_build_call (t
, 0);
8105 if (gimple_omp_return_lhs (gsi_stmt (si
)))
8106 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
8107 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8108 gsi_remove (&si
, true);
8110 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
8114 /* Expand code for an OpenMP single directive. We've already expanded
8115 much of the code, here we simply place the GOMP_barrier call. */
8118 expand_omp_single (struct omp_region
*region
)
8120 basic_block entry_bb
, exit_bb
;
8121 gimple_stmt_iterator si
;
8123 entry_bb
= region
->entry
;
8124 exit_bb
= region
->exit
;
8126 si
= gsi_last_bb (entry_bb
);
8127 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
8128 gsi_remove (&si
, true);
8129 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8131 si
= gsi_last_bb (exit_bb
);
8132 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
8134 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
8135 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
8137 gsi_remove (&si
, true);
8138 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8142 /* Generic expansion for OpenMP synchronization directives: master,
8143 ordered and critical. All we need to do here is remove the entry
8144 and exit markers for REGION. */
8147 expand_omp_synch (struct omp_region
*region
)
8149 basic_block entry_bb
, exit_bb
;
8150 gimple_stmt_iterator si
;
8152 entry_bb
= region
->entry
;
8153 exit_bb
= region
->exit
;
8155 si
= gsi_last_bb (entry_bb
);
8156 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
8157 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
8158 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
8159 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
8160 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
8161 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
8162 gsi_remove (&si
, true);
8163 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8167 si
= gsi_last_bb (exit_bb
);
8168 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8169 gsi_remove (&si
, true);
8170 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8174 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8175 operation as a normal volatile load. */
8178 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
8179 tree loaded_val
, int index
)
8181 enum built_in_function tmpbase
;
8182 gimple_stmt_iterator gsi
;
8183 basic_block store_bb
;
8186 tree decl
, call
, type
, itype
;
8188 gsi
= gsi_last_bb (load_bb
);
8189 stmt
= gsi_stmt (gsi
);
8190 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8191 loc
= gimple_location (stmt
);
8193 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8194 is smaller than word size, then expand_atomic_load assumes that the load
8195 is atomic. We could avoid the builtin entirely in this case. */
8197 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8198 decl
= builtin_decl_explicit (tmpbase
);
8199 if (decl
== NULL_TREE
)
8202 type
= TREE_TYPE (loaded_val
);
8203 itype
= TREE_TYPE (TREE_TYPE (decl
));
8205 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
8206 build_int_cst (NULL
,
8207 gimple_omp_atomic_seq_cst_p (stmt
)
8209 : MEMMODEL_RELAXED
));
8210 if (!useless_type_conversion_p (type
, itype
))
8211 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8212 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8214 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8215 gsi_remove (&gsi
, true);
8217 store_bb
= single_succ (load_bb
);
8218 gsi
= gsi_last_bb (store_bb
);
8219 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8220 gsi_remove (&gsi
, true);
8222 if (gimple_in_ssa_p (cfun
))
8223 update_ssa (TODO_update_ssa_no_phi
);
8228 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8229 operation as a normal volatile store. */
8232 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
8233 tree loaded_val
, tree stored_val
, int index
)
8235 enum built_in_function tmpbase
;
8236 gimple_stmt_iterator gsi
;
8237 basic_block store_bb
= single_succ (load_bb
);
8240 tree decl
, call
, type
, itype
;
8244 gsi
= gsi_last_bb (load_bb
);
8245 stmt
= gsi_stmt (gsi
);
8246 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8248 /* If the load value is needed, then this isn't a store but an exchange. */
8249 exchange
= gimple_omp_atomic_need_value_p (stmt
);
8251 gsi
= gsi_last_bb (store_bb
);
8252 stmt
= gsi_stmt (gsi
);
8253 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
8254 loc
= gimple_location (stmt
);
8256 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8257 is smaller than word size, then expand_atomic_store assumes that the store
8258 is atomic. We could avoid the builtin entirely in this case. */
8260 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
8261 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
8262 decl
= builtin_decl_explicit (tmpbase
);
8263 if (decl
== NULL_TREE
)
8266 type
= TREE_TYPE (stored_val
);
8268 /* Dig out the type of the function's second argument. */
8269 itype
= TREE_TYPE (decl
);
8270 itype
= TYPE_ARG_TYPES (itype
);
8271 itype
= TREE_CHAIN (itype
);
8272 itype
= TREE_VALUE (itype
);
8273 imode
= TYPE_MODE (itype
);
8275 if (exchange
&& !can_atomic_exchange_p (imode
, true))
8278 if (!useless_type_conversion_p (itype
, type
))
8279 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
8280 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
8281 build_int_cst (NULL
,
8282 gimple_omp_atomic_seq_cst_p (stmt
)
8284 : MEMMODEL_RELAXED
));
8287 if (!useless_type_conversion_p (type
, itype
))
8288 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8289 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8292 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8293 gsi_remove (&gsi
, true);
8295 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8296 gsi
= gsi_last_bb (load_bb
);
8297 gsi_remove (&gsi
, true);
8299 if (gimple_in_ssa_p (cfun
))
8300 update_ssa (TODO_update_ssa_no_phi
);
8305 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8306 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8307 size of the data type, and thus usable to find the index of the builtin
8308 decl. Returns false if the expression is not of the proper form. */
8311 expand_omp_atomic_fetch_op (basic_block load_bb
,
8312 tree addr
, tree loaded_val
,
8313 tree stored_val
, int index
)
8315 enum built_in_function oldbase
, newbase
, tmpbase
;
8316 tree decl
, itype
, call
;
8318 basic_block store_bb
= single_succ (load_bb
);
8319 gimple_stmt_iterator gsi
;
8322 enum tree_code code
;
8323 bool need_old
, need_new
;
8327 /* We expect to find the following sequences:
8330 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8333 val = tmp OP something; (or: something OP tmp)
8334 GIMPLE_OMP_STORE (val)
8336 ???FIXME: Allow a more flexible sequence.
8337 Perhaps use data flow to pick the statements.
8341 gsi
= gsi_after_labels (store_bb
);
8342 stmt
= gsi_stmt (gsi
);
8343 loc
= gimple_location (stmt
);
8344 if (!is_gimple_assign (stmt
))
8347 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
8349 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
8350 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
8351 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
8352 gcc_checking_assert (!need_old
|| !need_new
);
8354 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
8357 /* Check for one of the supported fetch-op operations. */
8358 code
= gimple_assign_rhs_code (stmt
);
8362 case POINTER_PLUS_EXPR
:
8363 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
8364 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
8367 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
8368 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
8371 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
8372 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
8375 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
8376 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
8379 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
8380 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
8386 /* Make sure the expression is of the proper form. */
8387 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
8388 rhs
= gimple_assign_rhs2 (stmt
);
8389 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
8390 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
8391 rhs
= gimple_assign_rhs1 (stmt
);
8395 tmpbase
= ((enum built_in_function
)
8396 ((need_new
? newbase
: oldbase
) + index
+ 1));
8397 decl
= builtin_decl_explicit (tmpbase
);
8398 if (decl
== NULL_TREE
)
8400 itype
= TREE_TYPE (TREE_TYPE (decl
));
8401 imode
= TYPE_MODE (itype
);
8403 /* We could test all of the various optabs involved, but the fact of the
8404 matter is that (with the exception of i486 vs i586 and xadd) all targets
8405 that support any atomic operaton optab also implements compare-and-swap.
8406 Let optabs.c take care of expanding any compare-and-swap loop. */
8407 if (!can_compare_and_swap_p (imode
, true))
8410 gsi
= gsi_last_bb (load_bb
);
8411 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8413 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8414 It only requires that the operation happen atomically. Thus we can
8415 use the RELAXED memory model. */
8416 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
8417 fold_convert_loc (loc
, itype
, rhs
),
8418 build_int_cst (NULL
,
8419 seq_cst
? MEMMODEL_SEQ_CST
8420 : MEMMODEL_RELAXED
));
8422 if (need_old
|| need_new
)
8424 lhs
= need_old
? loaded_val
: stored_val
;
8425 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
8426 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
8429 call
= fold_convert_loc (loc
, void_type_node
, call
);
8430 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8431 gsi_remove (&gsi
, true);
8433 gsi
= gsi_last_bb (store_bb
);
8434 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8435 gsi_remove (&gsi
, true);
8436 gsi
= gsi_last_bb (store_bb
);
8437 gsi_remove (&gsi
, true);
8439 if (gimple_in_ssa_p (cfun
))
8440 update_ssa (TODO_update_ssa_no_phi
);
8445 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8449 newval = rhs; // with oldval replacing *addr in rhs
8450 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8451 if (oldval != newval)
8454 INDEX is log2 of the size of the data type, and thus usable to find the
8455 index of the builtin decl. */
8458 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
8459 tree addr
, tree loaded_val
, tree stored_val
,
8462 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
8463 tree type
, itype
, cmpxchg
, iaddr
;
8464 gimple_stmt_iterator si
;
8465 basic_block loop_header
= single_succ (load_bb
);
8468 enum built_in_function fncode
;
8470 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8471 order to use the RELAXED memory model effectively. */
8472 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8474 cmpxchg
= builtin_decl_explicit (fncode
);
8475 if (cmpxchg
== NULL_TREE
)
8477 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8478 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
8480 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
8483 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8484 si
= gsi_last_bb (load_bb
);
8485 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8487 /* For floating-point values, we'll need to view-convert them to integers
8488 so that we can perform the atomic compare and swap. Simplify the
8489 following code by always setting up the "i"ntegral variables. */
8490 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
8494 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
8497 = force_gimple_operand_gsi (&si
,
8498 fold_convert (TREE_TYPE (iaddr
), addr
),
8499 false, NULL_TREE
, true, GSI_SAME_STMT
);
8500 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
8501 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8502 loadedi
= create_tmp_var (itype
);
8503 if (gimple_in_ssa_p (cfun
))
8504 loadedi
= make_ssa_name (loadedi
);
8509 loadedi
= loaded_val
;
8512 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8513 tree loaddecl
= builtin_decl_explicit (fncode
);
8516 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8517 build_call_expr (loaddecl
, 2, iaddr
,
8518 build_int_cst (NULL_TREE
,
8519 MEMMODEL_RELAXED
)));
8521 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8522 build_int_cst (TREE_TYPE (iaddr
), 0));
8525 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8528 /* Move the value to the LOADEDI temporary. */
8529 if (gimple_in_ssa_p (cfun
))
8531 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8532 phi
= create_phi_node (loadedi
, loop_header
);
8533 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8537 gsi_insert_before (&si
,
8538 gimple_build_assign (loadedi
, initial
),
8540 if (loadedi
!= loaded_val
)
8542 gimple_stmt_iterator gsi2
;
8545 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8546 gsi2
= gsi_start_bb (loop_header
);
8547 if (gimple_in_ssa_p (cfun
))
8550 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8551 true, GSI_SAME_STMT
);
8552 stmt
= gimple_build_assign (loaded_val
, x
);
8553 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8557 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8558 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8559 true, GSI_SAME_STMT
);
8562 gsi_remove (&si
, true);
8564 si
= gsi_last_bb (store_bb
);
8565 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8568 storedi
= stored_val
;
8571 force_gimple_operand_gsi (&si
,
8572 build1 (VIEW_CONVERT_EXPR
, itype
,
8573 stored_val
), true, NULL_TREE
, true,
8576 /* Build the compare&swap statement. */
8577 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8578 new_storedi
= force_gimple_operand_gsi (&si
,
8579 fold_convert (TREE_TYPE (loadedi
),
8582 true, GSI_SAME_STMT
);
8584 if (gimple_in_ssa_p (cfun
))
8588 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
8589 stmt
= gimple_build_assign (old_vali
, loadedi
);
8590 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8592 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8593 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8596 /* Note that we always perform the comparison as an integer, even for
8597 floating point. This allows the atomic operation to properly
8598 succeed even with NaNs and -0.0. */
8599 stmt
= gimple_build_cond_empty
8600 (build2 (NE_EXPR
, boolean_type_node
,
8601 new_storedi
, old_vali
));
8602 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8605 e
= single_succ_edge (store_bb
);
8606 e
->flags
&= ~EDGE_FALLTHRU
;
8607 e
->flags
|= EDGE_FALSE_VALUE
;
8609 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8611 /* Copy the new value to loadedi (we already did that before the condition
8612 if we are not in SSA). */
8613 if (gimple_in_ssa_p (cfun
))
8615 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8616 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8619 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8620 gsi_remove (&si
, true);
8622 struct loop
*loop
= alloc_loop ();
8623 loop
->header
= loop_header
;
8624 loop
->latch
= store_bb
;
8625 add_loop (loop
, loop_header
->loop_father
);
8627 if (gimple_in_ssa_p (cfun
))
8628 update_ssa (TODO_update_ssa_no_phi
);
8633 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8635 GOMP_atomic_start ();
8639 The result is not globally atomic, but works so long as all parallel
8640 references are within #pragma omp atomic directives. According to
8641 responses received from omp@openmp.org, appears to be within spec.
8642 Which makes sense, since that's how several other compilers handle
8643 this situation as well.
8644 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8645 expanding. STORED_VAL is the operand of the matching
8646 GIMPLE_OMP_ATOMIC_STORE.
8649 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8653 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8658 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8659 tree addr
, tree loaded_val
, tree stored_val
)
8661 gimple_stmt_iterator si
;
8665 si
= gsi_last_bb (load_bb
);
8666 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8668 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8669 t
= build_call_expr (t
, 0);
8670 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8672 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8673 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8674 gsi_remove (&si
, true);
8676 si
= gsi_last_bb (store_bb
);
8677 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8679 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8681 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8683 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8684 t
= build_call_expr (t
, 0);
8685 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8686 gsi_remove (&si
, true);
8688 if (gimple_in_ssa_p (cfun
))
8689 update_ssa (TODO_update_ssa_no_phi
);
8693 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8694 using expand_omp_atomic_fetch_op. If it failed, we try to
8695 call expand_omp_atomic_pipeline, and if it fails too, the
8696 ultimate fallback is wrapping the operation in a mutex
8697 (expand_omp_atomic_mutex). REGION is the atomic region built
8698 by build_omp_regions_1(). */
8701 expand_omp_atomic (struct omp_region
*region
)
8703 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8704 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
8705 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
8706 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8707 tree addr
= gimple_omp_atomic_load_rhs (load
);
8708 tree stored_val
= gimple_omp_atomic_store_val (store
);
8709 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8710 HOST_WIDE_INT index
;
8712 /* Make sure the type is one of the supported sizes. */
8713 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8714 index
= exact_log2 (index
);
8715 if (index
>= 0 && index
<= 4)
8717 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8719 /* __sync builtins require strict data alignment. */
8720 if (exact_log2 (align
) >= index
)
8723 if (loaded_val
== stored_val
8724 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8725 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8726 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8727 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8731 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8732 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8733 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8734 && store_bb
== single_succ (load_bb
)
8735 && first_stmt (store_bb
) == store
8736 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8740 /* When possible, use specialized atomic update functions. */
8741 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8742 && store_bb
== single_succ (load_bb
)
8743 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8744 loaded_val
, stored_val
, index
))
8747 /* If we don't have specialized __sync builtins, try and implement
8748 as a compare and swap loop. */
8749 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8750 loaded_val
, stored_val
, index
))
8755 /* The ultimate fallback is wrapping the operation in a mutex. */
8756 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8760 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8763 expand_omp_target (struct omp_region
*region
)
8765 basic_block entry_bb
, exit_bb
, new_bb
;
8766 struct function
*child_cfun
;
8767 tree child_fn
, block
, t
;
8768 gimple_stmt_iterator gsi
;
8769 gomp_target
*entry_stmt
;
8772 bool offloaded
, data_region
;
8774 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
8775 new_bb
= region
->entry
;
8777 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
8778 switch (gimple_omp_target_kind (entry_stmt
))
8780 case GF_OMP_TARGET_KIND_REGION
:
8781 case GF_OMP_TARGET_KIND_UPDATE
:
8782 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8783 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8784 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8785 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8786 data_region
= false;
8788 case GF_OMP_TARGET_KIND_DATA
:
8789 case GF_OMP_TARGET_KIND_OACC_DATA
:
8796 child_fn
= NULL_TREE
;
8800 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8801 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8804 /* Supported by expand_omp_taskreg, but not here. */
8805 if (child_cfun
!= NULL
)
8806 gcc_checking_assert (!child_cfun
->cfg
);
8807 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
8809 entry_bb
= region
->entry
;
8810 exit_bb
= region
->exit
;
8814 unsigned srcidx
, dstidx
, num
;
8816 /* If the offloading region needs data sent from the parent
8817 function, then the very first statement (except possible
8818 tree profile counter updates) of the offloading body
8819 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8820 &.OMP_DATA_O is passed as an argument to the child function,
8821 we need to replace it with the argument as seen by the child
8824 In most cases, this will end up being the identity assignment
8825 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8826 a function call that has been inlined, the original PARM_DECL
8827 .OMP_DATA_I may have been converted into a different local
8828 variable. In which case, we need to keep the assignment. */
8829 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
8832 basic_block entry_succ_bb
= single_succ (entry_bb
);
8833 gimple_stmt_iterator gsi
;
8835 gimple tgtcopy_stmt
= NULL
;
8836 tree sender
= TREE_VEC_ELT (data_arg
, 0);
8838 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8840 gcc_assert (!gsi_end_p (gsi
));
8841 stmt
= gsi_stmt (gsi
);
8842 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8845 if (gimple_num_ops (stmt
) == 2)
8847 tree arg
= gimple_assign_rhs1 (stmt
);
8849 /* We're ignoring the subcode because we're
8850 effectively doing a STRIP_NOPS. */
8852 if (TREE_CODE (arg
) == ADDR_EXPR
8853 && TREE_OPERAND (arg
, 0) == sender
)
8855 tgtcopy_stmt
= stmt
;
8861 gcc_assert (tgtcopy_stmt
!= NULL
);
8862 arg
= DECL_ARGUMENTS (child_fn
);
8864 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8865 gsi_remove (&gsi
, true);
8868 /* Declare local variables needed in CHILD_CFUN. */
8869 block
= DECL_INITIAL (child_fn
);
8870 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8871 /* The gimplifier could record temporaries in the offloading block
8872 rather than in containing function's local_decls chain,
8873 which would mean cgraph missed finalizing them. Do it now. */
8874 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8875 if (TREE_CODE (t
) == VAR_DECL
8877 && !DECL_EXTERNAL (t
))
8878 varpool_node::finalize_decl (t
);
8879 DECL_SAVED_TREE (child_fn
) = NULL
;
8880 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8881 gimple_set_body (child_fn
, NULL
);
8882 TREE_USED (block
) = 1;
8884 /* Reset DECL_CONTEXT on function arguments. */
8885 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8886 DECL_CONTEXT (t
) = child_fn
;
8888 /* Split ENTRY_BB at GIMPLE_*,
8889 so that it can be moved to the child function. */
8890 gsi
= gsi_last_bb (entry_bb
);
8891 stmt
= gsi_stmt (gsi
);
8893 && gimple_code (stmt
) == gimple_code (entry_stmt
));
8894 e
= split_block (entry_bb
, stmt
);
8895 gsi_remove (&gsi
, true);
8897 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8899 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8902 gsi
= gsi_last_bb (exit_bb
);
8903 gcc_assert (!gsi_end_p (gsi
)
8904 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8905 stmt
= gimple_build_return (NULL
);
8906 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8907 gsi_remove (&gsi
, true);
8910 /* Move the offloading region into CHILD_CFUN. */
8912 block
= gimple_block (entry_stmt
);
8914 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8916 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8917 /* When the OMP expansion process cannot guarantee an up-to-date
8918 loop tree arrange for the child function to fixup loops. */
8919 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8920 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8922 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8923 num
= vec_safe_length (child_cfun
->local_decls
);
8924 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8926 t
= (*child_cfun
->local_decls
)[srcidx
];
8927 if (DECL_CONTEXT (t
) == cfun
->decl
)
8929 if (srcidx
!= dstidx
)
8930 (*child_cfun
->local_decls
)[dstidx
] = t
;
8934 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8936 /* Inform the callgraph about the new function. */
8937 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8938 cgraph_node::add_new_function (child_fn
, true);
8940 #ifdef ENABLE_OFFLOADING
8941 /* Add the new function to the offload table. */
8942 vec_safe_push (offload_funcs
, child_fn
);
8945 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8946 fixed in a following pass. */
8947 push_cfun (child_cfun
);
8948 cgraph_edge::rebuild_edges ();
8950 #ifdef ENABLE_OFFLOADING
8951 /* Prevent IPA from removing child_fn as unreachable, since there are no
8952 refs from the parent function to child_fn in offload LTO mode. */
8953 struct cgraph_node
*node
= cgraph_node::get (child_fn
);
8954 node
->mark_force_output ();
8957 /* Some EH regions might become dead, see PR34608. If
8958 pass_cleanup_cfg isn't the first pass to happen with the
8959 new child, these dead EH edges might cause problems.
8960 Clean them up now. */
8961 if (flag_exceptions
)
8964 bool changed
= false;
8966 FOR_EACH_BB_FN (bb
, cfun
)
8967 changed
|= gimple_purge_dead_eh_edges (bb
);
8969 cleanup_tree_cfg ();
8974 /* Emit a library call to launch the offloading region, or do data
8976 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8977 enum built_in_function start_ix
;
8978 location_t clause_loc
;
8980 switch (gimple_omp_target_kind (entry_stmt
))
8982 case GF_OMP_TARGET_KIND_REGION
:
8983 start_ix
= BUILT_IN_GOMP_TARGET
;
8985 case GF_OMP_TARGET_KIND_DATA
:
8986 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8988 case GF_OMP_TARGET_KIND_UPDATE
:
8989 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8991 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8992 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8993 start_ix
= BUILT_IN_GOACC_PARALLEL
;
8995 case GF_OMP_TARGET_KIND_OACC_DATA
:
8996 start_ix
= BUILT_IN_GOACC_DATA_START
;
8998 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8999 start_ix
= BUILT_IN_GOACC_UPDATE
;
9001 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9002 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
9008 clauses
= gimple_omp_target_clauses (entry_stmt
);
9010 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
9011 library choose) and there is no conditional. */
9013 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
9015 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
9017 cond
= OMP_CLAUSE_IF_EXPR (c
);
9019 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
9022 /* Even if we pass it to all library function calls, it is currently only
9023 defined/used for the OpenMP target ones. */
9024 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
9025 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
9026 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
);
9028 device
= OMP_CLAUSE_DEVICE_ID (c
);
9029 clause_loc
= OMP_CLAUSE_LOCATION (c
);
9032 clause_loc
= gimple_location (entry_stmt
);
9034 /* Ensure 'device' is of the correct type. */
9035 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
9037 /* If we found the clause 'if (cond)', build
9038 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
9041 cond
= gimple_boolify (cond
);
9043 basic_block cond_bb
, then_bb
, else_bb
;
9047 tmp_var
= create_tmp_var (TREE_TYPE (device
));
9049 e
= split_block (new_bb
, NULL
);
9052 gsi
= gsi_last_bb (new_bb
);
9054 e
= split_block (new_bb
, gsi_stmt (gsi
));
9060 then_bb
= create_empty_bb (cond_bb
);
9061 else_bb
= create_empty_bb (then_bb
);
9062 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
9063 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
9065 stmt
= gimple_build_cond_empty (cond
);
9066 gsi
= gsi_last_bb (cond_bb
);
9067 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9069 gsi
= gsi_start_bb (then_bb
);
9070 stmt
= gimple_build_assign (tmp_var
, device
);
9071 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9073 gsi
= gsi_start_bb (else_bb
);
9074 stmt
= gimple_build_assign (tmp_var
,
9075 build_int_cst (integer_type_node
,
9076 GOMP_DEVICE_HOST_FALLBACK
));
9077 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9079 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
9080 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
9081 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
9082 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
9083 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
9084 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
9089 gsi
= gsi_last_bb (new_bb
);
9090 t
= gimple_omp_target_data_arg (entry_stmt
);
9093 t1
= size_zero_node
;
9094 t2
= build_zero_cst (ptr_type_node
);
9100 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
9101 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
9102 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
9103 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
9104 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
9108 /* The maximum number used by any start_ix, without varargs. */
9109 auto_vec
<tree
, 11> args
;
9110 args
.quick_push (device
);
9112 args
.quick_push (build_fold_addr_expr (child_fn
));
9115 case BUILT_IN_GOMP_TARGET
:
9116 case BUILT_IN_GOMP_TARGET_DATA
:
9117 case BUILT_IN_GOMP_TARGET_UPDATE
:
9118 /* This const void * is part of the current ABI, but we're not actually
9120 args
.quick_push (build_zero_cst (ptr_type_node
));
9122 case BUILT_IN_GOACC_DATA_START
:
9123 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9124 case BUILT_IN_GOACC_PARALLEL
:
9125 case BUILT_IN_GOACC_UPDATE
:
9130 args
.quick_push (t1
);
9131 args
.quick_push (t2
);
9132 args
.quick_push (t3
);
9133 args
.quick_push (t4
);
9136 case BUILT_IN_GOACC_DATA_START
:
9137 case BUILT_IN_GOMP_TARGET
:
9138 case BUILT_IN_GOMP_TARGET_DATA
:
9139 case BUILT_IN_GOMP_TARGET_UPDATE
:
9141 case BUILT_IN_GOACC_PARALLEL
:
9143 tree t_num_gangs
, t_num_workers
, t_vector_length
;
9145 /* Default values for num_gangs, num_workers, and vector_length. */
9146 t_num_gangs
= t_num_workers
= t_vector_length
9147 = fold_convert_loc (gimple_location (entry_stmt
),
9148 integer_type_node
, integer_one_node
);
9149 /* ..., but if present, use the value specified by the respective
9150 clause, making sure that are of the correct type. */
9151 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
9153 t_num_gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9155 OMP_CLAUSE_NUM_GANGS_EXPR (c
));
9156 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_WORKERS
);
9158 t_num_workers
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9160 OMP_CLAUSE_NUM_WORKERS_EXPR (c
));
9161 c
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
9163 t_vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9165 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c
));
9166 args
.quick_push (t_num_gangs
);
9167 args
.quick_push (t_num_workers
);
9168 args
.quick_push (t_vector_length
);
9171 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9172 case BUILT_IN_GOACC_UPDATE
:
9177 /* Default values for t_async. */
9178 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
9180 build_int_cst (integer_type_node
,
9182 /* ..., but if present, use the value specified by the respective
9183 clause, making sure that is of the correct type. */
9184 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
9186 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9188 OMP_CLAUSE_ASYNC_EXPR (c
));
9190 args
.quick_push (t_async
);
9191 /* Save the index, and... */
9192 t_wait_idx
= args
.length ();
9193 /* ... push a default value. */
9194 args
.quick_push (fold_convert_loc (gimple_location (entry_stmt
),
9196 integer_zero_node
));
9197 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
9202 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
9204 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
9206 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9208 OMP_CLAUSE_WAIT_EXPR (c
)));
9213 /* Now that we know the number, replace the default value. */
9214 args
.ordered_remove (t_wait_idx
);
9215 args
.quick_insert (t_wait_idx
,
9216 fold_convert_loc (gimple_location (entry_stmt
),
9218 build_int_cst (integer_type_node
, n
)));
9226 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
9227 gimple_set_location (g
, gimple_location (entry_stmt
));
9228 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
9232 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
9233 gsi_remove (&gsi
, true);
9238 gsi
= gsi_last_bb (region
->exit
);
9240 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
9241 gsi_remove (&gsi
, true);
9246 /* Expand the parallel region tree rooted at REGION. Expansion
9247 proceeds in depth-first order. Innermost regions are expanded
9248 first. This way, parallel regions that require a new function to
9249 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9250 internal dependencies in their body. */
9253 expand_omp (struct omp_region
*region
)
9257 location_t saved_location
;
9258 gimple inner_stmt
= NULL
;
9260 /* First, determine whether this is a combined parallel+workshare
9262 if (region
->type
== GIMPLE_OMP_PARALLEL
)
9263 determine_parallel_type (region
);
9265 if (region
->type
== GIMPLE_OMP_FOR
9266 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
9267 inner_stmt
= last_stmt (region
->inner
->entry
);
9270 expand_omp (region
->inner
);
9272 saved_location
= input_location
;
9273 if (gimple_has_location (last_stmt (region
->entry
)))
9274 input_location
= gimple_location (last_stmt (region
->entry
));
9276 switch (region
->type
)
9278 case GIMPLE_OMP_PARALLEL
:
9279 case GIMPLE_OMP_TASK
:
9280 expand_omp_taskreg (region
);
9283 case GIMPLE_OMP_FOR
:
9284 expand_omp_for (region
, inner_stmt
);
9287 case GIMPLE_OMP_SECTIONS
:
9288 expand_omp_sections (region
);
9291 case GIMPLE_OMP_SECTION
:
9292 /* Individual omp sections are handled together with their
9293 parent GIMPLE_OMP_SECTIONS region. */
9296 case GIMPLE_OMP_SINGLE
:
9297 expand_omp_single (region
);
9300 case GIMPLE_OMP_MASTER
:
9301 case GIMPLE_OMP_TASKGROUP
:
9302 case GIMPLE_OMP_ORDERED
:
9303 case GIMPLE_OMP_CRITICAL
:
9304 case GIMPLE_OMP_TEAMS
:
9305 expand_omp_synch (region
);
9308 case GIMPLE_OMP_ATOMIC_LOAD
:
9309 expand_omp_atomic (region
);
9312 case GIMPLE_OMP_TARGET
:
9313 expand_omp_target (region
);
9320 input_location
= saved_location
;
9321 region
= region
->next
;
9326 /* Helper for build_omp_regions. Scan the dominator tree starting at
9327 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9328 true, the function ends once a single tree is built (otherwise, whole
9329 forest of OMP constructs may be built). */
9332 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
9335 gimple_stmt_iterator gsi
;
9339 gsi
= gsi_last_bb (bb
);
9340 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
9342 struct omp_region
*region
;
9343 enum gimple_code code
;
9345 stmt
= gsi_stmt (gsi
);
9346 code
= gimple_code (stmt
);
9347 if (code
== GIMPLE_OMP_RETURN
)
9349 /* STMT is the return point out of region PARENT. Mark it
9350 as the exit point and make PARENT the immediately
9351 enclosing region. */
9352 gcc_assert (parent
);
9355 parent
= parent
->outer
;
9357 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
9359 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9360 GIMPLE_OMP_RETURN, but matches with
9361 GIMPLE_OMP_ATOMIC_LOAD. */
9362 gcc_assert (parent
);
9363 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
9366 parent
= parent
->outer
;
9368 else if (code
== GIMPLE_OMP_CONTINUE
)
9370 gcc_assert (parent
);
9373 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
9375 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9376 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9380 region
= new_omp_region (bb
, code
, parent
);
9382 if (code
== GIMPLE_OMP_TARGET
)
9384 switch (gimple_omp_target_kind (stmt
))
9386 case GF_OMP_TARGET_KIND_REGION
:
9387 case GF_OMP_TARGET_KIND_DATA
:
9388 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9389 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9390 case GF_OMP_TARGET_KIND_OACC_DATA
:
9392 case GF_OMP_TARGET_KIND_UPDATE
:
9393 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9394 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9395 /* ..., other than for those stand-alone directives... */
9402 /* ..., this directive becomes the parent for a new region. */
9408 if (single_tree
&& !parent
)
9411 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
9413 son
= next_dom_son (CDI_DOMINATORS
, son
))
9414 build_omp_regions_1 (son
, parent
, single_tree
);
9417 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9421 build_omp_regions_root (basic_block root
)
9423 gcc_assert (root_omp_region
== NULL
);
9424 build_omp_regions_1 (root
, NULL
, true);
9425 gcc_assert (root_omp_region
!= NULL
);
9428 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9431 omp_expand_local (basic_block head
)
9433 build_omp_regions_root (head
);
9434 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
9436 fprintf (dump_file
, "\nOMP region tree\n\n");
9437 dump_omp_region (dump_file
, root_omp_region
, 0);
9438 fprintf (dump_file
, "\n");
9441 remove_exit_barriers (root_omp_region
);
9442 expand_omp (root_omp_region
);
9444 free_omp_regions ();
9447 /* Scan the CFG and build a tree of OMP regions. Return the root of
9448 the OMP region tree. */
9451 build_omp_regions (void)
9453 gcc_assert (root_omp_region
== NULL
);
9454 calculate_dominance_info (CDI_DOMINATORS
);
9455 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
9458 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9461 execute_expand_omp (void)
9463 build_omp_regions ();
9465 if (!root_omp_region
)
9470 fprintf (dump_file
, "\nOMP region tree\n\n");
9471 dump_omp_region (dump_file
, root_omp_region
, 0);
9472 fprintf (dump_file
, "\n");
9475 remove_exit_barriers (root_omp_region
);
9477 expand_omp (root_omp_region
);
9479 cleanup_tree_cfg ();
9481 free_omp_regions ();
9486 /* OMP expansion -- the default pass, run before creation of SSA form. */
9490 const pass_data pass_data_expand_omp
=
9492 GIMPLE_PASS
, /* type */
9493 "ompexp", /* name */
9494 OPTGROUP_NONE
, /* optinfo_flags */
9495 TV_NONE
, /* tv_id */
9496 PROP_gimple_any
, /* properties_required */
9497 PROP_gimple_eomp
, /* properties_provided */
9498 0, /* properties_destroyed */
9499 0, /* todo_flags_start */
9500 0, /* todo_flags_finish */
9503 class pass_expand_omp
: public gimple_opt_pass
9506 pass_expand_omp (gcc::context
*ctxt
)
9507 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
9510 /* opt_pass methods: */
9511 virtual unsigned int execute (function
*)
9513 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
9514 || flag_openmp_simd
!= 0)
9517 /* This pass always runs, to provide PROP_gimple_eomp.
9518 But often, there is nothing to do. */
9522 return execute_expand_omp ();
9525 }; // class pass_expand_omp
9530 make_pass_expand_omp (gcc::context
*ctxt
)
9532 return new pass_expand_omp (ctxt
);
9537 const pass_data pass_data_expand_omp_ssa
=
9539 GIMPLE_PASS
, /* type */
9540 "ompexpssa", /* name */
9541 OPTGROUP_NONE
, /* optinfo_flags */
9542 TV_NONE
, /* tv_id */
9543 PROP_cfg
| PROP_ssa
, /* properties_required */
9544 PROP_gimple_eomp
, /* properties_provided */
9545 0, /* properties_destroyed */
9546 0, /* todo_flags_start */
9547 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
9550 class pass_expand_omp_ssa
: public gimple_opt_pass
9553 pass_expand_omp_ssa (gcc::context
*ctxt
)
9554 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
9557 /* opt_pass methods: */
9558 virtual bool gate (function
*fun
)
9560 return !(fun
->curr_properties
& PROP_gimple_eomp
);
9562 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
9564 }; // class pass_expand_omp_ssa
9569 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
9571 return new pass_expand_omp_ssa (ctxt
);
9574 /* Routines to lower OMP directives into OMP-GIMPLE. */
9576 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9577 convert it to gimple. */
9579 oacc_gimple_assign (tree dest
, tree_code op
, tree src
, gimple_seq
*seq
)
9583 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
9585 stmt
= gimple_build_assign (dest
, op
, dest
, src
);
9586 gimple_seq_add_stmt (seq
, stmt
);
9590 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9591 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9592 gimplify_assign (t
, rdest
, seq
);
9595 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9596 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9597 gimplify_assign (t
, idest
, seq
);
9600 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9601 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9602 gimplify_assign (t
, rsrc
, seq
);
9605 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9606 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9607 gimplify_assign (t
, isrc
, seq
);
9610 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9611 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9614 if (op
== PLUS_EXPR
)
9616 stmt
= gimple_build_assign (r
, op
, rdest
, rsrc
);
9617 gimple_seq_add_stmt (seq
, stmt
);
9619 stmt
= gimple_build_assign (i
, op
, idest
, isrc
);
9620 gimple_seq_add_stmt (seq
, stmt
);
9622 else if (op
== MULT_EXPR
)
9624 /* Let x = a + ib = dest, y = c + id = src.
9625 x * y = (ac - bd) + i(ad + bc) */
9626 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9627 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9628 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9629 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9631 stmt
= gimple_build_assign (ac
, MULT_EXPR
, rdest
, rsrc
);
9632 gimple_seq_add_stmt (seq
, stmt
);
9634 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, isrc
);
9635 gimple_seq_add_stmt (seq
, stmt
);
9637 stmt
= gimple_build_assign (r
, MINUS_EXPR
, ac
, bd
);
9638 gimple_seq_add_stmt (seq
, stmt
);
9640 stmt
= gimple_build_assign (ad
, MULT_EXPR
, rdest
, isrc
);
9641 gimple_seq_add_stmt (seq
, stmt
);
9643 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, rsrc
);
9644 gimple_seq_add_stmt (seq
, stmt
);
9646 stmt
= gimple_build_assign (i
, PLUS_EXPR
, ad
, bc
);
9647 gimple_seq_add_stmt (seq
, stmt
);
9652 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
9653 gimplify_assign (dest
, result
, seq
);
9656 /* Helper function to initialize local data for the reduction arrays.
9657 The reduction arrays need to be placed inside the calling function
9658 for accelerators, or else the host won't be able to preform the final
9662 oacc_initialize_reduction_data (tree clauses
, tree nthreads
,
9663 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9669 /* Find the innermost OpenACC parallel context. */
9670 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
9671 && (gimple_omp_target_kind (ctx
->stmt
)
9672 == GF_OMP_TARGET_KIND_OACC_PARALLEL
))
9676 gcc_checking_assert (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
9677 && (gimple_omp_target_kind (octx
->stmt
)
9678 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
9680 /* Extract the clauses. */
9681 oc
= gimple_omp_target_clauses (octx
->stmt
);
9683 /* Find the last outer clause. */
9684 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
9687 /* Allocate arrays for each reduction variable. */
9688 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9690 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9693 tree var
= OMP_CLAUSE_DECL (c
);
9694 tree type
= get_base_type (var
);
9695 tree array
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
),
9699 /* Calculate size of the reduction array. */
9700 t
= create_tmp_var (TREE_TYPE (nthreads
));
9701 stmt
= gimple_build_assign (t
, MULT_EXPR
, nthreads
,
9702 fold_convert (TREE_TYPE (nthreads
),
9703 TYPE_SIZE_UNIT (type
)));
9704 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9706 size
= create_tmp_var (sizetype
);
9707 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
9709 /* Now allocate memory for it. */
9710 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
9711 stmt
= gimple_build_call (call
, 1, size
);
9712 gimple_call_set_lhs (stmt
, array
);
9713 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9715 /* Map this array into the accelerator. */
9717 /* Add the reduction array to the list of clauses. */
9719 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
9720 OMP_CLAUSE_SET_MAP_KIND (t
, GOMP_MAP_FORCE_FROM
);
9721 OMP_CLAUSE_DECL (t
) = x
;
9722 OMP_CLAUSE_CHAIN (t
) = NULL
;
9724 OMP_CLAUSE_CHAIN (oc
) = t
;
9726 gimple_omp_target_set_clauses (as_a
<gomp_target
*> (octx
->stmt
), t
);
9727 OMP_CLAUSE_SIZE (t
) = size
;
9732 /* Helper function to process the array of partial reductions. Nthreads
9733 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9734 cannot be used here, because nthreads on the host may be different than
9735 on the accelerator. */
9738 oacc_finalize_reduction_data (tree clauses
, tree nthreads
,
9739 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9741 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
9746 let var = the original reduction variable
9747 let array = reduction variable array
9749 for (i = 0; i < nthreads; i++)
9753 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
9754 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
9755 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
9757 /* Create and initialize an index variable. */
9758 tree ix
= create_tmp_var (sizetype
);
9759 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
9762 /* Insert the loop header label here. */
9763 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
9765 /* Exit loop if ix >= nthreads. */
9766 x
= create_tmp_var (sizetype
);
9767 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
9768 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
9769 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9771 /* Insert the loop body label here. */
9772 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
9774 /* Collapse each reduction array, one element at a time. */
9775 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9777 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9780 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
9782 /* reduction(-:var) sums up the partial results, so it acts
9783 identically to reduction(+:var). */
9784 if (reduction_code
== MINUS_EXPR
)
9785 reduction_code
= PLUS_EXPR
;
9787 /* Set up reduction variable var. */
9788 var
= OMP_CLAUSE_DECL (c
);
9789 type
= get_base_type (var
);
9790 array
= lookup_oacc_reduction (oacc_get_reduction_array_id
9791 (OMP_CLAUSE_DECL (c
)), ctx
);
9793 /* Calculate the array offset. */
9794 tree offset
= create_tmp_var (sizetype
);
9795 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
9796 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, ix
);
9797 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9799 tree ptr
= create_tmp_var (TREE_TYPE (array
));
9800 stmt
= gimple_build_assign (ptr
, POINTER_PLUS_EXPR
, array
, offset
);
9801 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9803 /* Extract array[ix] into mem. */
9804 tree mem
= create_tmp_var (type
);
9805 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
9807 /* Find the original reduction variable. */
9808 if (is_reference (var
))
9809 var
= build_simple_mem_ref (var
);
9811 tree t
= create_tmp_var (type
);
9813 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
9814 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9816 /* var = var op mem */
9817 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
9819 case TRUTH_ANDIF_EXPR
:
9820 case TRUTH_ORIF_EXPR
:
9821 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
9823 gimplify_and_add (t
, stmt_seqp
);
9826 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9827 oacc_gimple_assign (t
, OMP_CLAUSE_REDUCTION_CODE (c
), mem
,
9831 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
9832 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
9833 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9836 /* Increment the induction variable. */
9837 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
9838 stmt
= gimple_build_assign (ix
, PLUS_EXPR
, ix
, one
);
9839 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9841 /* Go back to the top of the loop. */
9842 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
9844 /* Place the loop exit label here. */
9845 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
9848 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9849 scan that for reductions. */
9852 oacc_process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
9853 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
9855 gimple_stmt_iterator gsi
;
9856 gimple_seq inner
= NULL
;
9858 /* A collapse clause may have inserted a new bind block. */
9859 gsi
= gsi_start (*body
);
9860 while (!gsi_end_p (gsi
))
9862 gimple stmt
= gsi_stmt (gsi
);
9863 if (gbind
*bind_stmt
= dyn_cast
<gbind
*> (stmt
))
9865 inner
= gimple_bind_body (bind_stmt
);
9867 gsi
= gsi_start (*body
);
9869 else if (dyn_cast
<gomp_for
*> (stmt
))
9875 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9877 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
9879 bool reduction_found
= false;
9881 gimple stmt
= gsi_stmt (gsi
);
9883 switch (gimple_code (stmt
))
9885 case GIMPLE_OMP_FOR
:
9886 clauses
= gimple_omp_for_clauses (stmt
);
9888 /* Search for a reduction clause. */
9889 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9890 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
9892 reduction_found
= true;
9896 if (!reduction_found
)
9899 ctx
= maybe_lookup_ctx (stmt
);
9902 /* Extract the number of threads. */
9903 nthreads
= create_tmp_var (sizetype
);
9904 t
= oacc_max_threads (ctx
);
9905 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
9907 /* Determine if this is kernel will be executed on the host. */
9908 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
9909 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
9910 stmt
= gimple_build_call (call
, 0);
9911 gimple_call_set_lhs (stmt
, acc_device
);
9912 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9914 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9915 acc_device_host
= create_tmp_var (integer_type_node
,
9916 ".acc_device_host");
9917 gimplify_assign (acc_device_host
,
9918 build_int_cst (integer_type_node
,
9922 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9923 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9925 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9927 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9928 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9929 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9932 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9934 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9935 gimplify_assign (acc_device_host
,
9936 build_int_cst (integer_type_node
,
9937 GOMP_DEVICE_HOST_NONSHM
),
9940 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9941 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9943 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9945 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9946 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9947 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9950 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9952 oacc_initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
,
9954 oacc_finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
9957 // Scan for other directives which support reduction here.
9963 /* If ctx is a worksharing context inside of a cancellable parallel
9964 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
9965 and conditional branch to parallel's cancel_label to handle
9966 cancellation in the implicit barrier. */
9969 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
9971 gimple omp_return
= gimple_seq_last_stmt (*body
);
9972 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
9973 if (gimple_omp_return_nowait_p (omp_return
))
9976 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
9977 && ctx
->outer
->cancellable
)
9979 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
9980 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
9981 tree lhs
= create_tmp_var (c_bool_type
);
9982 gimple_omp_return_set_lhs (omp_return
, lhs
);
9983 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
9984 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
9985 fold_convert (c_bool_type
,
9986 boolean_false_node
),
9987 ctx
->outer
->cancel_label
, fallthru_label
);
9988 gimple_seq_add_stmt (body
, g
);
9989 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
9993 /* Lower the OpenMP sections directive in the current statement in GSI_P.
9994 CTX is the enclosing OMP context for the current statement. */
9997 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9999 tree block
, control
;
10000 gimple_stmt_iterator tgsi
;
10001 gomp_sections
*stmt
;
10003 gbind
*new_stmt
, *bind
;
10004 gimple_seq ilist
, dlist
, olist
, new_body
;
10006 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
10008 push_gimplify_context ();
10012 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
10013 &ilist
, &dlist
, ctx
, NULL
);
10015 new_body
= gimple_omp_body (stmt
);
10016 gimple_omp_set_body (stmt
, NULL
);
10017 tgsi
= gsi_start (new_body
);
10018 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
10023 sec_start
= gsi_stmt (tgsi
);
10024 sctx
= maybe_lookup_ctx (sec_start
);
10027 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
10028 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
10029 GSI_CONTINUE_LINKING
);
10030 gimple_omp_set_body (sec_start
, NULL
);
10032 if (gsi_one_before_end_p (tgsi
))
10034 gimple_seq l
= NULL
;
10035 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
10037 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
10038 gimple_omp_section_set_last (sec_start
);
10041 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
10042 GSI_CONTINUE_LINKING
);
10045 block
= make_node (BLOCK
);
10046 bind
= gimple_build_bind (NULL
, new_body
, block
);
10049 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
10051 block
= make_node (BLOCK
);
10052 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10053 gsi_replace (gsi_p
, new_stmt
, true);
10055 pop_gimplify_context (new_stmt
);
10056 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10057 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10058 if (BLOCK_VARS (block
))
10059 TREE_USED (block
) = 1;
10062 gimple_seq_add_seq (&new_body
, ilist
);
10063 gimple_seq_add_stmt (&new_body
, stmt
);
10064 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
10065 gimple_seq_add_stmt (&new_body
, bind
);
10067 control
= create_tmp_var (unsigned_type_node
, ".section");
10068 t
= gimple_build_omp_continue (control
, control
);
10069 gimple_omp_sections_set_control (stmt
, control
);
10070 gimple_seq_add_stmt (&new_body
, t
);
10072 gimple_seq_add_seq (&new_body
, olist
);
10073 if (ctx
->cancellable
)
10074 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10075 gimple_seq_add_seq (&new_body
, dlist
);
10077 new_body
= maybe_catch_exception (new_body
);
10079 t
= gimple_build_omp_return
10080 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
10081 OMP_CLAUSE_NOWAIT
));
10082 gimple_seq_add_stmt (&new_body
, t
);
10083 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
10085 gimple_bind_set_body (new_stmt
, new_body
);
10089 /* A subroutine of lower_omp_single. Expand the simple form of
10090 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10092 if (GOMP_single_start ())
10094 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10096 FIXME. It may be better to delay expanding the logic of this until
10097 pass_expand_omp. The expanded logic may make the job more difficult
10098 to a synchronization analysis pass. */
10101 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
10103 location_t loc
= gimple_location (single_stmt
);
10104 tree tlabel
= create_artificial_label (loc
);
10105 tree flabel
= create_artificial_label (loc
);
10109 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
10110 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
10111 call
= gimple_build_call (decl
, 0);
10112 gimple_call_set_lhs (call
, lhs
);
10113 gimple_seq_add_stmt (pre_p
, call
);
10115 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
10116 fold_convert_loc (loc
, TREE_TYPE (lhs
),
10117 boolean_true_node
),
10119 gimple_seq_add_stmt (pre_p
, cond
);
10120 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
10121 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10122 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
10126 /* A subroutine of lower_omp_single. Expand the simple form of
10127 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10129 #pragma omp single copyprivate (a, b, c)
10131 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10134 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10140 GOMP_single_copy_end (©out);
10151 FIXME. It may be better to delay expanding the logic of this until
10152 pass_expand_omp. The expanded logic may make the job more difficult
10153 to a synchronization analysis pass. */
10156 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
10159 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
10160 gimple_seq copyin_seq
;
10161 location_t loc
= gimple_location (single_stmt
);
10163 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
10165 ptr_type
= build_pointer_type (ctx
->record_type
);
10166 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
10168 l0
= create_artificial_label (loc
);
10169 l1
= create_artificial_label (loc
);
10170 l2
= create_artificial_label (loc
);
10172 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
10173 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
10174 t
= fold_convert_loc (loc
, ptr_type
, t
);
10175 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
10177 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
10178 build_int_cst (ptr_type
, 0));
10179 t
= build3 (COND_EXPR
, void_type_node
, t
,
10180 build_and_jump (&l0
), build_and_jump (&l1
));
10181 gimplify_and_add (t
, pre_p
);
10183 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
10185 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10188 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
10191 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10192 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
10193 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
10194 gimplify_and_add (t
, pre_p
);
10196 t
= build_and_jump (&l2
);
10197 gimplify_and_add (t
, pre_p
);
10199 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
10201 gimple_seq_add_seq (pre_p
, copyin_seq
);
10203 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
10207 /* Expand code for an OpenMP single directive. */
10210 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10214 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
10216 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
10218 push_gimplify_context ();
10220 block
= make_node (BLOCK
);
10221 bind
= gimple_build_bind (NULL
, NULL
, block
);
10222 gsi_replace (gsi_p
, bind
, true);
10225 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
10226 &bind_body
, &dlist
, ctx
, NULL
);
10227 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
10229 gimple_seq_add_stmt (&bind_body
, single_stmt
);
10231 if (ctx
->record_type
)
10232 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
10234 lower_omp_single_simple (single_stmt
, &bind_body
);
10236 gimple_omp_set_body (single_stmt
, NULL
);
10238 gimple_seq_add_seq (&bind_body
, dlist
);
10240 bind_body
= maybe_catch_exception (bind_body
);
10242 t
= gimple_build_omp_return
10243 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
10244 OMP_CLAUSE_NOWAIT
));
10245 gimple_seq_add_stmt (&bind_body_tail
, t
);
10246 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
10247 if (ctx
->record_type
)
10249 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
10250 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10251 TREE_THIS_VOLATILE (clobber
) = 1;
10252 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
10253 clobber
), GSI_SAME_STMT
);
10255 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
10256 gimple_bind_set_body (bind
, bind_body
);
10258 pop_gimplify_context (bind
);
10260 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10261 BLOCK_VARS (block
) = ctx
->block_vars
;
10262 if (BLOCK_VARS (block
))
10263 TREE_USED (block
) = 1;
10267 /* Expand code for an OpenMP master directive. */
10270 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10272 tree block
, lab
= NULL
, x
, bfn_decl
;
10273 gimple stmt
= gsi_stmt (*gsi_p
);
10275 location_t loc
= gimple_location (stmt
);
10278 push_gimplify_context ();
10280 block
= make_node (BLOCK
);
10281 bind
= gimple_build_bind (NULL
, NULL
, block
);
10282 gsi_replace (gsi_p
, bind
, true);
10283 gimple_bind_add_stmt (bind
, stmt
);
10285 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
10286 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
10287 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
10288 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
10290 gimplify_and_add (x
, &tseq
);
10291 gimple_bind_add_seq (bind
, tseq
);
10293 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10294 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10295 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10296 gimple_omp_set_body (stmt
, NULL
);
10298 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
10300 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10302 pop_gimplify_context (bind
);
10304 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10305 BLOCK_VARS (block
) = ctx
->block_vars
;
10309 /* Expand code for an OpenMP taskgroup directive. */
10312 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10314 gimple stmt
= gsi_stmt (*gsi_p
);
10317 tree block
= make_node (BLOCK
);
10319 bind
= gimple_build_bind (NULL
, NULL
, block
);
10320 gsi_replace (gsi_p
, bind
, true);
10321 gimple_bind_add_stmt (bind
, stmt
);
10323 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
10325 gimple_bind_add_stmt (bind
, x
);
10327 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10328 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10329 gimple_omp_set_body (stmt
, NULL
);
10331 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10333 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10334 BLOCK_VARS (block
) = ctx
->block_vars
;
10338 /* Expand code for an OpenMP ordered directive. */
10341 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10344 gimple stmt
= gsi_stmt (*gsi_p
);
10348 push_gimplify_context ();
10350 block
= make_node (BLOCK
);
10351 bind
= gimple_build_bind (NULL
, NULL
, block
);
10352 gsi_replace (gsi_p
, bind
, true);
10353 gimple_bind_add_stmt (bind
, stmt
);
10355 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
10357 gimple_bind_add_stmt (bind
, x
);
10359 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10360 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10361 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10362 gimple_omp_set_body (stmt
, NULL
);
10364 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
10365 gimple_bind_add_stmt (bind
, x
);
10367 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10369 pop_gimplify_context (bind
);
10371 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10372 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10376 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10377 substitution of a couple of function calls. But in the NAMED case,
10378 requires that languages coordinate a symbol name. It is therefore
10379 best put here in common code. */
10381 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
10384 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10387 tree name
, lock
, unlock
;
10388 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
10390 location_t loc
= gimple_location (stmt
);
10393 name
= gimple_omp_critical_name (stmt
);
10398 if (!critical_name_mutexes
)
10399 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
10401 tree
*n
= critical_name_mutexes
->get (name
);
10406 decl
= create_tmp_var_raw (ptr_type_node
);
10408 new_str
= ACONCAT ((".gomp_critical_user_",
10409 IDENTIFIER_POINTER (name
), NULL
));
10410 DECL_NAME (decl
) = get_identifier (new_str
);
10411 TREE_PUBLIC (decl
) = 1;
10412 TREE_STATIC (decl
) = 1;
10413 DECL_COMMON (decl
) = 1;
10414 DECL_ARTIFICIAL (decl
) = 1;
10415 DECL_IGNORED_P (decl
) = 1;
10417 varpool_node::finalize_decl (decl
);
10419 critical_name_mutexes
->put (name
, decl
);
10424 /* If '#pragma omp critical' is inside offloaded region or
10425 inside function marked as offloadable, the symbol must be
10426 marked as offloadable too. */
10428 if (cgraph_node::get (current_function_decl
)->offloadable
)
10429 varpool_node::get_create (decl
)->offloadable
= 1;
10431 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
10432 if (is_gimple_omp_offloaded (octx
->stmt
))
10434 varpool_node::get_create (decl
)->offloadable
= 1;
10438 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
10439 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
10441 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
10442 unlock
= build_call_expr_loc (loc
, unlock
, 1,
10443 build_fold_addr_expr_loc (loc
, decl
));
10447 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
10448 lock
= build_call_expr_loc (loc
, lock
, 0);
10450 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
10451 unlock
= build_call_expr_loc (loc
, unlock
, 0);
10454 push_gimplify_context ();
10456 block
= make_node (BLOCK
);
10457 bind
= gimple_build_bind (NULL
, NULL
, block
);
10458 gsi_replace (gsi_p
, bind
, true);
10459 gimple_bind_add_stmt (bind
, stmt
);
10461 tbody
= gimple_bind_body (bind
);
10462 gimplify_and_add (lock
, &tbody
);
10463 gimple_bind_set_body (bind
, tbody
);
10465 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10466 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10467 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10468 gimple_omp_set_body (stmt
, NULL
);
10470 tbody
= gimple_bind_body (bind
);
10471 gimplify_and_add (unlock
, &tbody
);
10472 gimple_bind_set_body (bind
, tbody
);
10474 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10476 pop_gimplify_context (bind
);
10477 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10478 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10482 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10483 for a lastprivate clause. Given a loop control predicate of (V
10484 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10485 is appended to *DLIST, iterator initialization is appended to
10489 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
10490 gimple_seq
*dlist
, struct omp_context
*ctx
)
10492 tree clauses
, cond
, vinit
;
10493 enum tree_code cond_code
;
10496 cond_code
= fd
->loop
.cond_code
;
10497 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
10499 /* When possible, use a strict equality expression. This can let VRP
10500 type optimizations deduce the value and remove a copy. */
10501 if (tree_fits_shwi_p (fd
->loop
.step
))
10503 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
10504 if (step
== 1 || step
== -1)
10505 cond_code
= EQ_EXPR
;
10508 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
10510 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
10512 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
10513 if (!gimple_seq_empty_p (stmts
))
10515 gimple_seq_add_seq (&stmts
, *dlist
);
10518 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10519 vinit
= fd
->loop
.n1
;
10520 if (cond_code
== EQ_EXPR
10521 && tree_fits_shwi_p (fd
->loop
.n2
)
10522 && ! integer_zerop (fd
->loop
.n2
))
10523 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
10525 vinit
= unshare_expr (vinit
);
10527 /* Initialize the iterator variable, so that threads that don't execute
10528 any iterations don't execute the lastprivate clauses by accident. */
10529 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
10534 /* Lower code for an OMP loop directive. */
10537 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10539 tree
*rhs_p
, block
;
10540 struct omp_for_data fd
, *fdp
= NULL
;
10541 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
10543 gimple_seq omp_for_body
, body
, dlist
;
10546 push_gimplify_context ();
10548 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
10550 block
= make_node (BLOCK
);
10551 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10552 /* Replace at gsi right away, so that 'stmt' is no member
10553 of a sequence anymore as we're going to add to to a different
10555 gsi_replace (gsi_p
, new_stmt
, true);
10557 /* Move declaration of temporaries in the loop body before we make
10559 omp_for_body
= gimple_omp_body (stmt
);
10560 if (!gimple_seq_empty_p (omp_for_body
)
10561 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
10564 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
10565 tree vars
= gimple_bind_vars (inner_bind
);
10566 gimple_bind_append_vars (new_stmt
, vars
);
10567 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10568 keep them on the inner_bind and it's block. */
10569 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
10570 if (gimple_bind_block (inner_bind
))
10571 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
10574 if (gimple_omp_for_combined_into_p (stmt
))
10576 extract_omp_for_data (stmt
, &fd
, NULL
);
10579 /* We need two temporaries with fd.loop.v type (istart/iend)
10580 and then (fd.collapse - 1) temporaries with the same
10581 type for count2 ... countN-1 vars if not constant. */
10583 tree type
= fd
.iter_type
;
10584 if (fd
.collapse
> 1
10585 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
10586 count
+= fd
.collapse
- 1;
10587 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
10588 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
10589 tree clauses
= *pc
;
10592 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
10593 OMP_CLAUSE__LOOPTEMP_
);
10594 for (i
= 0; i
< count
; i
++)
10599 gcc_assert (outerc
);
10600 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
10601 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
10602 OMP_CLAUSE__LOOPTEMP_
);
10606 temp
= create_tmp_var (type
);
10607 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
10609 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
10610 OMP_CLAUSE_DECL (*pc
) = temp
;
10611 pc
= &OMP_CLAUSE_CHAIN (*pc
);
10616 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10619 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
10621 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
10623 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10625 /* Lower the header expressions. At this point, we can assume that
10626 the header is of the form:
10628 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10630 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10631 using the .omp_data_s mapping, if needed. */
10632 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
10634 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
10635 if (!is_gimple_min_invariant (*rhs_p
))
10636 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10638 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
10639 if (!is_gimple_min_invariant (*rhs_p
))
10640 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10642 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
10643 if (!is_gimple_min_invariant (*rhs_p
))
10644 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10647 /* Once lowered, extract the bounds and clauses. */
10648 extract_omp_for_data (stmt
, &fd
, NULL
);
10650 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
10652 gimple_seq_add_stmt (&body
, stmt
);
10653 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
10655 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
10658 /* After the loop, add exit clauses. */
10659 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
10661 if (ctx
->cancellable
)
10662 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
10664 gimple_seq_add_seq (&body
, dlist
);
10666 body
= maybe_catch_exception (body
);
10668 /* Region exit marker goes at the end of the loop body. */
10669 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
10670 maybe_add_implicit_barrier_cancel (ctx
, &body
);
10671 pop_gimplify_context (new_stmt
);
10673 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10674 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
10675 if (BLOCK_VARS (block
))
10676 TREE_USED (block
) = 1;
10678 gimple_bind_set_body (new_stmt
, body
);
10679 gimple_omp_set_body (stmt
, NULL
);
10680 gimple_omp_for_set_pre_body (stmt
, NULL
);
10683 /* Callback for walk_stmts. Check if the current statement only contains
10684 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10687 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
10688 bool *handled_ops_p
,
10689 struct walk_stmt_info
*wi
)
10691 int *info
= (int *) wi
->info
;
10692 gimple stmt
= gsi_stmt (*gsi_p
);
10694 *handled_ops_p
= true;
10695 switch (gimple_code (stmt
))
10699 case GIMPLE_OMP_FOR
:
10700 case GIMPLE_OMP_SECTIONS
:
10701 *info
= *info
== 0 ? 1 : -1;
10710 struct omp_taskcopy_context
10712 /* This field must be at the beginning, as we do "inheritance": Some
10713 callback functions for tree-inline.c (e.g., omp_copy_decl)
10714 receive a copy_body_data pointer that is up-casted to an
10715 omp_context pointer. */
10721 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
10723 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
10725 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
10726 return create_tmp_var (TREE_TYPE (var
));
10732 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
10734 tree name
, new_fields
= NULL
, type
, f
;
10736 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
10737 name
= DECL_NAME (TYPE_NAME (orig_type
));
10738 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
10739 TYPE_DECL
, name
, type
);
10740 TYPE_NAME (type
) = name
;
10742 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
10744 tree new_f
= copy_node (f
);
10745 DECL_CONTEXT (new_f
) = type
;
10746 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
10747 TREE_CHAIN (new_f
) = new_fields
;
10748 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10749 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10750 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
10752 new_fields
= new_f
;
10753 tcctx
->cb
.decl_map
->put (f
, new_f
);
10755 TYPE_FIELDS (type
) = nreverse (new_fields
);
10756 layout_type (type
);
10760 /* Create task copyfn. */
10763 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
10765 struct function
*child_cfun
;
10766 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
10767 tree record_type
, srecord_type
, bind
, list
;
10768 bool record_needs_remap
= false, srecord_needs_remap
= false;
10770 struct omp_taskcopy_context tcctx
;
10771 location_t loc
= gimple_location (task_stmt
);
10773 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
10774 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
10775 gcc_assert (child_cfun
->cfg
== NULL
);
10776 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
10778 /* Reset DECL_CONTEXT on function arguments. */
10779 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
10780 DECL_CONTEXT (t
) = child_fn
;
10782 /* Populate the function. */
10783 push_gimplify_context ();
10784 push_cfun (child_cfun
);
10786 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
10787 TREE_SIDE_EFFECTS (bind
) = 1;
10789 DECL_SAVED_TREE (child_fn
) = bind
;
10790 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
10792 /* Remap src and dst argument types if needed. */
10793 record_type
= ctx
->record_type
;
10794 srecord_type
= ctx
->srecord_type
;
10795 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
10796 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10798 record_needs_remap
= true;
10801 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
10802 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10804 srecord_needs_remap
= true;
10808 if (record_needs_remap
|| srecord_needs_remap
)
10810 memset (&tcctx
, '\0', sizeof (tcctx
));
10811 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
10812 tcctx
.cb
.dst_fn
= child_fn
;
10813 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
10814 gcc_checking_assert (tcctx
.cb
.src_node
);
10815 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
10816 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
10817 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
10818 tcctx
.cb
.eh_lp_nr
= 0;
10819 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
10820 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
10823 if (record_needs_remap
)
10824 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
10825 if (srecord_needs_remap
)
10826 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
10829 tcctx
.cb
.decl_map
= NULL
;
10831 arg
= DECL_ARGUMENTS (child_fn
);
10832 TREE_TYPE (arg
) = build_pointer_type (record_type
);
10833 sarg
= DECL_CHAIN (arg
);
10834 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
10836 /* First pass: initialize temporaries used in record_type and srecord_type
10837 sizes and field offsets. */
10838 if (tcctx
.cb
.decl_map
)
10839 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10840 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10844 decl
= OMP_CLAUSE_DECL (c
);
10845 p
= tcctx
.cb
.decl_map
->get (decl
);
10848 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10849 sf
= (tree
) n
->value
;
10850 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10851 src
= build_simple_mem_ref_loc (loc
, sarg
);
10852 src
= omp_build_component_ref (src
, sf
);
10853 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
10854 append_to_statement_list (t
, &list
);
10857 /* Second pass: copy shared var pointers and copy construct non-VLA
10858 firstprivate vars. */
10859 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10860 switch (OMP_CLAUSE_CODE (c
))
10862 case OMP_CLAUSE_SHARED
:
10863 decl
= OMP_CLAUSE_DECL (c
);
10864 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10867 f
= (tree
) n
->value
;
10868 if (tcctx
.cb
.decl_map
)
10869 f
= *tcctx
.cb
.decl_map
->get (f
);
10870 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10871 sf
= (tree
) n
->value
;
10872 if (tcctx
.cb
.decl_map
)
10873 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10874 src
= build_simple_mem_ref_loc (loc
, sarg
);
10875 src
= omp_build_component_ref (src
, sf
);
10876 dst
= build_simple_mem_ref_loc (loc
, arg
);
10877 dst
= omp_build_component_ref (dst
, f
);
10878 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10879 append_to_statement_list (t
, &list
);
10881 case OMP_CLAUSE_FIRSTPRIVATE
:
10882 decl
= OMP_CLAUSE_DECL (c
);
10883 if (is_variable_sized (decl
))
10885 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10888 f
= (tree
) n
->value
;
10889 if (tcctx
.cb
.decl_map
)
10890 f
= *tcctx
.cb
.decl_map
->get (f
);
10891 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10894 sf
= (tree
) n
->value
;
10895 if (tcctx
.cb
.decl_map
)
10896 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10897 src
= build_simple_mem_ref_loc (loc
, sarg
);
10898 src
= omp_build_component_ref (src
, sf
);
10899 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
10900 src
= build_simple_mem_ref_loc (loc
, src
);
10904 dst
= build_simple_mem_ref_loc (loc
, arg
);
10905 dst
= omp_build_component_ref (dst
, f
);
10906 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10907 append_to_statement_list (t
, &list
);
10909 case OMP_CLAUSE_PRIVATE
:
10910 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
10912 decl
= OMP_CLAUSE_DECL (c
);
10913 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10914 f
= (tree
) n
->value
;
10915 if (tcctx
.cb
.decl_map
)
10916 f
= *tcctx
.cb
.decl_map
->get (f
);
10917 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10920 sf
= (tree
) n
->value
;
10921 if (tcctx
.cb
.decl_map
)
10922 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10923 src
= build_simple_mem_ref_loc (loc
, sarg
);
10924 src
= omp_build_component_ref (src
, sf
);
10925 if (use_pointer_for_field (decl
, NULL
))
10926 src
= build_simple_mem_ref_loc (loc
, src
);
10930 dst
= build_simple_mem_ref_loc (loc
, arg
);
10931 dst
= omp_build_component_ref (dst
, f
);
10932 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10933 append_to_statement_list (t
, &list
);
10939 /* Last pass: handle VLA firstprivates. */
10940 if (tcctx
.cb
.decl_map
)
10941 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10942 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10946 decl
= OMP_CLAUSE_DECL (c
);
10947 if (!is_variable_sized (decl
))
10949 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10952 f
= (tree
) n
->value
;
10953 f
= *tcctx
.cb
.decl_map
->get (f
);
10954 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
10955 ind
= DECL_VALUE_EXPR (decl
);
10956 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
10957 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
10958 n
= splay_tree_lookup (ctx
->sfield_map
,
10959 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10960 sf
= (tree
) n
->value
;
10961 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10962 src
= build_simple_mem_ref_loc (loc
, sarg
);
10963 src
= omp_build_component_ref (src
, sf
);
10964 src
= build_simple_mem_ref_loc (loc
, src
);
10965 dst
= build_simple_mem_ref_loc (loc
, arg
);
10966 dst
= omp_build_component_ref (dst
, f
);
10967 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10968 append_to_statement_list (t
, &list
);
10969 n
= splay_tree_lookup (ctx
->field_map
,
10970 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10971 df
= (tree
) n
->value
;
10972 df
= *tcctx
.cb
.decl_map
->get (df
);
10973 ptr
= build_simple_mem_ref_loc (loc
, arg
);
10974 ptr
= omp_build_component_ref (ptr
, df
);
10975 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
10976 build_fold_addr_expr_loc (loc
, dst
));
10977 append_to_statement_list (t
, &list
);
10980 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
10981 append_to_statement_list (t
, &list
);
10983 if (tcctx
.cb
.decl_map
)
10984 delete tcctx
.cb
.decl_map
;
10985 pop_gimplify_context (NULL
);
10986 BIND_EXPR_BODY (bind
) = list
;
10991 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
10995 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
10997 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
10998 OMP_CLAUSE_DEPEND
);
10999 gcc_assert (clauses
);
11000 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11001 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
11002 switch (OMP_CLAUSE_DEPEND_KIND (c
))
11004 case OMP_CLAUSE_DEPEND_IN
:
11007 case OMP_CLAUSE_DEPEND_OUT
:
11008 case OMP_CLAUSE_DEPEND_INOUT
:
11012 gcc_unreachable ();
11014 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
11015 tree array
= create_tmp_var (type
);
11016 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
11018 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
11019 gimple_seq_add_stmt (iseq
, g
);
11020 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
11022 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
11023 gimple_seq_add_stmt (iseq
, g
);
11024 for (i
= 0; i
< 2; i
++)
11026 if ((i
? n_in
: n_out
) == 0)
11028 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11029 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
11030 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
11032 tree t
= OMP_CLAUSE_DECL (c
);
11033 t
= fold_convert (ptr_type_node
, t
);
11034 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
11035 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
11036 NULL_TREE
, NULL_TREE
);
11037 g
= gimple_build_assign (r
, t
);
11038 gimple_seq_add_stmt (iseq
, g
);
11041 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
11042 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
11043 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
11044 OMP_CLAUSE_CHAIN (c
) = *p
;
11046 tree clobber
= build_constructor (type
, NULL
);
11047 TREE_THIS_VOLATILE (clobber
) = 1;
11048 g
= gimple_build_assign (array
, clobber
);
11049 gimple_seq_add_stmt (oseq
, g
);
11052 /* Lower the OpenMP parallel or task directive in the current statement
11053 in GSI_P. CTX holds context information for the directive. */
11056 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11060 gimple stmt
= gsi_stmt (*gsi_p
);
11061 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
11062 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
11063 location_t loc
= gimple_location (stmt
);
11065 clauses
= gimple_omp_taskreg_clauses (stmt
);
11067 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
11068 par_body
= gimple_bind_body (par_bind
);
11069 child_fn
= ctx
->cb
.dst_fn
;
11070 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11071 && !gimple_omp_parallel_combined_p (stmt
))
11073 struct walk_stmt_info wi
;
11076 memset (&wi
, 0, sizeof (wi
));
11078 wi
.val_only
= true;
11079 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
11081 gimple_omp_parallel_set_combined_p (stmt
, true);
11083 gimple_seq dep_ilist
= NULL
;
11084 gimple_seq dep_olist
= NULL
;
11085 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11086 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
11088 push_gimplify_context ();
11089 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11090 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
11093 if (ctx
->srecord_type
)
11094 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
11096 push_gimplify_context ();
11101 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
11102 lower_omp (&par_body
, ctx
);
11103 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
11104 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
11106 /* Declare all the variables created by mapping and the variables
11107 declared in the scope of the parallel body. */
11108 record_vars_into (ctx
->block_vars
, child_fn
);
11109 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
11111 if (ctx
->record_type
)
11114 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
11115 : ctx
->record_type
, ".omp_data_o");
11116 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11117 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11118 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
11123 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
11124 lower_send_shared_vars (&ilist
, &olist
, ctx
);
11126 if (ctx
->record_type
)
11128 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
11129 TREE_THIS_VOLATILE (clobber
) = 1;
11130 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11134 /* Once all the expansions are done, sequence all the different
11135 fragments inside gimple_omp_body. */
11139 if (ctx
->record_type
)
11141 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11142 /* fixup_child_record_type might have changed receiver_decl's type. */
11143 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11144 gimple_seq_add_stmt (&new_body
,
11145 gimple_build_assign (ctx
->receiver_decl
, t
));
11148 gimple_seq_add_seq (&new_body
, par_ilist
);
11149 gimple_seq_add_seq (&new_body
, par_body
);
11150 gimple_seq_add_seq (&new_body
, par_rlist
);
11151 if (ctx
->cancellable
)
11152 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
11153 gimple_seq_add_seq (&new_body
, par_olist
);
11154 new_body
= maybe_catch_exception (new_body
);
11155 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11156 gimple_omp_set_body (stmt
, new_body
);
11158 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
11159 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
11160 gimple_bind_add_seq (bind
, ilist
);
11161 gimple_bind_add_stmt (bind
, stmt
);
11162 gimple_bind_add_seq (bind
, olist
);
11164 pop_gimplify_context (NULL
);
11168 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11169 gimple_bind_add_stmt (dep_bind
, bind
);
11170 gimple_bind_add_seq (dep_bind
, dep_olist
);
11171 pop_gimplify_context (dep_bind
);
11175 /* Lower the GIMPLE_OMP_TARGET in the current statement
11176 in GSI_P. CTX holds context information for the directive. */
11179 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11182 tree child_fn
, t
, c
;
11183 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
11184 gbind
*tgt_bind
, *bind
;
11185 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
11186 location_t loc
= gimple_location (stmt
);
11187 bool offloaded
, data_region
;
11188 unsigned int map_cnt
= 0;
11190 offloaded
= is_gimple_omp_offloaded (stmt
);
11191 switch (gimple_omp_target_kind (stmt
))
11193 case GF_OMP_TARGET_KIND_REGION
:
11194 case GF_OMP_TARGET_KIND_UPDATE
:
11195 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11196 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11197 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11198 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11199 data_region
= false;
11201 case GF_OMP_TARGET_KIND_DATA
:
11202 case GF_OMP_TARGET_KIND_OACC_DATA
:
11203 data_region
= true;
11206 gcc_unreachable ();
11209 clauses
= gimple_omp_target_clauses (stmt
);
11215 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
11216 tgt_body
= gimple_bind_body (tgt_bind
);
11218 else if (data_region
)
11219 tgt_body
= gimple_omp_body (stmt
);
11220 child_fn
= ctx
->cb
.dst_fn
;
11222 push_gimplify_context ();
11227 && is_gimple_omp_oacc (stmt
))
11228 oacc_process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
11230 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11231 switch (OMP_CLAUSE_CODE (c
))
11237 case OMP_CLAUSE_MAP
:
11238 #ifdef ENABLE_CHECKING
11239 /* First check what we're prepared to handle in the following. */
11240 switch (OMP_CLAUSE_MAP_KIND (c
))
11242 case GOMP_MAP_ALLOC
:
11244 case GOMP_MAP_FROM
:
11245 case GOMP_MAP_TOFROM
:
11246 case GOMP_MAP_POINTER
:
11247 case GOMP_MAP_TO_PSET
:
11249 case GOMP_MAP_FORCE_ALLOC
:
11250 case GOMP_MAP_FORCE_TO
:
11251 case GOMP_MAP_FORCE_FROM
:
11252 case GOMP_MAP_FORCE_TOFROM
:
11253 case GOMP_MAP_FORCE_PRESENT
:
11254 case GOMP_MAP_FORCE_DEALLOC
:
11255 case GOMP_MAP_FORCE_DEVICEPTR
:
11256 gcc_assert (is_gimple_omp_oacc (stmt
));
11259 gcc_unreachable ();
11263 case OMP_CLAUSE_TO
:
11264 case OMP_CLAUSE_FROM
:
11265 var
= OMP_CLAUSE_DECL (c
);
11268 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11269 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11274 if (DECL_SIZE (var
)
11275 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
11277 tree var2
= DECL_VALUE_EXPR (var
);
11278 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
11279 var2
= TREE_OPERAND (var2
, 0);
11280 gcc_assert (DECL_P (var2
));
11284 if (!maybe_lookup_field (var
, ctx
))
11289 x
= build_receiver_ref (var
, true, ctx
);
11290 tree new_var
= lookup_decl (var
, ctx
);
11291 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11292 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11293 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11294 x
= build_simple_mem_ref (x
);
11295 SET_DECL_VALUE_EXPR (new_var
, x
);
11296 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11303 target_nesting_level
++;
11304 lower_omp (&tgt_body
, ctx
);
11305 target_nesting_level
--;
11307 else if (data_region
)
11308 lower_omp (&tgt_body
, ctx
);
11312 /* Declare all the variables created by mapping and the variables
11313 declared in the scope of the target body. */
11314 record_vars_into (ctx
->block_vars
, child_fn
);
11315 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
11320 if (ctx
->record_type
)
11323 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
11324 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11325 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11326 t
= make_tree_vec (3);
11327 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
11328 TREE_VEC_ELT (t
, 1)
11329 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
11330 ".omp_data_sizes");
11331 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
11332 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
11333 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
11336 if (is_gimple_omp_oacc (stmt
))
11338 tkind_type
= short_unsigned_type_node
;
11343 tkind_type
= unsigned_char_type_node
;
11346 TREE_VEC_ELT (t
, 2)
11347 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
11348 ".omp_data_kinds");
11349 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
11350 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
11351 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
11352 gimple_omp_target_set_data_arg (stmt
, t
);
11354 vec
<constructor_elt
, va_gc
> *vsize
;
11355 vec
<constructor_elt
, va_gc
> *vkind
;
11356 vec_alloc (vsize
, map_cnt
);
11357 vec_alloc (vkind
, map_cnt
);
11358 unsigned int map_idx
= 0;
11360 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11361 switch (OMP_CLAUSE_CODE (c
))
11367 case OMP_CLAUSE_MAP
:
11368 case OMP_CLAUSE_TO
:
11369 case OMP_CLAUSE_FROM
:
11371 ovar
= OMP_CLAUSE_DECL (c
);
11372 if (!DECL_P (ovar
))
11374 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11375 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11377 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
11378 == get_base_address (ovar
));
11379 nc
= OMP_CLAUSE_CHAIN (c
);
11380 ovar
= OMP_CLAUSE_DECL (nc
);
11384 tree x
= build_sender_ref (ovar
, ctx
);
11386 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
11387 gimplify_assign (x
, v
, &ilist
);
11393 if (DECL_SIZE (ovar
)
11394 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
11396 tree ovar2
= DECL_VALUE_EXPR (ovar
);
11397 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
11398 ovar2
= TREE_OPERAND (ovar2
, 0);
11399 gcc_assert (DECL_P (ovar2
));
11402 if (!maybe_lookup_field (ovar
, ctx
))
11406 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
11407 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
11408 talign
= DECL_ALIGN_UNIT (ovar
);
11411 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11412 tree x
= build_sender_ref (ovar
, ctx
);
11413 if (maybe_lookup_oacc_reduction (var
, ctx
))
11415 gcc_checking_assert (offloaded
11416 && is_gimple_omp_oacc (stmt
));
11417 gimplify_assign (x
, var
, &ilist
);
11419 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11420 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11421 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11422 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
11424 gcc_assert (offloaded
);
11426 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
11427 mark_addressable (avar
);
11428 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
11429 talign
= DECL_ALIGN_UNIT (avar
);
11430 avar
= build_fold_addr_expr (avar
);
11431 gimplify_assign (x
, avar
, &ilist
);
11433 else if (is_gimple_reg (var
))
11435 gcc_assert (offloaded
);
11436 tree avar
= create_tmp_var (TREE_TYPE (var
));
11437 mark_addressable (avar
);
11438 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
11439 if (GOMP_MAP_COPY_TO_P (map_kind
)
11440 || map_kind
== GOMP_MAP_POINTER
11441 || map_kind
== GOMP_MAP_TO_PSET
11442 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11443 gimplify_assign (avar
, var
, &ilist
);
11444 avar
= build_fold_addr_expr (avar
);
11445 gimplify_assign (x
, avar
, &ilist
);
11446 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
11447 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11448 && !TYPE_READONLY (TREE_TYPE (var
)))
11450 x
= build_sender_ref (ovar
, ctx
);
11451 x
= build_simple_mem_ref (x
);
11452 gimplify_assign (var
, x
, &olist
);
11457 var
= build_fold_addr_expr (var
);
11458 gimplify_assign (x
, var
, &ilist
);
11461 tree s
= OMP_CLAUSE_SIZE (c
);
11462 if (s
== NULL_TREE
)
11463 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11464 s
= fold_convert (size_type_node
, s
);
11465 tree purpose
= size_int (map_idx
++);
11466 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11467 if (TREE_CODE (s
) != INTEGER_CST
)
11468 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11470 unsigned HOST_WIDE_INT tkind
;
11471 switch (OMP_CLAUSE_CODE (c
))
11473 case OMP_CLAUSE_MAP
:
11474 tkind
= OMP_CLAUSE_MAP_KIND (c
);
11476 case OMP_CLAUSE_TO
:
11477 tkind
= GOMP_MAP_TO
;
11479 case OMP_CLAUSE_FROM
:
11480 tkind
= GOMP_MAP_FROM
;
11483 gcc_unreachable ();
11485 gcc_checking_assert (tkind
11486 < (HOST_WIDE_INT_C (1U) << talign_shift
));
11487 talign
= ceil_log2 (talign
);
11488 tkind
|= talign
<< talign_shift
;
11489 gcc_checking_assert (tkind
11490 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11491 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
11492 build_int_cstu (tkind_type
, tkind
));
11497 gcc_assert (map_idx
== map_cnt
);
11499 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
11500 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
11501 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
11502 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
11503 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
11505 gimple_seq initlist
= NULL
;
11506 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
11507 TREE_VEC_ELT (t
, 1)),
11508 &initlist
, true, NULL_TREE
);
11509 gimple_seq_add_seq (&ilist
, initlist
);
11511 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
11513 TREE_THIS_VOLATILE (clobber
) = 1;
11514 gimple_seq_add_stmt (&olist
,
11515 gimple_build_assign (TREE_VEC_ELT (t
, 1),
11519 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
11520 TREE_THIS_VOLATILE (clobber
) = 1;
11521 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11525 /* Once all the expansions are done, sequence all the different
11526 fragments inside gimple_omp_body. */
11531 && ctx
->record_type
)
11533 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11534 /* fixup_child_record_type might have changed receiver_decl's type. */
11535 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11536 gimple_seq_add_stmt (&new_body
,
11537 gimple_build_assign (ctx
->receiver_decl
, t
));
11542 gimple_seq_add_seq (&new_body
, tgt_body
);
11543 new_body
= maybe_catch_exception (new_body
);
11545 else if (data_region
)
11546 new_body
= tgt_body
;
11547 if (offloaded
|| data_region
)
11549 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11550 gimple_omp_set_body (stmt
, new_body
);
11553 bind
= gimple_build_bind (NULL
, NULL
,
11554 tgt_bind
? gimple_bind_block (tgt_bind
)
11556 gsi_replace (gsi_p
, bind
, true);
11557 gimple_bind_add_seq (bind
, irlist
);
11558 gimple_bind_add_seq (bind
, ilist
);
11559 gimple_bind_add_stmt (bind
, stmt
);
11560 gimple_bind_add_seq (bind
, olist
);
11561 gimple_bind_add_seq (bind
, orlist
);
11563 pop_gimplify_context (NULL
);
11566 /* Expand code for an OpenMP teams directive. */
11569 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11571 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
11572 push_gimplify_context ();
11574 tree block
= make_node (BLOCK
);
11575 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
11576 gsi_replace (gsi_p
, bind
, true);
11577 gimple_seq bind_body
= NULL
;
11578 gimple_seq dlist
= NULL
;
11579 gimple_seq olist
= NULL
;
11581 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11582 OMP_CLAUSE_NUM_TEAMS
);
11583 if (num_teams
== NULL_TREE
)
11584 num_teams
= build_int_cst (unsigned_type_node
, 0);
11587 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
11588 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
11589 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
11591 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11592 OMP_CLAUSE_THREAD_LIMIT
);
11593 if (thread_limit
== NULL_TREE
)
11594 thread_limit
= build_int_cst (unsigned_type_node
, 0);
11597 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
11598 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
11599 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
11603 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
11604 &bind_body
, &dlist
, ctx
, NULL
);
11605 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
11606 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
11607 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
11609 location_t loc
= gimple_location (teams_stmt
);
11610 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
11611 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
11612 gimple_set_location (call
, loc
);
11613 gimple_seq_add_stmt (&bind_body
, call
);
11615 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
11616 gimple_omp_set_body (teams_stmt
, NULL
);
11617 gimple_seq_add_seq (&bind_body
, olist
);
11618 gimple_seq_add_seq (&bind_body
, dlist
);
11619 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
11620 gimple_bind_set_body (bind
, bind_body
);
11622 pop_gimplify_context (bind
);
11624 gimple_bind_append_vars (bind
, ctx
->block_vars
);
11625 BLOCK_VARS (block
) = ctx
->block_vars
;
11626 if (BLOCK_VARS (block
))
11627 TREE_USED (block
) = 1;
11631 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11632 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11633 of OMP context, but with task_shared_vars set. */
11636 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
11641 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11642 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
11645 if (task_shared_vars
11647 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
11650 /* If a global variable has been privatized, TREE_CONSTANT on
11651 ADDR_EXPR might be wrong. */
11652 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
11653 recompute_tree_invariant_for_addr_expr (t
);
11655 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
11660 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11662 gimple stmt
= gsi_stmt (*gsi_p
);
11663 struct walk_stmt_info wi
;
11666 if (gimple_has_location (stmt
))
11667 input_location
= gimple_location (stmt
);
11669 if (task_shared_vars
)
11670 memset (&wi
, '\0', sizeof (wi
));
11672 /* If we have issued syntax errors, avoid doing any heavy lifting.
11673 Just replace the OMP directives with a NOP to avoid
11674 confusing RTL expansion. */
11675 if (seen_error () && is_gimple_omp (stmt
))
11677 gsi_replace (gsi_p
, gimple_build_nop (), true);
11681 switch (gimple_code (stmt
))
11685 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
11686 if ((ctx
|| task_shared_vars
)
11687 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
11688 lower_omp_regimplify_p
,
11689 ctx
? NULL
: &wi
, NULL
)
11690 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
11691 lower_omp_regimplify_p
,
11692 ctx
? NULL
: &wi
, NULL
)))
11693 gimple_regimplify_operands (cond_stmt
, gsi_p
);
11697 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
11699 case GIMPLE_EH_FILTER
:
11700 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
11703 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
11704 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
11706 case GIMPLE_TRANSACTION
:
11707 lower_omp (gimple_transaction_body_ptr (
11708 as_a
<gtransaction
*> (stmt
)),
11712 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
11714 case GIMPLE_OMP_PARALLEL
:
11715 case GIMPLE_OMP_TASK
:
11716 ctx
= maybe_lookup_ctx (stmt
);
11718 if (ctx
->cancellable
)
11719 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11720 lower_omp_taskreg (gsi_p
, ctx
);
11722 case GIMPLE_OMP_FOR
:
11723 ctx
= maybe_lookup_ctx (stmt
);
11725 if (ctx
->cancellable
)
11726 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11727 lower_omp_for (gsi_p
, ctx
);
11729 case GIMPLE_OMP_SECTIONS
:
11730 ctx
= maybe_lookup_ctx (stmt
);
11732 if (ctx
->cancellable
)
11733 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11734 lower_omp_sections (gsi_p
, ctx
);
11736 case GIMPLE_OMP_SINGLE
:
11737 ctx
= maybe_lookup_ctx (stmt
);
11739 lower_omp_single (gsi_p
, ctx
);
11741 case GIMPLE_OMP_MASTER
:
11742 ctx
= maybe_lookup_ctx (stmt
);
11744 lower_omp_master (gsi_p
, ctx
);
11746 case GIMPLE_OMP_TASKGROUP
:
11747 ctx
= maybe_lookup_ctx (stmt
);
11749 lower_omp_taskgroup (gsi_p
, ctx
);
11751 case GIMPLE_OMP_ORDERED
:
11752 ctx
= maybe_lookup_ctx (stmt
);
11754 lower_omp_ordered (gsi_p
, ctx
);
11756 case GIMPLE_OMP_CRITICAL
:
11757 ctx
= maybe_lookup_ctx (stmt
);
11759 lower_omp_critical (gsi_p
, ctx
);
11761 case GIMPLE_OMP_ATOMIC_LOAD
:
11762 if ((ctx
|| task_shared_vars
)
11763 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
11764 as_a
<gomp_atomic_load
*> (stmt
)),
11765 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
11766 gimple_regimplify_operands (stmt
, gsi_p
);
11768 case GIMPLE_OMP_TARGET
:
11769 ctx
= maybe_lookup_ctx (stmt
);
11771 lower_omp_target (gsi_p
, ctx
);
11773 case GIMPLE_OMP_TEAMS
:
11774 ctx
= maybe_lookup_ctx (stmt
);
11776 lower_omp_teams (gsi_p
, ctx
);
11780 call_stmt
= as_a
<gcall
*> (stmt
);
11781 fndecl
= gimple_call_fndecl (call_stmt
);
11783 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
11784 switch (DECL_FUNCTION_CODE (fndecl
))
11786 case BUILT_IN_GOMP_BARRIER
:
11790 case BUILT_IN_GOMP_CANCEL
:
11791 case BUILT_IN_GOMP_CANCELLATION_POINT
:
11794 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
11795 cctx
= cctx
->outer
;
11796 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
11797 if (!cctx
->cancellable
)
11799 if (DECL_FUNCTION_CODE (fndecl
)
11800 == BUILT_IN_GOMP_CANCELLATION_POINT
)
11802 stmt
= gimple_build_nop ();
11803 gsi_replace (gsi_p
, stmt
, false);
11807 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
11809 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
11810 gimple_call_set_fndecl (call_stmt
, fndecl
);
11811 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
11814 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
11815 gimple_call_set_lhs (call_stmt
, lhs
);
11816 tree fallthru_label
;
11817 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
11819 g
= gimple_build_label (fallthru_label
);
11820 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11821 g
= gimple_build_cond (NE_EXPR
, lhs
,
11822 fold_convert (TREE_TYPE (lhs
),
11823 boolean_false_node
),
11824 cctx
->cancel_label
, fallthru_label
);
11825 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11832 if ((ctx
|| task_shared_vars
)
11833 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
11836 /* Just remove clobbers, this should happen only if we have
11837 "privatized" local addressable variables in SIMD regions,
11838 the clobber isn't needed in that case and gimplifying address
11839 of the ARRAY_REF into a pointer and creating MEM_REF based
11840 clobber would create worse code than we get with the clobber
11842 if (gimple_clobber_p (stmt
))
11844 gsi_replace (gsi_p
, gimple_build_nop (), true);
11847 gimple_regimplify_operands (stmt
, gsi_p
);
11854 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
11856 location_t saved_location
= input_location
;
11857 gimple_stmt_iterator gsi
;
11858 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11859 lower_omp_1 (&gsi
, ctx
);
11860 /* During gimplification, we haven't folded statments inside offloading
11861 regions (gimplify.c:maybe_fold_stmt); do that now. */
11862 if (target_nesting_level
)
11863 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11865 input_location
= saved_location
;
11868 /* Main entry point. */
11870 static unsigned int
11871 execute_lower_omp (void)
11877 /* This pass always runs, to provide PROP_gimple_lomp.
11878 But often, there is nothing to do. */
11879 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
11880 && flag_openmp_simd
== 0)
11883 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
11884 delete_omp_context
);
11886 body
= gimple_body (current_function_decl
);
11887 scan_omp (&body
, NULL
);
11888 gcc_assert (taskreg_nesting_level
== 0);
11889 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
11890 finish_taskreg_scan (ctx
);
11891 taskreg_contexts
.release ();
11893 if (all_contexts
->root
)
11895 if (task_shared_vars
)
11896 push_gimplify_context ();
11897 lower_omp (&body
, NULL
);
11898 if (task_shared_vars
)
11899 pop_gimplify_context (NULL
);
11904 splay_tree_delete (all_contexts
);
11905 all_contexts
= NULL
;
11907 BITMAP_FREE (task_shared_vars
);
11913 const pass_data pass_data_lower_omp
=
11915 GIMPLE_PASS
, /* type */
11916 "omplower", /* name */
11917 OPTGROUP_NONE
, /* optinfo_flags */
11918 TV_NONE
, /* tv_id */
11919 PROP_gimple_any
, /* properties_required */
11920 PROP_gimple_lomp
, /* properties_provided */
11921 0, /* properties_destroyed */
11922 0, /* todo_flags_start */
11923 0, /* todo_flags_finish */
11926 class pass_lower_omp
: public gimple_opt_pass
11929 pass_lower_omp (gcc::context
*ctxt
)
11930 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
11933 /* opt_pass methods: */
11934 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
11936 }; // class pass_lower_omp
11938 } // anon namespace
11941 make_pass_lower_omp (gcc::context
*ctxt
)
11943 return new pass_lower_omp (ctxt
);
11946 /* The following is a utility to diagnose structured block violations.
11947 It is not part of the "omplower" pass, as that's invoked too late. It
11948 should be invoked by the respective front ends after gimplification. */
11950 static splay_tree all_labels
;
11952 /* Check for mismatched contexts and generate an error if needed. Return
11953 true if an error is detected. */
11956 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
11957 gimple branch_ctx
, gimple label_ctx
)
11959 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
11960 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
11962 if (label_ctx
== branch_ctx
)
11965 const char* kind
= NULL
;
11970 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
11971 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
11973 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
11974 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
11975 kind
= "Cilk Plus";
11979 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
11980 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
11982 gcc_checking_assert (kind
== NULL
);
11988 gcc_checking_assert (flag_openmp
);
11993 Previously we kept track of the label's entire context in diagnose_sb_[12]
11994 so we could traverse it and issue a correct "exit" or "enter" error
11995 message upon a structured block violation.
11997 We built the context by building a list with tree_cons'ing, but there is
11998 no easy counterpart in gimple tuples. It seems like far too much work
11999 for issuing exit/enter error messages. If someone really misses the
12000 distinct error message... patches welcome.
12004 /* Try to avoid confusing the user by producing and error message
12005 with correct "exit" or "enter" verbiage. We prefer "exit"
12006 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12007 if (branch_ctx
== NULL
)
12013 if (TREE_VALUE (label_ctx
) == branch_ctx
)
12018 label_ctx
= TREE_CHAIN (label_ctx
);
12023 error ("invalid exit from %s structured block", kind
);
12025 error ("invalid entry to %s structured block", kind
);
12028 /* If it's obvious we have an invalid entry, be specific about the error. */
12029 if (branch_ctx
== NULL
)
12030 error ("invalid entry to %s structured block", kind
);
12033 /* Otherwise, be vague and lazy, but efficient. */
12034 error ("invalid branch to/from %s structured block", kind
);
12037 gsi_replace (gsi_p
, gimple_build_nop (), false);
12041 /* Pass 1: Create a minimal tree of structured blocks, and record
12042 where each label is found. */
12045 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12046 struct walk_stmt_info
*wi
)
12048 gimple context
= (gimple
) wi
->info
;
12049 gimple inner_context
;
12050 gimple stmt
= gsi_stmt (*gsi_p
);
12052 *handled_ops_p
= true;
12054 switch (gimple_code (stmt
))
12058 case GIMPLE_OMP_PARALLEL
:
12059 case GIMPLE_OMP_TASK
:
12060 case GIMPLE_OMP_SECTIONS
:
12061 case GIMPLE_OMP_SINGLE
:
12062 case GIMPLE_OMP_SECTION
:
12063 case GIMPLE_OMP_MASTER
:
12064 case GIMPLE_OMP_ORDERED
:
12065 case GIMPLE_OMP_CRITICAL
:
12066 case GIMPLE_OMP_TARGET
:
12067 case GIMPLE_OMP_TEAMS
:
12068 case GIMPLE_OMP_TASKGROUP
:
12069 /* The minimal context here is just the current OMP construct. */
12070 inner_context
= stmt
;
12071 wi
->info
= inner_context
;
12072 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12073 wi
->info
= context
;
12076 case GIMPLE_OMP_FOR
:
12077 inner_context
= stmt
;
12078 wi
->info
= inner_context
;
12079 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12081 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
12082 diagnose_sb_1
, NULL
, wi
);
12083 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12084 wi
->info
= context
;
12088 splay_tree_insert (all_labels
,
12089 (splay_tree_key
) gimple_label_label (
12090 as_a
<glabel
*> (stmt
)),
12091 (splay_tree_value
) context
);
12101 /* Pass 2: Check each branch and see if its context differs from that of
12102 the destination label's context. */
12105 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12106 struct walk_stmt_info
*wi
)
12108 gimple context
= (gimple
) wi
->info
;
12110 gimple stmt
= gsi_stmt (*gsi_p
);
12112 *handled_ops_p
= true;
12114 switch (gimple_code (stmt
))
12118 case GIMPLE_OMP_PARALLEL
:
12119 case GIMPLE_OMP_TASK
:
12120 case GIMPLE_OMP_SECTIONS
:
12121 case GIMPLE_OMP_SINGLE
:
12122 case GIMPLE_OMP_SECTION
:
12123 case GIMPLE_OMP_MASTER
:
12124 case GIMPLE_OMP_ORDERED
:
12125 case GIMPLE_OMP_CRITICAL
:
12126 case GIMPLE_OMP_TARGET
:
12127 case GIMPLE_OMP_TEAMS
:
12128 case GIMPLE_OMP_TASKGROUP
:
12130 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12131 wi
->info
= context
;
12134 case GIMPLE_OMP_FOR
:
12136 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12138 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
12139 diagnose_sb_2
, NULL
, wi
);
12140 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12141 wi
->info
= context
;
12146 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
12147 tree lab
= gimple_cond_true_label (cond_stmt
);
12150 n
= splay_tree_lookup (all_labels
,
12151 (splay_tree_key
) lab
);
12152 diagnose_sb_0 (gsi_p
, context
,
12153 n
? (gimple
) n
->value
: NULL
);
12155 lab
= gimple_cond_false_label (cond_stmt
);
12158 n
= splay_tree_lookup (all_labels
,
12159 (splay_tree_key
) lab
);
12160 diagnose_sb_0 (gsi_p
, context
,
12161 n
? (gimple
) n
->value
: NULL
);
12168 tree lab
= gimple_goto_dest (stmt
);
12169 if (TREE_CODE (lab
) != LABEL_DECL
)
12172 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12173 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
12177 case GIMPLE_SWITCH
:
12179 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
12181 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
12183 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
12184 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12185 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
12191 case GIMPLE_RETURN
:
12192 diagnose_sb_0 (gsi_p
, context
, NULL
);
12202 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12205 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
12208 gimple last
= last_stmt (bb
);
12209 enum gimple_code code
= gimple_code (last
);
12210 struct omp_region
*cur_region
= *region
;
12211 bool fallthru
= false;
12215 case GIMPLE_OMP_PARALLEL
:
12216 case GIMPLE_OMP_TASK
:
12217 case GIMPLE_OMP_FOR
:
12218 case GIMPLE_OMP_SINGLE
:
12219 case GIMPLE_OMP_TEAMS
:
12220 case GIMPLE_OMP_MASTER
:
12221 case GIMPLE_OMP_TASKGROUP
:
12222 case GIMPLE_OMP_ORDERED
:
12223 case GIMPLE_OMP_CRITICAL
:
12224 case GIMPLE_OMP_SECTION
:
12225 cur_region
= new_omp_region (bb
, code
, cur_region
);
12229 case GIMPLE_OMP_TARGET
:
12230 cur_region
= new_omp_region (bb
, code
, cur_region
);
12232 switch (gimple_omp_target_kind (last
))
12234 case GF_OMP_TARGET_KIND_REGION
:
12235 case GF_OMP_TARGET_KIND_DATA
:
12236 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12237 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12238 case GF_OMP_TARGET_KIND_OACC_DATA
:
12240 case GF_OMP_TARGET_KIND_UPDATE
:
12241 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12242 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12243 cur_region
= cur_region
->outer
;
12246 gcc_unreachable ();
12250 case GIMPLE_OMP_SECTIONS
:
12251 cur_region
= new_omp_region (bb
, code
, cur_region
);
12255 case GIMPLE_OMP_SECTIONS_SWITCH
:
12259 case GIMPLE_OMP_ATOMIC_LOAD
:
12260 case GIMPLE_OMP_ATOMIC_STORE
:
12264 case GIMPLE_OMP_RETURN
:
12265 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12266 somewhere other than the next block. This will be
12268 cur_region
->exit
= bb
;
12269 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
12270 cur_region
= cur_region
->outer
;
12273 case GIMPLE_OMP_CONTINUE
:
12274 cur_region
->cont
= bb
;
12275 switch (cur_region
->type
)
12277 case GIMPLE_OMP_FOR
:
12278 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12279 succs edges as abnormal to prevent splitting
12281 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
12282 /* Make the loopback edge. */
12283 make_edge (bb
, single_succ (cur_region
->entry
),
12286 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12287 corresponds to the case that the body of the loop
12288 is not executed at all. */
12289 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
12290 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
12294 case GIMPLE_OMP_SECTIONS
:
12295 /* Wire up the edges into and out of the nested sections. */
12297 basic_block switch_bb
= single_succ (cur_region
->entry
);
12299 struct omp_region
*i
;
12300 for (i
= cur_region
->inner
; i
; i
= i
->next
)
12302 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
12303 make_edge (switch_bb
, i
->entry
, 0);
12304 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
12307 /* Make the loopback edge to the block with
12308 GIMPLE_OMP_SECTIONS_SWITCH. */
12309 make_edge (bb
, switch_bb
, 0);
12311 /* Make the edge from the switch to exit. */
12312 make_edge (switch_bb
, bb
->next_bb
, 0);
12318 gcc_unreachable ();
12323 gcc_unreachable ();
12326 if (*region
!= cur_region
)
12328 *region
= cur_region
;
12330 *region_idx
= cur_region
->entry
->index
;
12338 static unsigned int
12339 diagnose_omp_structured_block_errors (void)
12341 struct walk_stmt_info wi
;
12342 gimple_seq body
= gimple_body (current_function_decl
);
12344 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
12346 memset (&wi
, 0, sizeof (wi
));
12347 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
12349 memset (&wi
, 0, sizeof (wi
));
12350 wi
.want_locations
= true;
12351 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
12353 gimple_set_body (current_function_decl
, body
);
12355 splay_tree_delete (all_labels
);
12363 const pass_data pass_data_diagnose_omp_blocks
=
12365 GIMPLE_PASS
, /* type */
12366 "*diagnose_omp_blocks", /* name */
12367 OPTGROUP_NONE
, /* optinfo_flags */
12368 TV_NONE
, /* tv_id */
12369 PROP_gimple_any
, /* properties_required */
12370 0, /* properties_provided */
12371 0, /* properties_destroyed */
12372 0, /* todo_flags_start */
12373 0, /* todo_flags_finish */
12376 class pass_diagnose_omp_blocks
: public gimple_opt_pass
12379 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12380 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
12383 /* opt_pass methods: */
12384 virtual bool gate (function
*)
12386 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
12388 virtual unsigned int execute (function
*)
12390 return diagnose_omp_structured_block_errors ();
12393 }; // class pass_diagnose_omp_blocks
12395 } // anon namespace
12398 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12400 return new pass_diagnose_omp_blocks (ctxt
);
12403 /* SIMD clone supporting code. */
12405 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12406 of arguments to reserve space for. */
12408 static struct cgraph_simd_clone
*
12409 simd_clone_struct_alloc (int nargs
)
12411 struct cgraph_simd_clone
*clone_info
;
12412 size_t len
= (sizeof (struct cgraph_simd_clone
)
12413 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
12414 clone_info
= (struct cgraph_simd_clone
*)
12415 ggc_internal_cleared_alloc (len
);
12419 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12422 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
12423 struct cgraph_simd_clone
*from
)
12425 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
12426 + ((from
->nargs
- from
->inbranch
)
12427 * sizeof (struct cgraph_simd_clone_arg
))));
12430 /* Return vector of parameter types of function FNDECL. This uses
12431 TYPE_ARG_TYPES if available, otherwise falls back to types of
12432 DECL_ARGUMENTS types. */
12435 simd_clone_vector_of_formal_parm_types (tree fndecl
)
12437 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
12438 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
12439 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
12442 FOR_EACH_VEC_ELT (args
, i
, arg
)
12443 args
[i
] = TREE_TYPE (args
[i
]);
12447 /* Given a simd function in NODE, extract the simd specific
12448 information from the OMP clauses passed in CLAUSES, and return
12449 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12450 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12451 otherwise set to FALSE. */
12453 static struct cgraph_simd_clone
*
12454 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
12455 bool *inbranch_specified
)
12457 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12460 *inbranch_specified
= false;
12462 n
= args
.length ();
12463 if (n
> 0 && args
.last () == void_type_node
)
12466 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12467 be cloned have a distinctive artificial label in addition to "omp
12471 && lookup_attribute ("cilk simd function",
12472 DECL_ATTRIBUTES (node
->decl
)));
12474 /* Allocate one more than needed just in case this is an in-branch
12475 clone which will require a mask argument. */
12476 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
12477 clone_info
->nargs
= n
;
12478 clone_info
->cilk_elemental
= cilk_clone
;
12485 clauses
= TREE_VALUE (clauses
);
12486 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
12489 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
12491 switch (OMP_CLAUSE_CODE (t
))
12493 case OMP_CLAUSE_INBRANCH
:
12494 clone_info
->inbranch
= 1;
12495 *inbranch_specified
= true;
12497 case OMP_CLAUSE_NOTINBRANCH
:
12498 clone_info
->inbranch
= 0;
12499 *inbranch_specified
= true;
12501 case OMP_CLAUSE_SIMDLEN
:
12502 clone_info
->simdlen
12503 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
12505 case OMP_CLAUSE_LINEAR
:
12507 tree decl
= OMP_CLAUSE_DECL (t
);
12508 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
12509 int argno
= TREE_INT_CST_LOW (decl
);
12510 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
12512 clone_info
->args
[argno
].arg_type
12513 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
12514 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12515 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
12516 && clone_info
->args
[argno
].linear_step
< n
);
12520 if (POINTER_TYPE_P (args
[argno
]))
12521 step
= fold_convert (ssizetype
, step
);
12522 if (!tree_fits_shwi_p (step
))
12524 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12525 "ignoring large linear step");
12529 else if (integer_zerop (step
))
12531 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12532 "ignoring zero linear step");
12538 clone_info
->args
[argno
].arg_type
12539 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
12540 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12545 case OMP_CLAUSE_UNIFORM
:
12547 tree decl
= OMP_CLAUSE_DECL (t
);
12548 int argno
= tree_to_uhwi (decl
);
12549 clone_info
->args
[argno
].arg_type
12550 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
12553 case OMP_CLAUSE_ALIGNED
:
12555 tree decl
= OMP_CLAUSE_DECL (t
);
12556 int argno
= tree_to_uhwi (decl
);
12557 clone_info
->args
[argno
].alignment
12558 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
12569 /* Given a SIMD clone in NODE, calculate the characteristic data
12570 type and return the coresponding type. The characteristic data
12571 type is computed as described in the Intel Vector ABI. */
12574 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
12575 struct cgraph_simd_clone
*clone_info
)
12577 tree type
= integer_type_node
;
12578 tree fndecl
= node
->decl
;
12580 /* a) For non-void function, the characteristic data type is the
12582 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
12583 type
= TREE_TYPE (TREE_TYPE (fndecl
));
12585 /* b) If the function has any non-uniform, non-linear parameters,
12586 then the characteristic data type is the type of the first
12590 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
12591 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
12592 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
12600 /* c) If the characteristic data type determined by a) or b) above
12601 is struct, union, or class type which is pass-by-value (except
12602 for the type that maps to the built-in complex data type), the
12603 characteristic data type is int. */
12604 if (RECORD_OR_UNION_TYPE_P (type
)
12605 && !aggregate_value_p (type
, NULL
)
12606 && TREE_CODE (type
) != COMPLEX_TYPE
)
12607 return integer_type_node
;
12609 /* d) If none of the above three classes is applicable, the
12610 characteristic data type is int. */
12614 /* e) For Intel Xeon Phi native and offload compilation, if the
12615 resulting characteristic data type is 8-bit or 16-bit integer
12616 data type, the characteristic data type is int. */
12617 /* Well, we don't handle Xeon Phi yet. */
12621 simd_clone_mangle (struct cgraph_node
*node
,
12622 struct cgraph_simd_clone
*clone_info
)
12624 char vecsize_mangle
= clone_info
->vecsize_mangle
;
12625 char mask
= clone_info
->inbranch
? 'M' : 'N';
12626 unsigned int simdlen
= clone_info
->simdlen
;
12630 gcc_assert (vecsize_mangle
&& simdlen
);
12632 pp_string (&pp
, "_ZGV");
12633 pp_character (&pp
, vecsize_mangle
);
12634 pp_character (&pp
, mask
);
12635 pp_decimal_int (&pp
, simdlen
);
12637 for (n
= 0; n
< clone_info
->nargs
; ++n
)
12639 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
12641 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
12642 pp_character (&pp
, 'u');
12643 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12645 gcc_assert (arg
.linear_step
!= 0);
12646 pp_character (&pp
, 'l');
12647 if (arg
.linear_step
> 1)
12648 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12649 else if (arg
.linear_step
< 0)
12651 pp_character (&pp
, 'n');
12652 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
12656 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
12658 pp_character (&pp
, 's');
12659 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12662 pp_character (&pp
, 'v');
12665 pp_character (&pp
, 'a');
12666 pp_decimal_int (&pp
, arg
.alignment
);
12670 pp_underscore (&pp
);
12671 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
12674 pp_string (&pp
, str
);
12675 str
= pp_formatted_text (&pp
);
12677 /* If there already is a SIMD clone with the same mangled name, don't
12678 add another one. This can happen e.g. for
12679 #pragma omp declare simd
12680 #pragma omp declare simd simdlen(8)
12681 int foo (int, int);
12682 if the simdlen is assumed to be 8 for the first one, etc. */
12683 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
12684 clone
= clone
->simdclone
->next_clone
)
12685 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
12689 return get_identifier (str
);
12692 /* Create a simd clone of OLD_NODE and return it. */
12694 static struct cgraph_node
*
12695 simd_clone_create (struct cgraph_node
*old_node
)
12697 struct cgraph_node
*new_node
;
12698 if (old_node
->definition
)
12700 if (!old_node
->has_gimple_body_p ())
12702 old_node
->get_body ();
12703 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
12709 tree old_decl
= old_node
->decl
;
12710 tree new_decl
= copy_node (old_node
->decl
);
12711 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
12712 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
12713 SET_DECL_RTL (new_decl
, NULL
);
12714 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
12715 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
12716 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
12717 symtab
->call_cgraph_insertion_hooks (new_node
);
12719 if (new_node
== NULL
)
12722 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
12724 /* The function cgraph_function_versioning () will force the new
12725 symbol local. Undo this, and inherit external visability from
12727 new_node
->local
.local
= old_node
->local
.local
;
12728 new_node
->externally_visible
= old_node
->externally_visible
;
12733 /* Adjust the return type of the given function to its appropriate
12734 vector counterpart. Returns a simd array to be used throughout the
12735 function as a return value. */
12738 simd_clone_adjust_return_type (struct cgraph_node
*node
)
12740 tree fndecl
= node
->decl
;
12741 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
12742 unsigned int veclen
;
12745 /* Adjust the function return type. */
12746 if (orig_rettype
== void_type_node
)
12748 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
12749 t
= TREE_TYPE (TREE_TYPE (fndecl
));
12750 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
12751 veclen
= node
->simdclone
->vecsize_int
;
12753 veclen
= node
->simdclone
->vecsize_float
;
12754 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
12755 if (veclen
> node
->simdclone
->simdlen
)
12756 veclen
= node
->simdclone
->simdlen
;
12757 if (POINTER_TYPE_P (t
))
12758 t
= pointer_sized_int_node
;
12759 if (veclen
== node
->simdclone
->simdlen
)
12760 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
12763 t
= build_vector_type (t
, veclen
);
12764 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
12766 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
12767 if (!node
->definition
)
12770 t
= DECL_RESULT (fndecl
);
12771 /* Adjust the DECL_RESULT. */
12772 gcc_assert (TREE_TYPE (t
) != void_type_node
);
12773 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
12776 tree atype
= build_array_type_nelts (orig_rettype
,
12777 node
->simdclone
->simdlen
);
12778 if (veclen
!= node
->simdclone
->simdlen
)
12779 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
12781 /* Set up a SIMD array to use as the return value. */
12782 tree retval
= create_tmp_var_raw (atype
, "retval");
12783 gimple_add_tmp_var (retval
);
12787 /* Each vector argument has a corresponding array to be used locally
12788 as part of the eventual loop. Create such temporary array and
12791 PREFIX is the prefix to be used for the temporary.
12793 TYPE is the inner element type.
12795 SIMDLEN is the number of elements. */
12798 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
12800 tree atype
= build_array_type_nelts (type
, simdlen
);
12801 tree avar
= create_tmp_var_raw (atype
, prefix
);
12802 gimple_add_tmp_var (avar
);
12806 /* Modify the function argument types to their corresponding vector
12807 counterparts if appropriate. Also, create one array for each simd
12808 argument to be used locally when using the function arguments as
12811 NODE is the function whose arguments are to be adjusted.
12813 Returns an adjustment vector that will be filled describing how the
12814 argument types will be adjusted. */
12816 static ipa_parm_adjustment_vec
12817 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
12820 ipa_parm_adjustment_vec adjustments
;
12822 if (node
->definition
)
12823 args
= ipa_get_vector_of_formal_parms (node
->decl
);
12825 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12826 adjustments
.create (args
.length ());
12827 unsigned i
, j
, veclen
;
12828 struct ipa_parm_adjustment adj
;
12829 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
12831 memset (&adj
, 0, sizeof (adj
));
12832 tree parm
= args
[i
];
12833 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
12834 adj
.base_index
= i
;
12837 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
12838 node
->simdclone
->args
[i
].orig_type
= parm_type
;
12840 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
12842 /* No adjustment necessary for scalar arguments. */
12843 adj
.op
= IPA_PARM_OP_COPY
;
12847 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
12848 veclen
= node
->simdclone
->vecsize_int
;
12850 veclen
= node
->simdclone
->vecsize_float
;
12851 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
12852 if (veclen
> node
->simdclone
->simdlen
)
12853 veclen
= node
->simdclone
->simdlen
;
12854 adj
.arg_prefix
= "simd";
12855 if (POINTER_TYPE_P (parm_type
))
12856 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12858 adj
.type
= build_vector_type (parm_type
, veclen
);
12859 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
12860 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12862 adjustments
.safe_push (adj
);
12865 memset (&adj
, 0, sizeof (adj
));
12866 adj
.op
= IPA_PARM_OP_NEW
;
12867 adj
.arg_prefix
= "simd";
12868 adj
.base_index
= i
;
12869 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
12873 if (node
->definition
)
12874 node
->simdclone
->args
[i
].simd_array
12875 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
12876 parm_type
, node
->simdclone
->simdlen
);
12878 adjustments
.safe_push (adj
);
12881 if (node
->simdclone
->inbranch
)
12884 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
12887 memset (&adj
, 0, sizeof (adj
));
12888 adj
.op
= IPA_PARM_OP_NEW
;
12889 adj
.arg_prefix
= "mask";
12891 adj
.base_index
= i
;
12892 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
12893 veclen
= node
->simdclone
->vecsize_int
;
12895 veclen
= node
->simdclone
->vecsize_float
;
12896 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
12897 if (veclen
> node
->simdclone
->simdlen
)
12898 veclen
= node
->simdclone
->simdlen
;
12899 if (POINTER_TYPE_P (base_type
))
12900 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12902 adj
.type
= build_vector_type (base_type
, veclen
);
12903 adjustments
.safe_push (adj
);
12905 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12906 adjustments
.safe_push (adj
);
12908 /* We have previously allocated one extra entry for the mask. Use
12910 struct cgraph_simd_clone
*sc
= node
->simdclone
;
12912 if (node
->definition
)
12914 sc
->args
[i
].orig_arg
12915 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
12916 sc
->args
[i
].simd_array
12917 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
12919 sc
->args
[i
].orig_type
= base_type
;
12920 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
12923 if (node
->definition
)
12924 ipa_modify_formal_parameters (node
->decl
, adjustments
);
12927 tree new_arg_types
= NULL_TREE
, new_reversed
;
12928 bool last_parm_void
= false;
12929 if (args
.length () > 0 && args
.last () == void_type_node
)
12930 last_parm_void
= true;
12932 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
12933 j
= adjustments
.length ();
12934 for (i
= 0; i
< j
; i
++)
12936 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
12938 if (adj
->op
== IPA_PARM_OP_COPY
)
12939 ptype
= args
[adj
->base_index
];
12942 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
12944 new_reversed
= nreverse (new_arg_types
);
12945 if (last_parm_void
)
12948 TREE_CHAIN (new_arg_types
) = void_list_node
;
12950 new_reversed
= void_list_node
;
12953 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
12954 TYPE_ARG_TYPES (new_type
) = new_reversed
;
12955 TREE_TYPE (node
->decl
) = new_type
;
12957 adjustments
.release ();
12960 return adjustments
;
12963 /* Initialize and copy the function arguments in NODE to their
12964 corresponding local simd arrays. Returns a fresh gimple_seq with
12965 the instruction sequence generated. */
12968 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
12969 ipa_parm_adjustment_vec adjustments
)
12971 gimple_seq seq
= NULL
;
12972 unsigned i
= 0, j
= 0, k
;
12974 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
12976 arg
= DECL_CHAIN (arg
), i
++, j
++)
12978 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
12981 node
->simdclone
->args
[i
].vector_arg
= arg
;
12983 tree array
= node
->simdclone
->args
[i
].simd_array
;
12984 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
12986 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12987 tree ptr
= build_fold_addr_expr (array
);
12988 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
12989 build_int_cst (ptype
, 0));
12990 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
12991 gimplify_and_add (t
, &seq
);
12995 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
12996 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12997 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
12999 tree ptr
= build_fold_addr_expr (array
);
13003 arg
= DECL_CHAIN (arg
);
13007 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
13008 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
13009 build_int_cst (ptype
, k
* elemsize
));
13010 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
13011 gimplify_and_add (t
, &seq
);
13018 /* Callback info for ipa_simd_modify_stmt_ops below. */
13020 struct modify_stmt_info
{
13021 ipa_parm_adjustment_vec adjustments
;
13023 /* True if the parent statement was modified by
13024 ipa_simd_modify_stmt_ops. */
13028 /* Callback for walk_gimple_op.
13030 Adjust operands from a given statement as specified in the
13031 adjustments vector in the callback data. */
13034 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
13036 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13037 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
13038 tree
*orig_tp
= tp
;
13039 if (TREE_CODE (*tp
) == ADDR_EXPR
)
13040 tp
= &TREE_OPERAND (*tp
, 0);
13041 struct ipa_parm_adjustment
*cand
= NULL
;
13042 if (TREE_CODE (*tp
) == PARM_DECL
)
13043 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
13047 *walk_subtrees
= 0;
13050 tree repl
= NULL_TREE
;
13052 repl
= unshare_expr (cand
->new_decl
);
13057 *walk_subtrees
= 0;
13058 bool modified
= info
->modified
;
13059 info
->modified
= false;
13060 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
13061 if (!info
->modified
)
13063 info
->modified
= modified
;
13066 info
->modified
= modified
;
13075 repl
= build_fold_addr_expr (repl
);
13077 if (is_gimple_debug (info
->stmt
))
13079 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
13080 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
13081 DECL_ARTIFICIAL (vexpr
) = 1;
13082 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
13083 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
13088 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
13089 repl
= gimple_assign_lhs (stmt
);
13091 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
13092 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13095 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
13097 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
13103 info
->modified
= true;
13107 /* Traverse the function body and perform all modifications as
13108 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13109 modified such that the replacement/reduction value will now be an
13110 offset into the corresponding simd_array.
13112 This function will replace all function argument uses with their
13113 corresponding simd array elements, and ajust the return values
13117 ipa_simd_modify_function_body (struct cgraph_node
*node
,
13118 ipa_parm_adjustment_vec adjustments
,
13119 tree retval_array
, tree iter
)
13122 unsigned int i
, j
, l
;
13124 /* Re-use the adjustments array, but this time use it to replace
13125 every function argument use to an offset into the corresponding
13127 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
13129 if (!node
->simdclone
->args
[i
].vector_arg
)
13132 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
13133 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
13134 adjustments
[j
].new_decl
13135 = build4 (ARRAY_REF
,
13137 node
->simdclone
->args
[i
].simd_array
,
13139 NULL_TREE
, NULL_TREE
);
13140 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
13141 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
13142 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
13145 l
= adjustments
.length ();
13146 for (i
= 1; i
< num_ssa_names
; i
++)
13148 tree name
= ssa_name (i
);
13150 && SSA_NAME_VAR (name
)
13151 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
13153 for (j
= 0; j
< l
; j
++)
13154 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
13155 && adjustments
[j
].new_decl
)
13158 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
13161 = copy_var_decl (adjustments
[j
].base
,
13162 DECL_NAME (adjustments
[j
].base
),
13163 TREE_TYPE (adjustments
[j
].base
));
13164 adjustments
[j
].new_ssa_base
= base_var
;
13167 base_var
= adjustments
[j
].new_ssa_base
;
13168 if (SSA_NAME_IS_DEFAULT_DEF (name
))
13170 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13171 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
13172 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
13173 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
13174 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13175 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
13176 gimple stmt
= gimple_build_assign (name
, new_decl
);
13177 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13180 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13185 struct modify_stmt_info info
;
13186 info
.adjustments
= adjustments
;
13188 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
13190 gimple_stmt_iterator gsi
;
13192 gsi
= gsi_start_bb (bb
);
13193 while (!gsi_end_p (gsi
))
13195 gimple stmt
= gsi_stmt (gsi
);
13197 struct walk_stmt_info wi
;
13199 memset (&wi
, 0, sizeof (wi
));
13200 info
.modified
= false;
13202 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
13204 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
13206 tree retval
= gimple_return_retval (return_stmt
);
13209 gsi_remove (&gsi
, true);
13213 /* Replace `return foo' with `retval_array[iter] = foo'. */
13214 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
13215 retval_array
, iter
, NULL
, NULL
);
13216 stmt
= gimple_build_assign (ref
, retval
);
13217 gsi_replace (&gsi
, stmt
, true);
13218 info
.modified
= true;
13223 update_stmt (stmt
);
13224 if (maybe_clean_eh_stmt (stmt
))
13225 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
13232 /* Adjust the argument types in NODE to their appropriate vector
13236 simd_clone_adjust (struct cgraph_node
*node
)
13238 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
13240 targetm
.simd_clone
.adjust (node
);
13242 tree retval
= simd_clone_adjust_return_type (node
);
13243 ipa_parm_adjustment_vec adjustments
13244 = simd_clone_adjust_argument_types (node
);
13246 push_gimplify_context ();
13248 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
13250 /* Adjust all uses of vector arguments accordingly. Adjust all
13251 return values accordingly. */
13252 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
13253 tree iter1
= make_ssa_name (iter
);
13254 tree iter2
= make_ssa_name (iter
);
13255 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
13257 /* Initialize the iteration variable. */
13258 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13259 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
13260 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
13261 /* Insert the SIMD array and iv initialization at function
13263 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
13265 pop_gimplify_context (NULL
);
13267 /* Create a new BB right before the original exit BB, to hold the
13268 iteration increment and the condition/branch. */
13269 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
13270 basic_block incr_bb
= create_empty_bb (orig_exit
);
13271 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
13272 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13273 flag. Set it now to be a FALLTHRU_EDGE. */
13274 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
13275 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
13276 for (unsigned i
= 0;
13277 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
13279 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
13280 redirect_edge_succ (e
, incr_bb
);
13282 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
13283 e
->probability
= REG_BR_PROB_BASE
;
13284 gsi
= gsi_last_bb (incr_bb
);
13285 gimple g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
13286 build_int_cst (unsigned_type_node
, 1));
13287 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13289 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13290 struct loop
*loop
= alloc_loop ();
13291 cfun
->has_force_vectorize_loops
= true;
13292 loop
->safelen
= node
->simdclone
->simdlen
;
13293 loop
->force_vectorize
= true;
13294 loop
->header
= body_bb
;
13296 /* Branch around the body if the mask applies. */
13297 if (node
->simdclone
->inbranch
)
13299 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
13301 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
13302 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
13303 tree aref
= build4 (ARRAY_REF
,
13304 TREE_TYPE (TREE_TYPE (mask_array
)),
13307 g
= gimple_build_assign (mask
, aref
);
13308 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13309 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
13310 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
13312 aref
= build1 (VIEW_CONVERT_EXPR
,
13313 build_nonstandard_integer_type (bitsize
, 0), mask
);
13314 mask
= make_ssa_name (TREE_TYPE (aref
));
13315 g
= gimple_build_assign (mask
, aref
);
13316 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13319 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
13321 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13322 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
13323 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
13326 /* Generate the condition. */
13327 g
= gimple_build_cond (LT_EXPR
,
13329 build_int_cst (unsigned_type_node
,
13330 node
->simdclone
->simdlen
),
13332 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13333 e
= split_block (incr_bb
, gsi_stmt (gsi
));
13334 basic_block latch_bb
= e
->dest
;
13335 basic_block new_exit_bb
;
13336 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
13337 loop
->latch
= latch_bb
;
13339 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
13341 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
13342 /* The successor of incr_bb is already pointing to latch_bb; just
13344 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13345 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
13347 gphi
*phi
= create_phi_node (iter1
, body_bb
);
13348 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
13349 edge latch_edge
= single_succ_edge (latch_bb
);
13350 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
13352 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13354 /* Generate the new return. */
13355 gsi
= gsi_last_bb (new_exit_bb
);
13357 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
13358 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
13359 retval
= TREE_OPERAND (retval
, 0);
13362 retval
= build1 (VIEW_CONVERT_EXPR
,
13363 TREE_TYPE (TREE_TYPE (node
->decl
)),
13365 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
13366 false, GSI_CONTINUE_LINKING
);
13368 g
= gimple_build_return (retval
);
13369 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13371 /* Handle aligned clauses by replacing default defs of the aligned
13372 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13373 lhs. Handle linear by adding PHIs. */
13374 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
13375 if (node
->simdclone
->args
[i
].alignment
13376 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13377 && (node
->simdclone
->args
[i
].alignment
13378 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
13379 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13382 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
13383 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13384 tree def
= ssa_default_def (cfun
, orig_arg
);
13385 if (def
&& !has_zero_uses (def
))
13387 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
13388 gimple_seq seq
= NULL
;
13389 bool need_cvt
= false;
13391 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
13393 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
13396 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
13397 gimple_call_set_lhs (g
, t
);
13398 gimple_seq_add_stmt_without_update (&seq
, g
);
13401 t
= make_ssa_name (orig_arg
);
13402 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
13403 gimple_seq_add_stmt_without_update (&seq
, g
);
13405 gsi_insert_seq_on_edge_immediate
13406 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
13408 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13409 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
13411 node
->create_edge (cgraph_node::get_create (fn
),
13412 call
, entry_bb
->count
, freq
);
13414 imm_use_iterator iter
;
13415 use_operand_p use_p
;
13417 tree repl
= gimple_get_lhs (g
);
13418 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13419 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
13422 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13423 SET_USE (use_p
, repl
);
13426 else if (node
->simdclone
->args
[i
].arg_type
13427 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
13429 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13430 tree def
= ssa_default_def (cfun
, orig_arg
);
13431 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13432 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
13433 if (def
&& !has_zero_uses (def
))
13435 iter1
= make_ssa_name (orig_arg
);
13436 iter2
= make_ssa_name (orig_arg
);
13437 phi
= create_phi_node (iter1
, body_bb
);
13438 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
13439 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13440 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13441 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
13442 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13443 ? TREE_TYPE (orig_arg
) : sizetype
;
13445 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
13446 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
13447 gsi
= gsi_last_bb (incr_bb
);
13448 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13450 imm_use_iterator iter
;
13451 use_operand_p use_p
;
13453 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13454 if (use_stmt
== phi
)
13457 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13458 SET_USE (use_p
, iter1
);
13462 calculate_dominance_info (CDI_DOMINATORS
);
13463 add_loop (loop
, loop
->header
->loop_father
);
13464 update_ssa (TODO_update_ssa
);
13469 /* If the function in NODE is tagged as an elemental SIMD function,
13470 create the appropriate SIMD clones. */
13473 expand_simd_clones (struct cgraph_node
*node
)
13475 tree attr
= lookup_attribute ("omp declare simd",
13476 DECL_ATTRIBUTES (node
->decl
));
13477 if (attr
== NULL_TREE
13478 || node
->global
.inlined_to
13479 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
13483 #pragma omp declare simd
13485 in C, there we don't know the argument types at all. */
13486 if (!node
->definition
13487 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
13492 /* Start with parsing the "omp declare simd" attribute(s). */
13493 bool inbranch_clause_specified
;
13494 struct cgraph_simd_clone
*clone_info
13495 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
13496 &inbranch_clause_specified
);
13497 if (clone_info
== NULL
)
13500 int orig_simdlen
= clone_info
->simdlen
;
13501 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
13502 /* The target can return 0 (no simd clones should be created),
13503 1 (just one ISA of simd clones should be created) or higher
13504 count of ISA variants. In that case, clone_info is initialized
13505 for the first ISA variant. */
13507 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
13512 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13513 also create one inbranch and one !inbranch clone of it. */
13514 for (int i
= 0; i
< count
* 2; i
++)
13516 struct cgraph_simd_clone
*clone
= clone_info
;
13517 if (inbranch_clause_specified
&& (i
& 1) != 0)
13522 clone
= simd_clone_struct_alloc (clone_info
->nargs
13524 simd_clone_struct_copy (clone
, clone_info
);
13525 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13526 and simd_clone_adjust_argument_types did to the first
13528 clone
->nargs
-= clone_info
->inbranch
;
13529 clone
->simdlen
= orig_simdlen
;
13530 /* And call the target hook again to get the right ISA. */
13531 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
13535 clone
->inbranch
= 1;
13538 /* simd_clone_mangle might fail if such a clone has been created
13540 tree id
= simd_clone_mangle (node
, clone
);
13541 if (id
== NULL_TREE
)
13544 /* Only when we are sure we want to create the clone actually
13545 clone the function (or definitions) or create another
13546 extern FUNCTION_DECL (for prototypes without definitions). */
13547 struct cgraph_node
*n
= simd_clone_create (node
);
13551 n
->simdclone
= clone
;
13552 clone
->origin
= node
;
13553 clone
->next_clone
= NULL
;
13554 if (node
->simd_clones
== NULL
)
13556 clone
->prev_clone
= n
;
13557 node
->simd_clones
= n
;
13561 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
13562 clone
->prev_clone
->simdclone
->next_clone
= n
;
13563 node
->simd_clones
->simdclone
->prev_clone
= n
;
13565 symtab
->change_decl_assembler_name (n
->decl
, id
);
13566 /* And finally adjust the return type, parameters and for
13567 definitions also function body. */
13568 if (node
->definition
)
13569 simd_clone_adjust (n
);
13572 simd_clone_adjust_return_type (n
);
13573 simd_clone_adjust_argument_types (n
);
13577 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
13580 /* Entry point for IPA simd clone creation pass. */
13582 static unsigned int
13583 ipa_omp_simd_clone (void)
13585 struct cgraph_node
*node
;
13586 FOR_EACH_FUNCTION (node
)
13587 expand_simd_clones (node
);
13593 const pass_data pass_data_omp_simd_clone
=
13595 SIMPLE_IPA_PASS
, /* type */
13596 "simdclone", /* name */
13597 OPTGROUP_NONE
, /* optinfo_flags */
13598 TV_NONE
, /* tv_id */
13599 ( PROP_ssa
| PROP_cfg
), /* properties_required */
13600 0, /* properties_provided */
13601 0, /* properties_destroyed */
13602 0, /* todo_flags_start */
13603 0, /* todo_flags_finish */
13606 class pass_omp_simd_clone
: public simple_ipa_opt_pass
13609 pass_omp_simd_clone(gcc::context
*ctxt
)
13610 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
13613 /* opt_pass methods: */
13614 virtual bool gate (function
*);
13615 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
13619 pass_omp_simd_clone::gate (function
*)
13621 return ((flag_openmp
|| flag_openmp_simd
13623 || (in_lto_p
&& !flag_wpa
))
13624 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
13627 } // anon namespace
13629 simple_ipa_opt_pass
*
13630 make_pass_omp_simd_clone (gcc::context
*ctxt
)
13632 return new pass_omp_simd_clone (ctxt
);
13635 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
13636 adds their addresses and sizes to constructor-vector V_CTOR. */
13638 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
13639 vec
<constructor_elt
, va_gc
> *v_ctor
)
13641 unsigned len
= vec_safe_length (v_decls
);
13642 for (unsigned i
= 0; i
< len
; i
++)
13644 tree it
= (*v_decls
)[i
];
13645 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
13647 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
13649 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
13650 fold_convert (const_ptr_type_node
,
13651 DECL_SIZE_UNIT (it
)));
13655 /* Create new symbols containing (address, size) pairs for global variables,
13656 marked with "omp declare target" attribute, as well as addresses for the
13657 functions, which are outlined offloading regions. */
13659 omp_finish_file (void)
13661 unsigned num_funcs
= vec_safe_length (offload_funcs
);
13662 unsigned num_vars
= vec_safe_length (offload_vars
);
13664 if (num_funcs
== 0 && num_vars
== 0)
13667 if (targetm_common
.have_named_sections
)
13669 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
13670 vec_alloc (v_f
, num_funcs
);
13671 vec_alloc (v_v
, num_vars
* 2);
13673 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
13674 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
13676 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13678 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13680 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13681 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13682 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
13683 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
13684 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
13685 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
13686 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13687 get_identifier (".offload_func_table"),
13689 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13690 get_identifier (".offload_var_table"),
13692 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
13693 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
13694 otherwise a joint table in a binary will contain padding between
13695 tables from multiple object files. */
13696 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
13697 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
13698 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
13699 DECL_INITIAL (funcs_decl
) = ctor_f
;
13700 DECL_INITIAL (vars_decl
) = ctor_v
;
13701 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
13702 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
13704 varpool_node::finalize_decl (vars_decl
);
13705 varpool_node::finalize_decl (funcs_decl
);
13709 for (unsigned i
= 0; i
< num_funcs
; i
++)
13711 tree it
= (*offload_funcs
)[i
];
13712 targetm
.record_offload_symbol (it
);
13714 for (unsigned i
= 0; i
< num_vars
; i
++)
13716 tree it
= (*offload_vars
)[i
];
13717 targetm
.record_offload_symbol (it
);
13722 #include "gt-omp-low.h"