1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
32 #include "double-int.h"
39 #include "fold-const.h"
40 #include "stringpool.h"
41 #include "stor-layout.h"
44 #include "hard-reg-set.h"
46 #include "dominance.h"
49 #include "basic-block.h"
50 #include "tree-ssa-alias.h"
51 #include "internal-fn.h"
52 #include "gimple-fold.h"
53 #include "gimple-expr.h"
57 #include "gimple-iterator.h"
58 #include "gimplify-me.h"
59 #include "gimple-walk.h"
60 #include "tree-iterator.h"
61 #include "tree-inline.h"
62 #include "langhooks.h"
63 #include "diagnostic-core.h"
64 #include "gimple-ssa.h"
66 #include "plugin-api.h"
70 #include "tree-phinodes.h"
71 #include "ssa-iterators.h"
72 #include "tree-ssanames.h"
73 #include "tree-into-ssa.h"
76 #include "statistics.h"
78 #include "fixed-value.h"
79 #include "insn-config.h"
90 #include "tree-pass.h"
92 #include "splay-tree.h"
93 #include "insn-codes.h"
97 #include "common/common-target.h"
99 #include "gimple-low.h"
100 #include "tree-cfgcleanup.h"
101 #include "pretty-print.h"
102 #include "alloc-pool.h"
103 #include "symbol-summary.h"
104 #include "ipa-prop.h"
105 #include "tree-nested.h"
109 #include "lto-section-names.h"
110 #include "gomp-constants.h"
113 /* Lowering of OMP parallel and workshare constructs proceeds in two
114 phases. The first phase scans the function looking for OMP statements
115 and then for variables that must be replaced to satisfy data sharing
116 clauses. The second phase expands code for the constructs, as well as
117 re-gimplifying things when variables have been replaced with complex
120 Final code generation is done by pass_expand_omp. The flowgraph is
121 scanned for regions which are then moved to a new
122 function, to be invoked by the thread library, or offloaded. */
124 /* OMP region information. Every parallel and workshare
125 directive is enclosed between two markers, the OMP_* directive
126 and a corresponding OMP_RETURN statement. */
130 /* The enclosing region. */
131 struct omp_region
*outer
;
133 /* First child region. */
134 struct omp_region
*inner
;
136 /* Next peer region. */
137 struct omp_region
*next
;
139 /* Block containing the omp directive as its last stmt. */
142 /* Block containing the OMP_RETURN as its last stmt. */
145 /* Block containing the OMP_CONTINUE as its last stmt. */
148 /* If this is a combined parallel+workshare region, this is a list
149 of additional arguments needed by the combined parallel+workshare
151 vec
<tree
, va_gc
> *ws_args
;
153 /* The code for the omp directive of this region. */
154 enum gimple_code type
;
156 /* Schedule kind, only used for OMP_FOR type regions. */
157 enum omp_clause_schedule_kind sched_kind
;
159 /* True if this is a combined parallel+workshare region. */
160 bool is_combined_parallel
;
163 /* Levels of parallelism as defined by OpenACC. Increasing numbers
164 correspond to deeper loop nesting levels. */
166 #define MASK_WORKER 2
167 #define MASK_VECTOR 4
169 /* Context structure. Used to store information about each parallel
170 directive in the code. */
172 typedef struct omp_context
174 /* This field must be at the beginning, as we do "inheritance": Some
175 callback functions for tree-inline.c (e.g., omp_copy_decl)
176 receive a copy_body_data pointer that is up-casted to an
177 omp_context pointer. */
180 /* The tree of contexts corresponding to the encountered constructs. */
181 struct omp_context
*outer
;
184 /* Map variables to fields in a structure that allows communication
185 between sending and receiving threads. */
186 splay_tree field_map
;
191 /* These are used just by task contexts, if task firstprivate fn is
192 needed. srecord_type is used to communicate from the thread
193 that encountered the task construct to task firstprivate fn,
194 record_type is allocated by GOMP_task, initialized by task firstprivate
195 fn and passed to the task body fn. */
196 splay_tree sfield_map
;
199 /* A chain of variables to add to the top-level block surrounding the
200 construct. In the case of a parallel, this is in the child function. */
203 /* A map of reduction pointer variables. For accelerators, each
204 reduction variable is replaced with an array. Each thread, in turn,
205 is assigned to a slot on that array. */
206 splay_tree reduction_map
;
208 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
209 barriers should jump to during omplower pass. */
212 /* What to do with variables with implicitly determined sharing
214 enum omp_clause_default_kind default_kind
;
216 /* Nesting depth of this context. Used to beautify error messages re
217 invalid gotos. The outermost ctx is depth 1, with depth 0 being
218 reserved for the main body of the function. */
221 /* True if this parallel directive is nested within another. */
224 /* True if this construct can be cancelled. */
227 /* For OpenACC loops, a mask of gang, worker and vector used at
228 levels below this one. */
230 /* For OpenACC loops, a mask of gang, worker and vector used at
231 this level and above. For parallel and kernels clauses, a mask
232 indicating which of num_gangs/num_workers/num_vectors was used. */
236 /* A structure holding the elements of:
237 for (V = N1; V cond N2; V += STEP) [...] */
239 struct omp_for_data_loop
241 tree v
, n1
, n2
, step
;
242 enum tree_code cond_code
;
245 /* A structure describing the main elements of a parallel loop. */
249 struct omp_for_data_loop loop
;
254 bool have_nowait
, have_ordered
;
255 enum omp_clause_schedule_kind sched_kind
;
256 struct omp_for_data_loop
*loops
;
260 static splay_tree all_contexts
;
261 static int taskreg_nesting_level
;
262 static int target_nesting_level
;
263 static struct omp_region
*root_omp_region
;
264 static bitmap task_shared_vars
;
265 static vec
<omp_context
*> taskreg_contexts
;
267 static void scan_omp (gimple_seq
*, omp_context
*);
268 static tree
scan_omp_1_op (tree
*, int *, void *);
270 #define WALK_SUBSTMTS \
274 case GIMPLE_EH_FILTER: \
275 case GIMPLE_TRANSACTION: \
276 /* The sub-statements for these should be walked. */ \
277 *handled_ops_p = false; \
280 /* Helper function to get the name of the array containing the partial
281 reductions for OpenACC reductions. */
283 oacc_get_reduction_array_id (tree node
)
285 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
286 int len
= strlen ("OACC") + strlen (id
);
287 char *temp_name
= XALLOCAVEC (char, len
+ 1);
288 snprintf (temp_name
, len
+ 1, "OACC%s", id
);
289 return IDENTIFIER_POINTER (get_identifier (temp_name
));
292 /* Determine the number of threads OpenACC threads used to determine the
293 size of the array of partial reductions. Currently, this is num_gangs
294 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
295 because it is independed of the device used. */
298 oacc_max_threads (omp_context
*ctx
)
300 tree nthreads
, vector_length
, gangs
, clauses
;
302 gangs
= fold_convert (sizetype
, integer_one_node
);
303 vector_length
= gangs
;
305 /* The reduction clause may be nested inside a loop directive.
306 Scan for the innermost vector_length clause. */
307 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
309 if (gimple_code (oc
->stmt
) != GIMPLE_OMP_TARGET
310 || (gimple_omp_target_kind (oc
->stmt
)
311 != GF_OMP_TARGET_KIND_OACC_PARALLEL
))
314 clauses
= gimple_omp_target_clauses (oc
->stmt
);
316 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
318 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
320 OMP_CLAUSE_VECTOR_LENGTH_EXPR
323 vector_length
= fold_convert (sizetype
, integer_one_node
);
325 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
327 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
328 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
330 gangs
= fold_convert (sizetype
, integer_one_node
);
335 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
340 /* Holds offload tables with decls. */
341 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
343 /* Convenience function for calling scan_omp_1_op on tree operands. */
346 scan_omp_op (tree
*tp
, omp_context
*ctx
)
348 struct walk_stmt_info wi
;
350 memset (&wi
, 0, sizeof (wi
));
352 wi
.want_locations
= true;
354 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
357 static void lower_omp (gimple_seq
*, omp_context
*);
358 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
359 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
361 /* Find an OMP clause of type KIND within CLAUSES. */
364 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
366 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
367 if (OMP_CLAUSE_CODE (clauses
) == kind
)
373 /* Return true if CTX is for an omp parallel. */
376 is_parallel_ctx (omp_context
*ctx
)
378 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
382 /* Return true if CTX is for an omp task. */
385 is_task_ctx (omp_context
*ctx
)
387 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
391 /* Return true if CTX is for an omp parallel or omp task. */
394 is_taskreg_ctx (omp_context
*ctx
)
396 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
397 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
401 /* Return true if REGION is a combined parallel+workshare region. */
404 is_combined_parallel (struct omp_region
*region
)
406 return region
->is_combined_parallel
;
410 /* Extract the header elements of parallel loop FOR_STMT and store
414 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
415 struct omp_for_data_loop
*loops
)
417 tree t
, var
, *collapse_iter
, *collapse_count
;
418 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
419 struct omp_for_data_loop
*loop
;
421 struct omp_for_data_loop dummy_loop
;
422 location_t loc
= gimple_location (for_stmt
);
423 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
424 bool distribute
= gimple_omp_for_kind (for_stmt
)
425 == GF_OMP_FOR_KIND_DISTRIBUTE
;
427 fd
->for_stmt
= for_stmt
;
429 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
430 if (fd
->collapse
> 1)
433 fd
->loops
= &fd
->loop
;
435 fd
->have_nowait
= distribute
|| simd
;
436 fd
->have_ordered
= false;
437 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
438 fd
->chunk_size
= NULL_TREE
;
439 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
440 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
441 collapse_iter
= NULL
;
442 collapse_count
= NULL
;
444 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
445 switch (OMP_CLAUSE_CODE (t
))
447 case OMP_CLAUSE_NOWAIT
:
448 fd
->have_nowait
= true;
450 case OMP_CLAUSE_ORDERED
:
451 fd
->have_ordered
= true;
453 case OMP_CLAUSE_SCHEDULE
:
454 gcc_assert (!distribute
);
455 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
456 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
458 case OMP_CLAUSE_DIST_SCHEDULE
:
459 gcc_assert (distribute
);
460 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
462 case OMP_CLAUSE_COLLAPSE
:
463 if (fd
->collapse
> 1)
465 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
466 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
473 /* FIXME: for now map schedule(auto) to schedule(static).
474 There should be analysis to determine whether all iterations
475 are approximately the same amount of work (then schedule(static)
476 is best) or if it varies (then schedule(dynamic,N) is better). */
477 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
479 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
480 gcc_assert (fd
->chunk_size
== NULL
);
482 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
483 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
484 gcc_assert (fd
->chunk_size
== NULL
);
485 else if (fd
->chunk_size
== NULL
)
487 /* We only need to compute a default chunk size for ordered
488 static loops and dynamic loops. */
489 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
491 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
492 ? integer_zero_node
: integer_one_node
;
495 for (i
= 0; i
< fd
->collapse
; i
++)
497 if (fd
->collapse
== 1)
499 else if (loops
!= NULL
)
504 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
505 gcc_assert (SSA_VAR_P (loop
->v
));
506 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
507 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
508 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
509 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
511 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
512 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
513 switch (loop
->cond_code
)
519 gcc_assert (gimple_omp_for_kind (for_stmt
)
520 == GF_OMP_FOR_KIND_CILKSIMD
521 || (gimple_omp_for_kind (for_stmt
)
522 == GF_OMP_FOR_KIND_CILKFOR
));
525 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
526 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
528 loop
->n2
= fold_build2_loc (loc
,
529 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
530 build_int_cst (TREE_TYPE (loop
->n2
), 1));
531 loop
->cond_code
= LT_EXPR
;
534 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
535 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
537 loop
->n2
= fold_build2_loc (loc
,
538 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
539 build_int_cst (TREE_TYPE (loop
->n2
), 1));
540 loop
->cond_code
= GT_EXPR
;
546 t
= gimple_omp_for_incr (for_stmt
, i
);
547 gcc_assert (TREE_OPERAND (t
, 0) == var
);
548 switch (TREE_CODE (t
))
551 loop
->step
= TREE_OPERAND (t
, 1);
553 case POINTER_PLUS_EXPR
:
554 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
557 loop
->step
= TREE_OPERAND (t
, 1);
558 loop
->step
= fold_build1_loc (loc
,
559 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
567 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
568 && !fd
->have_ordered
))
570 if (fd
->collapse
== 1)
571 iter_type
= TREE_TYPE (loop
->v
);
573 || TYPE_PRECISION (iter_type
)
574 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
576 = build_nonstandard_integer_type
577 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
579 else if (iter_type
!= long_long_unsigned_type_node
)
581 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
582 iter_type
= long_long_unsigned_type_node
;
583 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
584 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
585 >= TYPE_PRECISION (iter_type
))
589 if (loop
->cond_code
== LT_EXPR
)
590 n
= fold_build2_loc (loc
,
591 PLUS_EXPR
, TREE_TYPE (loop
->v
),
592 loop
->n2
, loop
->step
);
595 if (TREE_CODE (n
) != INTEGER_CST
596 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
597 iter_type
= long_long_unsigned_type_node
;
599 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
600 > TYPE_PRECISION (iter_type
))
604 if (loop
->cond_code
== LT_EXPR
)
607 n2
= fold_build2_loc (loc
,
608 PLUS_EXPR
, TREE_TYPE (loop
->v
),
609 loop
->n2
, loop
->step
);
613 n1
= fold_build2_loc (loc
,
614 MINUS_EXPR
, TREE_TYPE (loop
->v
),
615 loop
->n2
, loop
->step
);
618 if (TREE_CODE (n1
) != INTEGER_CST
619 || TREE_CODE (n2
) != INTEGER_CST
620 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
621 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
622 iter_type
= long_long_unsigned_type_node
;
626 if (collapse_count
&& *collapse_count
== NULL
)
628 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
629 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
630 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
631 if (t
&& integer_zerop (t
))
632 count
= build_zero_cst (long_long_unsigned_type_node
);
633 else if ((i
== 0 || count
!= NULL_TREE
)
634 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
635 && TREE_CONSTANT (loop
->n1
)
636 && TREE_CONSTANT (loop
->n2
)
637 && TREE_CODE (loop
->step
) == INTEGER_CST
)
639 tree itype
= TREE_TYPE (loop
->v
);
641 if (POINTER_TYPE_P (itype
))
642 itype
= signed_type_for (itype
);
643 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
644 t
= fold_build2_loc (loc
,
646 fold_convert_loc (loc
, itype
, loop
->step
), t
);
647 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
648 fold_convert_loc (loc
, itype
, loop
->n2
));
649 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
650 fold_convert_loc (loc
, itype
, loop
->n1
));
651 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
652 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
653 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
654 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
655 fold_convert_loc (loc
, itype
,
658 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
659 fold_convert_loc (loc
, itype
, loop
->step
));
660 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
661 if (count
!= NULL_TREE
)
662 count
= fold_build2_loc (loc
,
663 MULT_EXPR
, long_long_unsigned_type_node
,
667 if (TREE_CODE (count
) != INTEGER_CST
)
670 else if (count
&& !integer_zerop (count
))
677 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
678 || fd
->have_ordered
))
680 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
681 iter_type
= long_long_unsigned_type_node
;
683 iter_type
= long_integer_type_node
;
685 else if (collapse_iter
&& *collapse_iter
!= NULL
)
686 iter_type
= TREE_TYPE (*collapse_iter
);
687 fd
->iter_type
= iter_type
;
688 if (collapse_iter
&& *collapse_iter
== NULL
)
689 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
690 if (collapse_count
&& *collapse_count
== NULL
)
693 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
695 *collapse_count
= create_tmp_var (iter_type
, ".count");
698 if (fd
->collapse
> 1)
700 fd
->loop
.v
= *collapse_iter
;
701 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
702 fd
->loop
.n2
= *collapse_count
;
703 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
704 fd
->loop
.cond_code
= LT_EXPR
;
707 /* For OpenACC loops, force a chunk size of one, as this avoids the default
708 scheduling where several subsequent iterations are being executed by the
710 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
712 gcc_assert (fd
->chunk_size
== NULL_TREE
);
713 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
718 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
719 is the immediate dominator of PAR_ENTRY_BB, return true if there
720 are no data dependencies that would prevent expanding the parallel
721 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
723 When expanding a combined parallel+workshare region, the call to
724 the child function may need additional arguments in the case of
725 GIMPLE_OMP_FOR regions. In some cases, these arguments are
726 computed out of variables passed in from the parent to the child
727 via 'struct .omp_data_s'. For instance:
729 #pragma omp parallel for schedule (guided, i * 4)
734 # BLOCK 2 (PAR_ENTRY_BB)
736 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
738 # BLOCK 3 (WS_ENTRY_BB)
739 .omp_data_i = &.omp_data_o;
740 D.1667 = .omp_data_i->i;
742 #pragma omp for schedule (guided, D.1598)
744 When we outline the parallel region, the call to the child function
745 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
746 that value is computed *after* the call site. So, in principle we
747 cannot do the transformation.
749 To see whether the code in WS_ENTRY_BB blocks the combined
750 parallel+workshare call, we collect all the variables used in the
751 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
752 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
755 FIXME. If we had the SSA form built at this point, we could merely
756 hoist the code in block 3 into block 2 and be done with it. But at
757 this point we don't have dataflow information and though we could
758 hack something up here, it is really not worth the aggravation. */
761 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
763 struct omp_for_data fd
;
764 gimple ws_stmt
= last_stmt (ws_entry_bb
);
766 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
769 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
771 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
773 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
775 if (fd
.iter_type
!= long_integer_type_node
)
778 /* FIXME. We give up too easily here. If any of these arguments
779 are not constants, they will likely involve variables that have
780 been mapped into fields of .omp_data_s for sharing with the child
781 function. With appropriate data flow, it would be possible to
783 if (!is_gimple_min_invariant (fd
.loop
.n1
)
784 || !is_gimple_min_invariant (fd
.loop
.n2
)
785 || !is_gimple_min_invariant (fd
.loop
.step
)
786 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
793 /* Collect additional arguments needed to emit a combined
794 parallel+workshare call. WS_STMT is the workshare directive being
797 static vec
<tree
, va_gc
> *
798 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
801 location_t loc
= gimple_location (ws_stmt
);
802 vec
<tree
, va_gc
> *ws_args
;
804 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
806 struct omp_for_data fd
;
809 extract_omp_for_data (for_stmt
, &fd
, NULL
);
813 if (gimple_omp_for_combined_into_p (for_stmt
))
816 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
817 OMP_CLAUSE__LOOPTEMP_
);
819 n1
= OMP_CLAUSE_DECL (innerc
);
820 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
821 OMP_CLAUSE__LOOPTEMP_
);
823 n2
= OMP_CLAUSE_DECL (innerc
);
826 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
828 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
829 ws_args
->quick_push (t
);
831 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
832 ws_args
->quick_push (t
);
834 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
835 ws_args
->quick_push (t
);
839 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
840 ws_args
->quick_push (t
);
845 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
847 /* Number of sections is equal to the number of edges from the
848 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
849 the exit of the sections region. */
850 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
851 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
852 vec_alloc (ws_args
, 1);
853 ws_args
->quick_push (t
);
861 /* Discover whether REGION is a combined parallel+workshare region. */
864 determine_parallel_type (struct omp_region
*region
)
866 basic_block par_entry_bb
, par_exit_bb
;
867 basic_block ws_entry_bb
, ws_exit_bb
;
869 if (region
== NULL
|| region
->inner
== NULL
870 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
871 || region
->inner
->cont
== NULL
)
874 /* We only support parallel+for and parallel+sections. */
875 if (region
->type
!= GIMPLE_OMP_PARALLEL
876 || (region
->inner
->type
!= GIMPLE_OMP_FOR
877 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
880 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
881 WS_EXIT_BB -> PAR_EXIT_BB. */
882 par_entry_bb
= region
->entry
;
883 par_exit_bb
= region
->exit
;
884 ws_entry_bb
= region
->inner
->entry
;
885 ws_exit_bb
= region
->inner
->exit
;
887 if (single_succ (par_entry_bb
) == ws_entry_bb
888 && single_succ (ws_exit_bb
) == par_exit_bb
889 && workshare_safe_to_combine_p (ws_entry_bb
)
890 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
891 || (last_and_only_stmt (ws_entry_bb
)
892 && last_and_only_stmt (par_exit_bb
))))
894 gimple par_stmt
= last_stmt (par_entry_bb
);
895 gimple ws_stmt
= last_stmt (ws_entry_bb
);
897 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
899 /* If this is a combined parallel loop, we need to determine
900 whether or not to use the combined library calls. There
901 are two cases where we do not apply the transformation:
902 static loops and any kind of ordered loop. In the first
903 case, we already open code the loop so there is no need
904 to do anything else. In the latter case, the combined
905 parallel loop call would still need extra synchronization
906 to implement ordered semantics, so there would not be any
907 gain in using the combined call. */
908 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
909 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
911 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
912 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
914 region
->is_combined_parallel
= false;
915 region
->inner
->is_combined_parallel
= false;
920 region
->is_combined_parallel
= true;
921 region
->inner
->is_combined_parallel
= true;
922 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
927 /* Return true if EXPR is variable sized. */
930 is_variable_sized (const_tree expr
)
932 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
935 /* Return true if DECL is a reference type. */
938 is_reference (tree decl
)
940 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
943 /* Return the type of a decl. If the decl is reference type,
944 return its base type. */
946 get_base_type (tree decl
)
948 tree type
= TREE_TYPE (decl
);
949 if (is_reference (decl
))
950 type
= TREE_TYPE (type
);
954 /* Lookup variables. The "maybe" form
955 allows for the variable form to not have been entered, otherwise we
956 assert that the variable must have been entered. */
959 lookup_decl (tree var
, omp_context
*ctx
)
961 tree
*n
= ctx
->cb
.decl_map
->get (var
);
966 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
968 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
969 return n
? *n
: NULL_TREE
;
973 lookup_field (tree var
, omp_context
*ctx
)
976 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
977 return (tree
) n
->value
;
981 lookup_sfield (tree var
, omp_context
*ctx
)
984 n
= splay_tree_lookup (ctx
->sfield_map
985 ? ctx
->sfield_map
: ctx
->field_map
,
986 (splay_tree_key
) var
);
987 return (tree
) n
->value
;
991 maybe_lookup_field (tree var
, omp_context
*ctx
)
994 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
995 return n
? (tree
) n
->value
: NULL_TREE
;
999 lookup_oacc_reduction (const char *id
, omp_context
*ctx
)
1002 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
1003 return (tree
) n
->value
;
1007 maybe_lookup_oacc_reduction (tree var
, omp_context
*ctx
)
1009 splay_tree_node n
= NULL
;
1010 if (ctx
->reduction_map
)
1011 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
1012 return n
? (tree
) n
->value
: NULL_TREE
;
1015 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1016 the parallel context if DECL is to be shared. */
1019 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1021 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1024 /* We can only use copy-in/copy-out semantics for shared variables
1025 when we know the value is not accessible from an outer scope. */
1028 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1030 /* ??? Trivially accessible from anywhere. But why would we even
1031 be passing an address in this case? Should we simply assert
1032 this to be false, or should we have a cleanup pass that removes
1033 these from the list of mappings? */
1034 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1037 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1038 without analyzing the expression whether or not its location
1039 is accessible to anyone else. In the case of nested parallel
1040 regions it certainly may be. */
1041 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1044 /* Do not use copy-in/copy-out for variables that have their
1046 if (TREE_ADDRESSABLE (decl
))
1049 /* lower_send_shared_vars only uses copy-in, but not copy-out
1051 if (TREE_READONLY (decl
)
1052 || ((TREE_CODE (decl
) == RESULT_DECL
1053 || TREE_CODE (decl
) == PARM_DECL
)
1054 && DECL_BY_REFERENCE (decl
)))
1057 /* Disallow copy-in/out in nested parallel if
1058 decl is shared in outer parallel, otherwise
1059 each thread could store the shared variable
1060 in its own copy-in location, making the
1061 variable no longer really shared. */
1062 if (shared_ctx
->is_nested
)
1066 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1067 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1074 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1075 c
; c
= OMP_CLAUSE_CHAIN (c
))
1076 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1077 && OMP_CLAUSE_DECL (c
) == decl
)
1081 goto maybe_mark_addressable_and_ret
;
1085 /* For tasks avoid using copy-in/out. As tasks can be
1086 deferred or executed in different thread, when GOMP_task
1087 returns, the task hasn't necessarily terminated. */
1088 if (is_task_ctx (shared_ctx
))
1091 maybe_mark_addressable_and_ret
:
1092 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1093 if (is_gimple_reg (outer
))
1095 /* Taking address of OUTER in lower_send_shared_vars
1096 might need regimplification of everything that uses the
1098 if (!task_shared_vars
)
1099 task_shared_vars
= BITMAP_ALLOC (NULL
);
1100 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1101 TREE_ADDRESSABLE (outer
) = 1;
1110 /* Construct a new automatic decl similar to VAR. */
1113 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1115 tree copy
= copy_var_decl (var
, name
, type
);
1117 DECL_CONTEXT (copy
) = current_function_decl
;
1118 DECL_CHAIN (copy
) = ctx
->block_vars
;
1119 ctx
->block_vars
= copy
;
1125 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1127 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1130 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1133 omp_build_component_ref (tree obj
, tree field
)
1135 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1136 if (TREE_THIS_VOLATILE (field
))
1137 TREE_THIS_VOLATILE (ret
) |= 1;
1138 if (TREE_READONLY (field
))
1139 TREE_READONLY (ret
) |= 1;
1143 /* Build tree nodes to access the field for VAR on the receiver side. */
1146 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1148 tree x
, field
= lookup_field (var
, ctx
);
1150 /* If the receiver record type was remapped in the child function,
1151 remap the field into the new record type. */
1152 x
= maybe_lookup_field (field
, ctx
);
1156 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1157 x
= omp_build_component_ref (x
, field
);
1159 x
= build_simple_mem_ref (x
);
1164 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1165 of a parallel, this is a component reference; for workshare constructs
1166 this is some variable. */
1169 build_outer_var_ref (tree var
, omp_context
*ctx
)
1173 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1175 else if (is_variable_sized (var
))
1177 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1178 x
= build_outer_var_ref (x
, ctx
);
1179 x
= build_simple_mem_ref (x
);
1181 else if (is_taskreg_ctx (ctx
))
1183 bool by_ref
= use_pointer_for_field (var
, NULL
);
1184 x
= build_receiver_ref (var
, by_ref
, ctx
);
1186 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1187 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1189 /* #pragma omp simd isn't a worksharing construct, and can reference even
1190 private vars in its linear etc. clauses. */
1192 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1193 x
= lookup_decl (var
, ctx
->outer
);
1194 else if (ctx
->outer
)
1195 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1199 else if (ctx
->outer
)
1200 x
= lookup_decl (var
, ctx
->outer
);
1201 else if (is_reference (var
))
1202 /* This can happen with orphaned constructs. If var is reference, it is
1203 possible it is shared and as such valid. */
1208 if (is_reference (var
))
1209 x
= build_simple_mem_ref (x
);
1214 /* Build tree nodes to access the field for VAR on the sender side. */
1217 build_sender_ref (tree var
, omp_context
*ctx
)
1219 tree field
= lookup_sfield (var
, ctx
);
1220 return omp_build_component_ref (ctx
->sender_decl
, field
);
1223 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1226 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1228 tree field
, type
, sfield
= NULL_TREE
;
1230 gcc_assert ((mask
& 1) == 0
1231 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1232 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1233 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1234 gcc_assert ((mask
& 3) == 3
1235 || !is_gimple_omp_oacc (ctx
->stmt
));
1237 type
= TREE_TYPE (var
);
1240 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1241 type
= build_pointer_type (build_pointer_type (type
));
1244 type
= build_pointer_type (type
);
1245 else if ((mask
& 3) == 1 && is_reference (var
))
1246 type
= TREE_TYPE (type
);
1248 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1249 FIELD_DECL
, DECL_NAME (var
), type
);
1251 /* Remember what variable this field was created for. This does have a
1252 side effect of making dwarf2out ignore this member, so for helpful
1253 debugging we clear it later in delete_omp_context. */
1254 DECL_ABSTRACT_ORIGIN (field
) = var
;
1255 if (type
== TREE_TYPE (var
))
1257 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1258 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1259 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1262 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1264 if ((mask
& 3) == 3)
1266 insert_field_into_struct (ctx
->record_type
, field
);
1267 if (ctx
->srecord_type
)
1269 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1270 FIELD_DECL
, DECL_NAME (var
), type
);
1271 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1272 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1273 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1274 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1275 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1280 if (ctx
->srecord_type
== NULL_TREE
)
1284 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1285 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1286 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1288 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1289 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1290 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1291 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1292 splay_tree_insert (ctx
->sfield_map
,
1293 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1294 (splay_tree_value
) sfield
);
1298 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1299 : ctx
->srecord_type
, field
);
1303 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1304 (splay_tree_value
) field
);
1305 if ((mask
& 2) && ctx
->sfield_map
)
1306 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1307 (splay_tree_value
) sfield
);
1311 install_var_local (tree var
, omp_context
*ctx
)
1313 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1314 insert_decl_map (&ctx
->cb
, var
, new_var
);
1318 /* Adjust the replacement for DECL in CTX for the new context. This means
1319 copying the DECL_VALUE_EXPR, and fixing up the type. */
1322 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1324 tree new_decl
, size
;
1326 new_decl
= lookup_decl (decl
, ctx
);
1328 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1330 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1331 && DECL_HAS_VALUE_EXPR_P (decl
))
1333 tree ve
= DECL_VALUE_EXPR (decl
);
1334 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1335 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1336 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1339 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1341 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1342 if (size
== error_mark_node
)
1343 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1344 DECL_SIZE (new_decl
) = size
;
1346 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1347 if (size
== error_mark_node
)
1348 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1349 DECL_SIZE_UNIT (new_decl
) = size
;
1353 /* The callback for remap_decl. Search all containing contexts for a
1354 mapping of the variable; this avoids having to duplicate the splay
1355 tree ahead of time. We know a mapping doesn't already exist in the
1356 given context. Create new mappings to implement default semantics. */
1359 omp_copy_decl (tree var
, copy_body_data
*cb
)
1361 omp_context
*ctx
= (omp_context
*) cb
;
1364 if (TREE_CODE (var
) == LABEL_DECL
)
1366 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1367 DECL_CONTEXT (new_var
) = current_function_decl
;
1368 insert_decl_map (&ctx
->cb
, var
, new_var
);
1372 while (!is_taskreg_ctx (ctx
))
1377 new_var
= maybe_lookup_decl (var
, ctx
);
1382 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1385 return error_mark_node
;
1389 /* Debugging dumps for parallel regions. */
1390 void dump_omp_region (FILE *, struct omp_region
*, int);
1391 void debug_omp_region (struct omp_region
*);
1392 void debug_all_omp_regions (void);
1394 /* Dump the parallel region tree rooted at REGION. */
1397 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1399 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1400 gimple_code_name
[region
->type
]);
1403 dump_omp_region (file
, region
->inner
, indent
+ 4);
1407 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1408 region
->cont
->index
);
1412 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1413 region
->exit
->index
);
1415 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1418 dump_omp_region (file
, region
->next
, indent
);
1422 debug_omp_region (struct omp_region
*region
)
1424 dump_omp_region (stderr
, region
, 0);
1428 debug_all_omp_regions (void)
1430 dump_omp_region (stderr
, root_omp_region
, 0);
1434 /* Create a new parallel region starting at STMT inside region PARENT. */
1436 static struct omp_region
*
1437 new_omp_region (basic_block bb
, enum gimple_code type
,
1438 struct omp_region
*parent
)
1440 struct omp_region
*region
= XCNEW (struct omp_region
);
1442 region
->outer
= parent
;
1444 region
->type
= type
;
1448 /* This is a nested region. Add it to the list of inner
1449 regions in PARENT. */
1450 region
->next
= parent
->inner
;
1451 parent
->inner
= region
;
1455 /* This is a toplevel region. Add it to the list of toplevel
1456 regions in ROOT_OMP_REGION. */
1457 region
->next
= root_omp_region
;
1458 root_omp_region
= region
;
1464 /* Release the memory associated with the region tree rooted at REGION. */
1467 free_omp_region_1 (struct omp_region
*region
)
1469 struct omp_region
*i
, *n
;
1471 for (i
= region
->inner
; i
; i
= n
)
1474 free_omp_region_1 (i
);
1480 /* Release the memory for the entire omp region tree. */
1483 free_omp_regions (void)
1485 struct omp_region
*r
, *n
;
1486 for (r
= root_omp_region
; r
; r
= n
)
1489 free_omp_region_1 (r
);
1491 root_omp_region
= NULL
;
1495 /* Create a new context, with OUTER_CTX being the surrounding context. */
1497 static omp_context
*
1498 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1500 omp_context
*ctx
= XCNEW (omp_context
);
1502 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1503 (splay_tree_value
) ctx
);
1508 ctx
->outer
= outer_ctx
;
1509 ctx
->cb
= outer_ctx
->cb
;
1510 ctx
->cb
.block
= NULL
;
1511 ctx
->depth
= outer_ctx
->depth
+ 1;
1512 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1516 ctx
->cb
.src_fn
= current_function_decl
;
1517 ctx
->cb
.dst_fn
= current_function_decl
;
1518 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1519 gcc_checking_assert (ctx
->cb
.src_node
);
1520 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1521 ctx
->cb
.src_cfun
= cfun
;
1522 ctx
->cb
.copy_decl
= omp_copy_decl
;
1523 ctx
->cb
.eh_lp_nr
= 0;
1524 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1528 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1533 static gimple_seq
maybe_catch_exception (gimple_seq
);
1535 /* Finalize task copyfn. */
1538 finalize_task_copyfn (gomp_task
*task_stmt
)
1540 struct function
*child_cfun
;
1542 gimple_seq seq
= NULL
, new_seq
;
1545 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1546 if (child_fn
== NULL_TREE
)
1549 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1550 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1552 push_cfun (child_cfun
);
1553 bind
= gimplify_body (child_fn
, false);
1554 gimple_seq_add_stmt (&seq
, bind
);
1555 new_seq
= maybe_catch_exception (seq
);
1558 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1560 gimple_seq_add_stmt (&seq
, bind
);
1562 gimple_set_body (child_fn
, seq
);
1565 /* Inform the callgraph about the new function. */
1566 cgraph_node::add_new_function (child_fn
, false);
1569 /* Destroy a omp_context data structures. Called through the splay tree
1570 value delete callback. */
1573 delete_omp_context (splay_tree_value value
)
1575 omp_context
*ctx
= (omp_context
*) value
;
1577 delete ctx
->cb
.decl_map
;
1580 splay_tree_delete (ctx
->field_map
);
1581 if (ctx
->sfield_map
)
1582 splay_tree_delete (ctx
->sfield_map
);
1583 if (ctx
->reduction_map
1584 /* Shared over several omp_contexts. */
1585 && (ctx
->outer
== NULL
1586 || ctx
->reduction_map
!= ctx
->outer
->reduction_map
))
1587 splay_tree_delete (ctx
->reduction_map
);
1589 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1590 it produces corrupt debug information. */
1591 if (ctx
->record_type
)
1594 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1595 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1597 if (ctx
->srecord_type
)
1600 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1601 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1604 if (is_task_ctx (ctx
))
1605 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1610 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1614 fixup_child_record_type (omp_context
*ctx
)
1616 tree f
, type
= ctx
->record_type
;
1618 /* ??? It isn't sufficient to just call remap_type here, because
1619 variably_modified_type_p doesn't work the way we expect for
1620 record types. Testing each field for whether it needs remapping
1621 and creating a new record by hand works, however. */
1622 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1623 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1627 tree name
, new_fields
= NULL
;
1629 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1630 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1631 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1632 TYPE_DECL
, name
, type
);
1633 TYPE_NAME (type
) = name
;
1635 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1637 tree new_f
= copy_node (f
);
1638 DECL_CONTEXT (new_f
) = type
;
1639 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1640 DECL_CHAIN (new_f
) = new_fields
;
1641 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1642 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1644 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1648 /* Arrange to be able to look up the receiver field
1649 given the sender field. */
1650 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1651 (splay_tree_value
) new_f
);
1653 TYPE_FIELDS (type
) = nreverse (new_fields
);
1657 TREE_TYPE (ctx
->receiver_decl
)
1658 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1661 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1662 specified by CLAUSES. */
1665 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1668 bool scan_array_reductions
= false;
1670 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1674 switch (OMP_CLAUSE_CODE (c
))
1676 case OMP_CLAUSE_PRIVATE
:
1677 decl
= OMP_CLAUSE_DECL (c
);
1678 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1680 else if (!is_variable_sized (decl
))
1681 install_var_local (decl
, ctx
);
1684 case OMP_CLAUSE_SHARED
:
1685 decl
= OMP_CLAUSE_DECL (c
);
1686 /* Ignore shared directives in teams construct. */
1687 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1689 /* Global variables don't need to be copied,
1690 the receiver side will use them directly. */
1691 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1692 if (is_global_var (odecl
))
1694 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1697 gcc_assert (is_taskreg_ctx (ctx
));
1698 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1699 || !is_variable_sized (decl
));
1700 /* Global variables don't need to be copied,
1701 the receiver side will use them directly. */
1702 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1704 by_ref
= use_pointer_for_field (decl
, ctx
);
1705 if (! TREE_READONLY (decl
)
1706 || TREE_ADDRESSABLE (decl
)
1708 || is_reference (decl
))
1710 install_var_field (decl
, by_ref
, 3, ctx
);
1711 install_var_local (decl
, ctx
);
1714 /* We don't need to copy const scalar vars back. */
1715 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1718 case OMP_CLAUSE_LASTPRIVATE
:
1719 /* Let the corresponding firstprivate clause create
1721 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1725 case OMP_CLAUSE_FIRSTPRIVATE
:
1726 if (is_gimple_omp_oacc (ctx
->stmt
))
1728 sorry ("clause not supported yet");
1732 case OMP_CLAUSE_REDUCTION
:
1733 case OMP_CLAUSE_LINEAR
:
1734 decl
= OMP_CLAUSE_DECL (c
);
1736 if (is_variable_sized (decl
))
1738 if (is_task_ctx (ctx
))
1739 install_var_field (decl
, false, 1, ctx
);
1742 else if (is_taskreg_ctx (ctx
))
1745 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1746 by_ref
= use_pointer_for_field (decl
, NULL
);
1748 if (is_task_ctx (ctx
)
1749 && (global
|| by_ref
|| is_reference (decl
)))
1751 install_var_field (decl
, false, 1, ctx
);
1753 install_var_field (decl
, by_ref
, 2, ctx
);
1756 install_var_field (decl
, by_ref
, 3, ctx
);
1758 install_var_local (decl
, ctx
);
1759 if (is_gimple_omp_oacc (ctx
->stmt
)
1760 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
1762 /* Create a decl for the reduction array. */
1763 tree var
= OMP_CLAUSE_DECL (c
);
1764 tree type
= get_base_type (var
);
1765 tree ptype
= build_pointer_type (type
);
1766 tree array
= create_tmp_var (ptype
,
1767 oacc_get_reduction_array_id (var
));
1768 omp_context
*c
= (ctx
->field_map
? ctx
: ctx
->outer
);
1769 install_var_field (array
, true, 3, c
);
1770 install_var_local (array
, c
);
1772 /* Insert it into the current context. */
1773 splay_tree_insert (ctx
->reduction_map
, (splay_tree_key
)
1774 oacc_get_reduction_array_id (var
),
1775 (splay_tree_value
) array
);
1776 splay_tree_insert (ctx
->reduction_map
,
1777 (splay_tree_key
) array
,
1778 (splay_tree_value
) array
);
1782 case OMP_CLAUSE__LOOPTEMP_
:
1783 gcc_assert (is_parallel_ctx (ctx
));
1784 decl
= OMP_CLAUSE_DECL (c
);
1785 install_var_field (decl
, false, 3, ctx
);
1786 install_var_local (decl
, ctx
);
1789 case OMP_CLAUSE_COPYPRIVATE
:
1790 case OMP_CLAUSE_COPYIN
:
1791 decl
= OMP_CLAUSE_DECL (c
);
1792 by_ref
= use_pointer_for_field (decl
, NULL
);
1793 install_var_field (decl
, by_ref
, 3, ctx
);
1796 case OMP_CLAUSE_DEFAULT
:
1797 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1800 case OMP_CLAUSE_FINAL
:
1802 case OMP_CLAUSE_NUM_THREADS
:
1803 case OMP_CLAUSE_NUM_TEAMS
:
1804 case OMP_CLAUSE_THREAD_LIMIT
:
1805 case OMP_CLAUSE_DEVICE
:
1806 case OMP_CLAUSE_SCHEDULE
:
1807 case OMP_CLAUSE_DIST_SCHEDULE
:
1808 case OMP_CLAUSE_DEPEND
:
1809 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1810 case OMP_CLAUSE_NUM_GANGS
:
1811 case OMP_CLAUSE_NUM_WORKERS
:
1812 case OMP_CLAUSE_VECTOR_LENGTH
:
1814 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1818 case OMP_CLAUSE_FROM
:
1819 case OMP_CLAUSE_MAP
:
1821 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1822 decl
= OMP_CLAUSE_DECL (c
);
1823 /* Global variables with "omp declare target" attribute
1824 don't need to be copied, the receiver side will use them
1826 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1828 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1829 && varpool_node::get_create (decl
)->offloadable
)
1831 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1832 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
1834 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1835 not offloaded; there is nothing to map for those. */
1836 if (!is_gimple_omp_offloaded (ctx
->stmt
)
1837 && !POINTER_TYPE_P (TREE_TYPE (decl
))
1838 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
1843 if (DECL_SIZE (decl
)
1844 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1846 tree decl2
= DECL_VALUE_EXPR (decl
);
1847 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1848 decl2
= TREE_OPERAND (decl2
, 0);
1849 gcc_assert (DECL_P (decl2
));
1850 install_var_field (decl2
, true, 3, ctx
);
1851 install_var_local (decl2
, ctx
);
1852 install_var_local (decl
, ctx
);
1856 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1857 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1858 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1859 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1860 install_var_field (decl
, true, 7, ctx
);
1862 install_var_field (decl
, true, 3, ctx
);
1863 if (is_gimple_omp_offloaded (ctx
->stmt
))
1864 install_var_local (decl
, ctx
);
1869 tree base
= get_base_address (decl
);
1870 tree nc
= OMP_CLAUSE_CHAIN (c
);
1873 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1874 && OMP_CLAUSE_DECL (nc
) == base
1875 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
1876 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1878 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1879 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1885 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1886 decl
= OMP_CLAUSE_DECL (c
);
1888 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1889 (splay_tree_key
) decl
));
1891 = build_decl (OMP_CLAUSE_LOCATION (c
),
1892 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1893 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1894 insert_field_into_struct (ctx
->record_type
, field
);
1895 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1896 (splay_tree_value
) field
);
1901 case OMP_CLAUSE_NOWAIT
:
1902 case OMP_CLAUSE_ORDERED
:
1903 case OMP_CLAUSE_COLLAPSE
:
1904 case OMP_CLAUSE_UNTIED
:
1905 case OMP_CLAUSE_MERGEABLE
:
1906 case OMP_CLAUSE_PROC_BIND
:
1907 case OMP_CLAUSE_SAFELEN
:
1908 case OMP_CLAUSE_ASYNC
:
1909 case OMP_CLAUSE_WAIT
:
1910 case OMP_CLAUSE_GANG
:
1911 case OMP_CLAUSE_WORKER
:
1912 case OMP_CLAUSE_VECTOR
:
1915 case OMP_CLAUSE_ALIGNED
:
1916 decl
= OMP_CLAUSE_DECL (c
);
1917 if (is_global_var (decl
)
1918 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1919 install_var_local (decl
, ctx
);
1922 case OMP_CLAUSE_DEVICE_RESIDENT
:
1923 case OMP_CLAUSE_USE_DEVICE
:
1924 case OMP_CLAUSE__CACHE_
:
1925 case OMP_CLAUSE_INDEPENDENT
:
1926 case OMP_CLAUSE_AUTO
:
1927 case OMP_CLAUSE_SEQ
:
1928 sorry ("Clause not supported yet");
1936 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1938 switch (OMP_CLAUSE_CODE (c
))
1940 case OMP_CLAUSE_LASTPRIVATE
:
1941 /* Let the corresponding firstprivate clause create
1943 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1944 scan_array_reductions
= true;
1945 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1949 case OMP_CLAUSE_FIRSTPRIVATE
:
1950 if (is_gimple_omp_oacc (ctx
->stmt
))
1952 sorry ("clause not supported yet");
1956 case OMP_CLAUSE_PRIVATE
:
1957 case OMP_CLAUSE_REDUCTION
:
1958 case OMP_CLAUSE_LINEAR
:
1959 decl
= OMP_CLAUSE_DECL (c
);
1960 if (is_variable_sized (decl
))
1961 install_var_local (decl
, ctx
);
1962 fixup_remapped_decl (decl
, ctx
,
1963 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1964 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1965 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1966 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1967 scan_array_reductions
= true;
1968 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1969 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1970 scan_array_reductions
= true;
1973 case OMP_CLAUSE_SHARED
:
1974 /* Ignore shared directives in teams construct. */
1975 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1977 decl
= OMP_CLAUSE_DECL (c
);
1978 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1979 fixup_remapped_decl (decl
, ctx
, false);
1982 case OMP_CLAUSE_MAP
:
1983 if (!is_gimple_omp_offloaded (ctx
->stmt
))
1985 decl
= OMP_CLAUSE_DECL (c
);
1987 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1988 && varpool_node::get_create (decl
)->offloadable
)
1992 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1993 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1994 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1996 tree new_decl
= lookup_decl (decl
, ctx
);
1997 TREE_TYPE (new_decl
)
1998 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2000 else if (DECL_SIZE (decl
)
2001 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2003 tree decl2
= DECL_VALUE_EXPR (decl
);
2004 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2005 decl2
= TREE_OPERAND (decl2
, 0);
2006 gcc_assert (DECL_P (decl2
));
2007 fixup_remapped_decl (decl2
, ctx
, false);
2008 fixup_remapped_decl (decl
, ctx
, true);
2011 fixup_remapped_decl (decl
, ctx
, false);
2015 case OMP_CLAUSE_COPYPRIVATE
:
2016 case OMP_CLAUSE_COPYIN
:
2017 case OMP_CLAUSE_DEFAULT
:
2019 case OMP_CLAUSE_NUM_THREADS
:
2020 case OMP_CLAUSE_NUM_TEAMS
:
2021 case OMP_CLAUSE_THREAD_LIMIT
:
2022 case OMP_CLAUSE_DEVICE
:
2023 case OMP_CLAUSE_SCHEDULE
:
2024 case OMP_CLAUSE_DIST_SCHEDULE
:
2025 case OMP_CLAUSE_NOWAIT
:
2026 case OMP_CLAUSE_ORDERED
:
2027 case OMP_CLAUSE_COLLAPSE
:
2028 case OMP_CLAUSE_UNTIED
:
2029 case OMP_CLAUSE_FINAL
:
2030 case OMP_CLAUSE_MERGEABLE
:
2031 case OMP_CLAUSE_PROC_BIND
:
2032 case OMP_CLAUSE_SAFELEN
:
2033 case OMP_CLAUSE_ALIGNED
:
2034 case OMP_CLAUSE_DEPEND
:
2035 case OMP_CLAUSE__LOOPTEMP_
:
2037 case OMP_CLAUSE_FROM
:
2038 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2039 case OMP_CLAUSE_ASYNC
:
2040 case OMP_CLAUSE_WAIT
:
2041 case OMP_CLAUSE_NUM_GANGS
:
2042 case OMP_CLAUSE_NUM_WORKERS
:
2043 case OMP_CLAUSE_VECTOR_LENGTH
:
2044 case OMP_CLAUSE_GANG
:
2045 case OMP_CLAUSE_WORKER
:
2046 case OMP_CLAUSE_VECTOR
:
2049 case OMP_CLAUSE_DEVICE_RESIDENT
:
2050 case OMP_CLAUSE_USE_DEVICE
:
2051 case OMP_CLAUSE__CACHE_
:
2052 case OMP_CLAUSE_INDEPENDENT
:
2053 case OMP_CLAUSE_AUTO
:
2054 case OMP_CLAUSE_SEQ
:
2055 sorry ("Clause not supported yet");
2063 gcc_checking_assert (!scan_array_reductions
2064 || !is_gimple_omp_oacc (ctx
->stmt
));
2065 if (scan_array_reductions
)
2066 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2067 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2068 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2070 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2071 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2073 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2074 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2075 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2076 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2077 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2078 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2081 /* Create a new name for omp child function. Returns an identifier. If
2082 IS_CILK_FOR is true then the suffix for the child function is
2086 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2089 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2090 return clone_function_name (current_function_decl
,
2091 task_copy
? "_omp_cpyfn" : "_omp_fn");
2094 /* Returns the type of the induction variable for the child function for
2095 _Cilk_for and the types for _high and _low variables based on TYPE. */
2098 cilk_for_check_loop_diff_type (tree type
)
2100 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2102 if (TYPE_UNSIGNED (type
))
2103 return uint32_type_node
;
2105 return integer_type_node
;
2109 if (TYPE_UNSIGNED (type
))
2110 return uint64_type_node
;
2112 return long_long_integer_type_node
;
2116 /* Build a decl for the omp child function. It'll not contain a body
2117 yet, just the bare decl. */
2120 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2122 tree decl
, type
, name
, t
;
2125 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2126 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2127 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2128 tree cilk_var_type
= NULL_TREE
;
2130 name
= create_omp_child_function_name (task_copy
,
2131 cilk_for_count
!= NULL_TREE
);
2133 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2134 ptr_type_node
, NULL_TREE
);
2135 else if (cilk_for_count
)
2137 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2138 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2139 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2140 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2143 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2145 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2147 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2150 ctx
->cb
.dst_fn
= decl
;
2152 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2154 TREE_STATIC (decl
) = 1;
2155 TREE_USED (decl
) = 1;
2156 DECL_ARTIFICIAL (decl
) = 1;
2157 DECL_IGNORED_P (decl
) = 0;
2158 TREE_PUBLIC (decl
) = 0;
2159 DECL_UNINLINABLE (decl
) = 1;
2160 DECL_EXTERNAL (decl
) = 0;
2161 DECL_CONTEXT (decl
) = NULL_TREE
;
2162 DECL_INITIAL (decl
) = make_node (BLOCK
);
2163 if (cgraph_node::get (current_function_decl
)->offloadable
)
2164 cgraph_node::get_create (decl
)->offloadable
= 1;
2168 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2169 if (is_gimple_omp_offloaded (octx
->stmt
))
2171 cgraph_node::get_create (decl
)->offloadable
= 1;
2172 #ifdef ENABLE_OFFLOADING
2173 g
->have_offload
= true;
2179 if (cgraph_node::get_create (decl
)->offloadable
2180 && !lookup_attribute ("omp declare target",
2181 DECL_ATTRIBUTES (current_function_decl
)))
2182 DECL_ATTRIBUTES (decl
)
2183 = tree_cons (get_identifier ("omp target entrypoint"),
2184 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2186 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2187 RESULT_DECL
, NULL_TREE
, void_type_node
);
2188 DECL_ARTIFICIAL (t
) = 1;
2189 DECL_IGNORED_P (t
) = 1;
2190 DECL_CONTEXT (t
) = decl
;
2191 DECL_RESULT (decl
) = t
;
2193 /* _Cilk_for's child function requires two extra parameters called
2194 __low and __high that are set the by Cilk runtime when it calls this
2198 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2199 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2200 DECL_ARTIFICIAL (t
) = 1;
2201 DECL_NAMELESS (t
) = 1;
2202 DECL_ARG_TYPE (t
) = ptr_type_node
;
2203 DECL_CONTEXT (t
) = current_function_decl
;
2205 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2206 DECL_ARGUMENTS (decl
) = t
;
2208 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2209 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2210 DECL_ARTIFICIAL (t
) = 1;
2211 DECL_NAMELESS (t
) = 1;
2212 DECL_ARG_TYPE (t
) = ptr_type_node
;
2213 DECL_CONTEXT (t
) = current_function_decl
;
2215 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2216 DECL_ARGUMENTS (decl
) = t
;
2219 tree data_name
= get_identifier (".omp_data_i");
2220 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2222 DECL_ARTIFICIAL (t
) = 1;
2223 DECL_NAMELESS (t
) = 1;
2224 DECL_ARG_TYPE (t
) = ptr_type_node
;
2225 DECL_CONTEXT (t
) = current_function_decl
;
2228 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2229 DECL_ARGUMENTS (decl
) = t
;
2231 ctx
->receiver_decl
= t
;
2234 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2235 PARM_DECL
, get_identifier (".omp_data_o"),
2237 DECL_ARTIFICIAL (t
) = 1;
2238 DECL_NAMELESS (t
) = 1;
2239 DECL_ARG_TYPE (t
) = ptr_type_node
;
2240 DECL_CONTEXT (t
) = current_function_decl
;
2242 TREE_ADDRESSABLE (t
) = 1;
2243 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2244 DECL_ARGUMENTS (decl
) = t
;
2247 /* Allocate memory for the function structure. The call to
2248 allocate_struct_function clobbers CFUN, so we need to restore
2250 push_struct_function (decl
);
2251 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2255 /* Callback for walk_gimple_seq. Check if combined parallel
2256 contains gimple_omp_for_combined_into_p OMP_FOR. */
2259 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2260 bool *handled_ops_p
,
2261 struct walk_stmt_info
*wi
)
2263 gimple stmt
= gsi_stmt (*gsi_p
);
2265 *handled_ops_p
= true;
2266 switch (gimple_code (stmt
))
2270 case GIMPLE_OMP_FOR
:
2271 if (gimple_omp_for_combined_into_p (stmt
)
2272 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2275 return integer_zero_node
;
2284 /* Scan an OpenMP parallel directive. */
2287 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2291 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2293 /* Ignore parallel directives with empty bodies, unless there
2294 are copyin clauses. */
2296 && empty_body_p (gimple_omp_body (stmt
))
2297 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2298 OMP_CLAUSE_COPYIN
) == NULL
)
2300 gsi_replace (gsi
, gimple_build_nop (), false);
2304 if (gimple_omp_parallel_combined_p (stmt
))
2306 struct walk_stmt_info wi
;
2308 memset (&wi
, 0, sizeof (wi
));
2310 walk_gimple_seq (gimple_omp_body (stmt
),
2311 find_combined_for
, NULL
, &wi
);
2314 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
) wi
.info
);
2315 struct omp_for_data fd
;
2316 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2317 /* We need two temporaries with fd.loop.v type (istart/iend)
2318 and then (fd.collapse - 1) temporaries with the same
2319 type for count2 ... countN-1 vars if not constant. */
2320 size_t count
= 2, i
;
2321 tree type
= fd
.iter_type
;
2323 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2324 count
+= fd
.collapse
- 1;
2325 for (i
= 0; i
< count
; i
++)
2327 tree temp
= create_tmp_var (type
);
2328 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2329 OMP_CLAUSE__LOOPTEMP_
);
2330 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2331 OMP_CLAUSE_DECL (c
) = temp
;
2332 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2333 gimple_omp_parallel_set_clauses (stmt
, c
);
2338 ctx
= new_omp_context (stmt
, outer_ctx
);
2339 taskreg_contexts
.safe_push (ctx
);
2340 if (taskreg_nesting_level
> 1)
2341 ctx
->is_nested
= true;
2342 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2343 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2344 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2345 name
= create_tmp_var_name (".omp_data_s");
2346 name
= build_decl (gimple_location (stmt
),
2347 TYPE_DECL
, name
, ctx
->record_type
);
2348 DECL_ARTIFICIAL (name
) = 1;
2349 DECL_NAMELESS (name
) = 1;
2350 TYPE_NAME (ctx
->record_type
) = name
;
2351 create_omp_child_function (ctx
, false);
2352 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2354 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2355 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2357 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2358 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2361 /* Scan an OpenMP task directive. */
2364 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2368 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2370 /* Ignore task directives with empty bodies. */
2372 && empty_body_p (gimple_omp_body (stmt
)))
2374 gsi_replace (gsi
, gimple_build_nop (), false);
2378 ctx
= new_omp_context (stmt
, outer_ctx
);
2379 taskreg_contexts
.safe_push (ctx
);
2380 if (taskreg_nesting_level
> 1)
2381 ctx
->is_nested
= true;
2382 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2383 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2384 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2385 name
= create_tmp_var_name (".omp_data_s");
2386 name
= build_decl (gimple_location (stmt
),
2387 TYPE_DECL
, name
, ctx
->record_type
);
2388 DECL_ARTIFICIAL (name
) = 1;
2389 DECL_NAMELESS (name
) = 1;
2390 TYPE_NAME (ctx
->record_type
) = name
;
2391 create_omp_child_function (ctx
, false);
2392 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2394 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2396 if (ctx
->srecord_type
)
2398 name
= create_tmp_var_name (".omp_data_a");
2399 name
= build_decl (gimple_location (stmt
),
2400 TYPE_DECL
, name
, ctx
->srecord_type
);
2401 DECL_ARTIFICIAL (name
) = 1;
2402 DECL_NAMELESS (name
) = 1;
2403 TYPE_NAME (ctx
->srecord_type
) = name
;
2404 create_omp_child_function (ctx
, true);
2407 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2409 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2411 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2412 t
= build_int_cst (long_integer_type_node
, 0);
2413 gimple_omp_task_set_arg_size (stmt
, t
);
2414 t
= build_int_cst (long_integer_type_node
, 1);
2415 gimple_omp_task_set_arg_align (stmt
, t
);
2420 /* If any decls have been made addressable during scan_omp,
2421 adjust their fields if needed, and layout record types
2422 of parallel/task constructs. */
2425 finish_taskreg_scan (omp_context
*ctx
)
2427 if (ctx
->record_type
== NULL_TREE
)
2430 /* If any task_shared_vars were needed, verify all
2431 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2432 statements if use_pointer_for_field hasn't changed
2433 because of that. If it did, update field types now. */
2434 if (task_shared_vars
)
2438 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2439 c
; c
= OMP_CLAUSE_CHAIN (c
))
2440 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2442 tree decl
= OMP_CLAUSE_DECL (c
);
2444 /* Global variables don't need to be copied,
2445 the receiver side will use them directly. */
2446 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2448 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2449 || !use_pointer_for_field (decl
, ctx
))
2451 tree field
= lookup_field (decl
, ctx
);
2452 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2453 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2455 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2456 TREE_THIS_VOLATILE (field
) = 0;
2457 DECL_USER_ALIGN (field
) = 0;
2458 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2459 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2460 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2461 if (ctx
->srecord_type
)
2463 tree sfield
= lookup_sfield (decl
, ctx
);
2464 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2465 TREE_THIS_VOLATILE (sfield
) = 0;
2466 DECL_USER_ALIGN (sfield
) = 0;
2467 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2468 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2469 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2474 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2476 layout_type (ctx
->record_type
);
2477 fixup_child_record_type (ctx
);
2481 location_t loc
= gimple_location (ctx
->stmt
);
2482 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2483 /* Move VLA fields to the end. */
2484 p
= &TYPE_FIELDS (ctx
->record_type
);
2486 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2487 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2490 *p
= TREE_CHAIN (*p
);
2491 TREE_CHAIN (*q
) = NULL_TREE
;
2492 q
= &TREE_CHAIN (*q
);
2495 p
= &DECL_CHAIN (*p
);
2497 layout_type (ctx
->record_type
);
2498 fixup_child_record_type (ctx
);
2499 if (ctx
->srecord_type
)
2500 layout_type (ctx
->srecord_type
);
2501 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2502 TYPE_SIZE_UNIT (ctx
->record_type
));
2503 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2504 t
= build_int_cst (long_integer_type_node
,
2505 TYPE_ALIGN_UNIT (ctx
->record_type
));
2506 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2511 static omp_context
*
2512 enclosing_target_ctx (omp_context
*ctx
)
2515 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2517 gcc_assert (ctx
!= NULL
);
2522 oacc_loop_or_target_p (gimple stmt
)
2524 enum gimple_code outer_type
= gimple_code (stmt
);
2525 return ((outer_type
== GIMPLE_OMP_TARGET
2526 && ((gimple_omp_target_kind (stmt
)
2527 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2528 || (gimple_omp_target_kind (stmt
)
2529 == GF_OMP_TARGET_KIND_OACC_KERNELS
)))
2530 || (outer_type
== GIMPLE_OMP_FOR
2531 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
));
2534 /* Scan a GIMPLE_OMP_FOR. */
2537 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2539 enum gimple_code outer_type
= GIMPLE_ERROR_MARK
;
2542 tree clauses
= gimple_omp_for_clauses (stmt
);
2545 outer_type
= gimple_code (outer_ctx
->stmt
);
2547 ctx
= new_omp_context (stmt
, outer_ctx
);
2549 if (is_gimple_omp_oacc (stmt
))
2551 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2552 ctx
->gwv_this
= outer_ctx
->gwv_this
;
2553 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2556 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_GANG
)
2558 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WORKER
)
2560 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR
)
2564 ctx
->gwv_this
|= val
;
2567 /* Skip; not nested inside a region. */
2570 if (!oacc_loop_or_target_p (outer_ctx
->stmt
))
2572 /* Skip; not nested inside an OpenACC region. */
2575 if (outer_type
== GIMPLE_OMP_FOR
)
2576 outer_ctx
->gwv_below
|= val
;
2577 if (OMP_CLAUSE_OPERAND (c
, 0) != NULL_TREE
)
2579 omp_context
*enclosing
= enclosing_target_ctx (outer_ctx
);
2580 if (gimple_omp_target_kind (enclosing
->stmt
)
2581 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2582 error_at (gimple_location (stmt
),
2583 "no arguments allowed to gang, worker and vector clauses inside parallel");
2588 scan_sharing_clauses (clauses
, ctx
);
2590 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2591 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2593 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2594 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2595 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2596 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2598 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2600 if (is_gimple_omp_oacc (stmt
))
2602 if (ctx
->gwv_this
& ctx
->gwv_below
)
2603 error_at (gimple_location (stmt
),
2604 "gang, worker and vector may occur only once in a loop nest");
2605 else if (ctx
->gwv_below
!= 0
2606 && ctx
->gwv_this
> ctx
->gwv_below
)
2607 error_at (gimple_location (stmt
),
2608 "gang, worker and vector must occur in this order in a loop nest");
2609 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2610 outer_ctx
->gwv_below
|= ctx
->gwv_below
;
2614 /* Scan an OpenMP sections directive. */
2617 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
2621 ctx
= new_omp_context (stmt
, outer_ctx
);
2622 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2623 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2626 /* Scan an OpenMP single directive. */
2629 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
2634 ctx
= new_omp_context (stmt
, outer_ctx
);
2635 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2636 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2637 name
= create_tmp_var_name (".omp_copy_s");
2638 name
= build_decl (gimple_location (stmt
),
2639 TYPE_DECL
, name
, ctx
->record_type
);
2640 TYPE_NAME (ctx
->record_type
) = name
;
2642 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2643 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2645 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2646 ctx
->record_type
= NULL
;
2648 layout_type (ctx
->record_type
);
2651 /* Scan a GIMPLE_OMP_TARGET. */
2654 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
2658 bool offloaded
= is_gimple_omp_offloaded (stmt
);
2659 tree clauses
= gimple_omp_target_clauses (stmt
);
2661 ctx
= new_omp_context (stmt
, outer_ctx
);
2662 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2663 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2664 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2665 name
= create_tmp_var_name (".omp_data_t");
2666 name
= build_decl (gimple_location (stmt
),
2667 TYPE_DECL
, name
, ctx
->record_type
);
2668 DECL_ARTIFICIAL (name
) = 1;
2669 DECL_NAMELESS (name
) = 1;
2670 TYPE_NAME (ctx
->record_type
) = name
;
2673 if (is_gimple_omp_oacc (stmt
))
2674 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
2677 create_omp_child_function (ctx
, false);
2678 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2681 if (is_gimple_omp_oacc (stmt
))
2683 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2685 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_GANGS
)
2686 ctx
->gwv_this
|= MASK_GANG
;
2687 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_WORKERS
)
2688 ctx
->gwv_this
|= MASK_WORKER
;
2689 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR_LENGTH
)
2690 ctx
->gwv_this
|= MASK_VECTOR
;
2694 scan_sharing_clauses (clauses
, ctx
);
2695 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2697 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2698 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2701 TYPE_FIELDS (ctx
->record_type
)
2702 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2703 #ifdef ENABLE_CHECKING
2705 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2706 for (field
= TYPE_FIELDS (ctx
->record_type
);
2708 field
= DECL_CHAIN (field
))
2709 gcc_assert (DECL_ALIGN (field
) == align
);
2711 layout_type (ctx
->record_type
);
2713 fixup_child_record_type (ctx
);
2717 /* Scan an OpenMP teams directive. */
2720 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
2722 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2723 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2724 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2727 /* Check nesting restrictions. */
2729 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2731 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2732 inside an OpenACC CTX. */
2733 if (!(is_gimple_omp (stmt
)
2734 && is_gimple_omp_oacc (stmt
)))
2736 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
2737 if (is_gimple_omp (ctx_
->stmt
)
2738 && is_gimple_omp_oacc (ctx_
->stmt
))
2740 error_at (gimple_location (stmt
),
2741 "non-OpenACC construct inside of OpenACC region");
2748 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2749 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2751 error_at (gimple_location (stmt
),
2752 "OpenMP constructs may not be nested inside simd region");
2755 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2757 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2758 || (gimple_omp_for_kind (stmt
)
2759 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2760 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2762 error_at (gimple_location (stmt
),
2763 "only distribute or parallel constructs are allowed to "
2764 "be closely nested inside teams construct");
2769 switch (gimple_code (stmt
))
2771 case GIMPLE_OMP_FOR
:
2772 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2774 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2776 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2778 error_at (gimple_location (stmt
),
2779 "distribute construct must be closely nested inside "
2787 if (is_gimple_call (stmt
)
2788 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2789 == BUILT_IN_GOMP_CANCEL
2790 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2791 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2793 const char *bad
= NULL
;
2794 const char *kind
= NULL
;
2797 error_at (gimple_location (stmt
), "orphaned %qs construct",
2798 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2799 == BUILT_IN_GOMP_CANCEL
2800 ? "#pragma omp cancel"
2801 : "#pragma omp cancellation point");
2804 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2805 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2809 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2810 bad
= "#pragma omp parallel";
2811 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2812 == BUILT_IN_GOMP_CANCEL
2813 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2814 ctx
->cancellable
= true;
2818 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2819 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2820 bad
= "#pragma omp for";
2821 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2822 == BUILT_IN_GOMP_CANCEL
2823 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2825 ctx
->cancellable
= true;
2826 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2828 warning_at (gimple_location (stmt
), 0,
2829 "%<#pragma omp cancel for%> inside "
2830 "%<nowait%> for construct");
2831 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2832 OMP_CLAUSE_ORDERED
))
2833 warning_at (gimple_location (stmt
), 0,
2834 "%<#pragma omp cancel for%> inside "
2835 "%<ordered%> for construct");
2840 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2841 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2842 bad
= "#pragma omp sections";
2843 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2844 == BUILT_IN_GOMP_CANCEL
2845 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2847 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2849 ctx
->cancellable
= true;
2850 if (find_omp_clause (gimple_omp_sections_clauses
2853 warning_at (gimple_location (stmt
), 0,
2854 "%<#pragma omp cancel sections%> inside "
2855 "%<nowait%> sections construct");
2859 gcc_assert (ctx
->outer
2860 && gimple_code (ctx
->outer
->stmt
)
2861 == GIMPLE_OMP_SECTIONS
);
2862 ctx
->outer
->cancellable
= true;
2863 if (find_omp_clause (gimple_omp_sections_clauses
2866 warning_at (gimple_location (stmt
), 0,
2867 "%<#pragma omp cancel sections%> inside "
2868 "%<nowait%> sections construct");
2874 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2875 bad
= "#pragma omp task";
2877 ctx
->cancellable
= true;
2881 error_at (gimple_location (stmt
), "invalid arguments");
2886 error_at (gimple_location (stmt
),
2887 "%<%s %s%> construct not closely nested inside of %qs",
2888 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2889 == BUILT_IN_GOMP_CANCEL
2890 ? "#pragma omp cancel"
2891 : "#pragma omp cancellation point", kind
, bad
);
2896 case GIMPLE_OMP_SECTIONS
:
2897 case GIMPLE_OMP_SINGLE
:
2898 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2899 switch (gimple_code (ctx
->stmt
))
2901 case GIMPLE_OMP_FOR
:
2902 case GIMPLE_OMP_SECTIONS
:
2903 case GIMPLE_OMP_SINGLE
:
2904 case GIMPLE_OMP_ORDERED
:
2905 case GIMPLE_OMP_MASTER
:
2906 case GIMPLE_OMP_TASK
:
2907 case GIMPLE_OMP_CRITICAL
:
2908 if (is_gimple_call (stmt
))
2910 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2911 != BUILT_IN_GOMP_BARRIER
)
2913 error_at (gimple_location (stmt
),
2914 "barrier region may not be closely nested inside "
2915 "of work-sharing, critical, ordered, master or "
2916 "explicit task region");
2919 error_at (gimple_location (stmt
),
2920 "work-sharing region may not be closely nested inside "
2921 "of work-sharing, critical, ordered, master or explicit "
2924 case GIMPLE_OMP_PARALLEL
:
2930 case GIMPLE_OMP_MASTER
:
2931 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2932 switch (gimple_code (ctx
->stmt
))
2934 case GIMPLE_OMP_FOR
:
2935 case GIMPLE_OMP_SECTIONS
:
2936 case GIMPLE_OMP_SINGLE
:
2937 case GIMPLE_OMP_TASK
:
2938 error_at (gimple_location (stmt
),
2939 "master region may not be closely nested inside "
2940 "of work-sharing or explicit task region");
2942 case GIMPLE_OMP_PARALLEL
:
2948 case GIMPLE_OMP_ORDERED
:
2949 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2950 switch (gimple_code (ctx
->stmt
))
2952 case GIMPLE_OMP_CRITICAL
:
2953 case GIMPLE_OMP_TASK
:
2954 error_at (gimple_location (stmt
),
2955 "ordered region may not be closely nested inside "
2956 "of critical or explicit task region");
2958 case GIMPLE_OMP_FOR
:
2959 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2960 OMP_CLAUSE_ORDERED
) == NULL
)
2962 error_at (gimple_location (stmt
),
2963 "ordered region must be closely nested inside "
2964 "a loop region with an ordered clause");
2968 case GIMPLE_OMP_PARALLEL
:
2969 error_at (gimple_location (stmt
),
2970 "ordered region must be closely nested inside "
2971 "a loop region with an ordered clause");
2977 case GIMPLE_OMP_CRITICAL
:
2980 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
2981 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2982 if (gomp_critical
*other_crit
2983 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
2984 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
2986 error_at (gimple_location (stmt
),
2987 "critical region may not be nested inside a critical "
2988 "region with the same name");
2993 case GIMPLE_OMP_TEAMS
:
2995 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2996 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2998 error_at (gimple_location (stmt
),
2999 "teams construct not closely nested inside of target "
3004 case GIMPLE_OMP_TARGET
:
3005 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3007 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3009 if (is_gimple_omp (stmt
)
3010 && is_gimple_omp_oacc (stmt
)
3011 && is_gimple_omp (ctx
->stmt
))
3013 error_at (gimple_location (stmt
),
3014 "OpenACC construct inside of non-OpenACC region");
3020 const char *stmt_name
, *ctx_stmt_name
;
3021 switch (gimple_omp_target_kind (stmt
))
3023 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3024 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3025 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3026 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3027 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3028 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3029 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3030 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
: stmt_name
= "enter/exit data"; break;
3031 default: gcc_unreachable ();
3033 switch (gimple_omp_target_kind (ctx
->stmt
))
3035 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3036 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3037 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: ctx_stmt_name
= "parallel"; break;
3038 case GF_OMP_TARGET_KIND_OACC_KERNELS
: ctx_stmt_name
= "kernels"; break;
3039 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3040 default: gcc_unreachable ();
3043 /* OpenACC/OpenMP mismatch? */
3044 if (is_gimple_omp_oacc (stmt
)
3045 != is_gimple_omp_oacc (ctx
->stmt
))
3047 error_at (gimple_location (stmt
),
3048 "%s %s construct inside of %s %s region",
3049 (is_gimple_omp_oacc (stmt
)
3050 ? "OpenACC" : "OpenMP"), stmt_name
,
3051 (is_gimple_omp_oacc (ctx
->stmt
)
3052 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3055 if (is_gimple_omp_offloaded (ctx
->stmt
))
3057 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3058 if (is_gimple_omp_oacc (ctx
->stmt
))
3060 error_at (gimple_location (stmt
),
3061 "%s construct inside of %s region",
3062 stmt_name
, ctx_stmt_name
);
3067 gcc_checking_assert (!is_gimple_omp_oacc (stmt
));
3068 warning_at (gimple_location (stmt
), 0,
3069 "%s construct inside of %s region",
3070 stmt_name
, ctx_stmt_name
);
3082 /* Helper function scan_omp.
3084 Callback for walk_tree or operators in walk_gimple_stmt used to
3085 scan for OMP directives in TP. */
3088 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3090 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3091 omp_context
*ctx
= (omp_context
*) wi
->info
;
3094 switch (TREE_CODE (t
))
3101 *tp
= remap_decl (t
, &ctx
->cb
);
3105 if (ctx
&& TYPE_P (t
))
3106 *tp
= remap_type (t
, &ctx
->cb
);
3107 else if (!DECL_P (t
))
3112 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3113 if (tem
!= TREE_TYPE (t
))
3115 if (TREE_CODE (t
) == INTEGER_CST
)
3116 *tp
= wide_int_to_tree (tem
, t
);
3118 TREE_TYPE (t
) = tem
;
3128 /* Return true if FNDECL is a setjmp or a longjmp. */
3131 setjmp_or_longjmp_p (const_tree fndecl
)
3133 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3134 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3135 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3138 tree declname
= DECL_NAME (fndecl
);
3141 const char *name
= IDENTIFIER_POINTER (declname
);
3142 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3146 /* Helper function for scan_omp.
3148 Callback for walk_gimple_stmt used to scan for OMP directives in
3149 the current statement in GSI. */
3152 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3153 struct walk_stmt_info
*wi
)
3155 gimple stmt
= gsi_stmt (*gsi
);
3156 omp_context
*ctx
= (omp_context
*) wi
->info
;
3158 if (gimple_has_location (stmt
))
3159 input_location
= gimple_location (stmt
);
3161 /* Check the nesting restrictions. */
3162 bool remove
= false;
3163 if (is_gimple_omp (stmt
))
3164 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3165 else if (is_gimple_call (stmt
))
3167 tree fndecl
= gimple_call_fndecl (stmt
);
3170 if (setjmp_or_longjmp_p (fndecl
)
3172 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3173 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3176 error_at (gimple_location (stmt
),
3177 "setjmp/longjmp inside simd construct");
3179 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3180 switch (DECL_FUNCTION_CODE (fndecl
))
3182 case BUILT_IN_GOMP_BARRIER
:
3183 case BUILT_IN_GOMP_CANCEL
:
3184 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3185 case BUILT_IN_GOMP_TASKYIELD
:
3186 case BUILT_IN_GOMP_TASKWAIT
:
3187 case BUILT_IN_GOMP_TASKGROUP_START
:
3188 case BUILT_IN_GOMP_TASKGROUP_END
:
3189 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3198 stmt
= gimple_build_nop ();
3199 gsi_replace (gsi
, stmt
, false);
3202 *handled_ops_p
= true;
3204 switch (gimple_code (stmt
))
3206 case GIMPLE_OMP_PARALLEL
:
3207 taskreg_nesting_level
++;
3208 scan_omp_parallel (gsi
, ctx
);
3209 taskreg_nesting_level
--;
3212 case GIMPLE_OMP_TASK
:
3213 taskreg_nesting_level
++;
3214 scan_omp_task (gsi
, ctx
);
3215 taskreg_nesting_level
--;
3218 case GIMPLE_OMP_FOR
:
3219 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3222 case GIMPLE_OMP_SECTIONS
:
3223 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3226 case GIMPLE_OMP_SINGLE
:
3227 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3230 case GIMPLE_OMP_SECTION
:
3231 case GIMPLE_OMP_MASTER
:
3232 case GIMPLE_OMP_TASKGROUP
:
3233 case GIMPLE_OMP_ORDERED
:
3234 case GIMPLE_OMP_CRITICAL
:
3235 ctx
= new_omp_context (stmt
, ctx
);
3236 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3239 case GIMPLE_OMP_TARGET
:
3240 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3243 case GIMPLE_OMP_TEAMS
:
3244 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3251 *handled_ops_p
= false;
3253 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3255 var
= DECL_CHAIN (var
))
3256 insert_decl_map (&ctx
->cb
, var
, var
);
3260 *handled_ops_p
= false;
3268 /* Scan all the statements starting at the current statement. CTX
3269 contains context information about the OMP directives and
3270 clauses found during the scan. */
3273 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3275 location_t saved_location
;
3276 struct walk_stmt_info wi
;
3278 memset (&wi
, 0, sizeof (wi
));
3280 wi
.want_locations
= true;
3282 saved_location
= input_location
;
3283 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3284 input_location
= saved_location
;
3287 /* Re-gimplification and code generation routines. */
3289 /* Build a call to GOMP_barrier. */
3292 build_omp_barrier (tree lhs
)
3294 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3295 : BUILT_IN_GOMP_BARRIER
);
3296 gcall
*g
= gimple_build_call (fndecl
, 0);
3298 gimple_call_set_lhs (g
, lhs
);
3302 /* If a context was created for STMT when it was scanned, return it. */
3304 static omp_context
*
3305 maybe_lookup_ctx (gimple stmt
)
3308 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3309 return n
? (omp_context
*) n
->value
: NULL
;
3313 /* Find the mapping for DECL in CTX or the immediately enclosing
3314 context that has a mapping for DECL.
3316 If CTX is a nested parallel directive, we may have to use the decl
3317 mappings created in CTX's parent context. Suppose that we have the
3318 following parallel nesting (variable UIDs showed for clarity):
3321 #omp parallel shared(iD.1562) -> outer parallel
3322 iD.1562 = iD.1562 + 1;
3324 #omp parallel shared (iD.1562) -> inner parallel
3325 iD.1562 = iD.1562 - 1;
3327 Each parallel structure will create a distinct .omp_data_s structure
3328 for copying iD.1562 in/out of the directive:
3330 outer parallel .omp_data_s.1.i -> iD.1562
3331 inner parallel .omp_data_s.2.i -> iD.1562
3333 A shared variable mapping will produce a copy-out operation before
3334 the parallel directive and a copy-in operation after it. So, in
3335 this case we would have:
3338 .omp_data_o.1.i = iD.1562;
3339 #omp parallel shared(iD.1562) -> outer parallel
3340 .omp_data_i.1 = &.omp_data_o.1
3341 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3343 .omp_data_o.2.i = iD.1562; -> **
3344 #omp parallel shared(iD.1562) -> inner parallel
3345 .omp_data_i.2 = &.omp_data_o.2
3346 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3349 ** This is a problem. The symbol iD.1562 cannot be referenced
3350 inside the body of the outer parallel region. But since we are
3351 emitting this copy operation while expanding the inner parallel
3352 directive, we need to access the CTX structure of the outer
3353 parallel directive to get the correct mapping:
3355 .omp_data_o.2.i = .omp_data_i.1->i
3357 Since there may be other workshare or parallel directives enclosing
3358 the parallel directive, it may be necessary to walk up the context
3359 parent chain. This is not a problem in general because nested
3360 parallelism happens only rarely. */
3363 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3368 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3369 t
= maybe_lookup_decl (decl
, up
);
3371 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3373 return t
? t
: decl
;
3377 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3378 in outer contexts. */
3381 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3386 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3387 t
= maybe_lookup_decl (decl
, up
);
3389 return t
? t
: decl
;
3393 /* Construct the initialization value for reduction CLAUSE. */
3396 omp_reduction_init (tree clause
, tree type
)
3398 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3399 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3406 case TRUTH_ORIF_EXPR
:
3407 case TRUTH_XOR_EXPR
:
3409 return build_zero_cst (type
);
3412 case TRUTH_AND_EXPR
:
3413 case TRUTH_ANDIF_EXPR
:
3415 return fold_convert_loc (loc
, type
, integer_one_node
);
3418 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3421 if (SCALAR_FLOAT_TYPE_P (type
))
3423 REAL_VALUE_TYPE max
, min
;
3424 if (HONOR_INFINITIES (type
))
3427 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3430 real_maxval (&min
, 1, TYPE_MODE (type
));
3431 return build_real (type
, min
);
3435 gcc_assert (INTEGRAL_TYPE_P (type
));
3436 return TYPE_MIN_VALUE (type
);
3440 if (SCALAR_FLOAT_TYPE_P (type
))
3442 REAL_VALUE_TYPE max
;
3443 if (HONOR_INFINITIES (type
))
3446 real_maxval (&max
, 0, TYPE_MODE (type
));
3447 return build_real (type
, max
);
3451 gcc_assert (INTEGRAL_TYPE_P (type
));
3452 return TYPE_MAX_VALUE (type
);
3460 /* Return alignment to be assumed for var in CLAUSE, which should be
3461 OMP_CLAUSE_ALIGNED. */
3464 omp_clause_aligned_alignment (tree clause
)
3466 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3467 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3469 /* Otherwise return implementation defined alignment. */
3470 unsigned int al
= 1;
3471 machine_mode mode
, vmode
;
3472 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3474 vs
= 1 << floor_log2 (vs
);
3475 static enum mode_class classes
[]
3476 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3477 for (int i
= 0; i
< 4; i
+= 2)
3478 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3480 mode
= GET_MODE_WIDER_MODE (mode
))
3482 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3483 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3486 && GET_MODE_SIZE (vmode
) < vs
3487 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3488 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3490 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3491 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3493 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3494 / GET_MODE_SIZE (mode
));
3495 if (TYPE_MODE (type
) != vmode
)
3497 if (TYPE_ALIGN_UNIT (type
) > al
)
3498 al
= TYPE_ALIGN_UNIT (type
);
3500 return build_int_cst (integer_type_node
, al
);
3503 /* Return maximum possible vectorization factor for the target. */
3510 || !flag_tree_loop_optimize
3511 || (!flag_tree_loop_vectorize
3512 && (global_options_set
.x_flag_tree_loop_vectorize
3513 || global_options_set
.x_flag_tree_vectorize
)))
3516 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3519 vs
= 1 << floor_log2 (vs
);
3522 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3523 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3524 return GET_MODE_NUNITS (vqimode
);
3528 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3532 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3533 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3537 max_vf
= omp_max_vf ();
3540 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3541 OMP_CLAUSE_SAFELEN
);
3542 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3544 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3546 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3550 idx
= create_tmp_var (unsigned_type_node
);
3551 lane
= create_tmp_var (unsigned_type_node
);
3557 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3558 tree avar
= create_tmp_var_raw (atype
);
3559 if (TREE_ADDRESSABLE (new_var
))
3560 TREE_ADDRESSABLE (avar
) = 1;
3561 DECL_ATTRIBUTES (avar
)
3562 = tree_cons (get_identifier ("omp simd array"), NULL
,
3563 DECL_ATTRIBUTES (avar
));
3564 gimple_add_tmp_var (avar
);
3565 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3566 NULL_TREE
, NULL_TREE
);
3567 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3568 NULL_TREE
, NULL_TREE
);
3569 if (DECL_P (new_var
))
3571 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3572 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3577 /* Helper function of lower_rec_input_clauses. For a reference
3578 in simd reduction, add an underlying variable it will reference. */
3581 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3583 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3584 if (TREE_CONSTANT (z
))
3586 const char *name
= NULL
;
3587 if (DECL_NAME (new_vard
))
3588 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3590 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3591 gimple_add_tmp_var (z
);
3592 TREE_ADDRESSABLE (z
) = 1;
3593 z
= build_fold_addr_expr_loc (loc
, z
);
3594 gimplify_assign (new_vard
, z
, ilist
);
3598 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3599 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3600 private variables. Initialization statements go in ILIST, while calls
3601 to destructors go in DLIST. */
3604 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3605 omp_context
*ctx
, struct omp_for_data
*fd
)
3607 tree c
, dtor
, copyin_seq
, x
, ptr
;
3608 bool copyin_by_ref
= false;
3609 bool lastprivate_firstprivate
= false;
3610 bool reduction_omp_orig_ref
= false;
3612 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3613 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3615 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3616 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3617 gimple_seq llist
[2] = { NULL
, NULL
};
3621 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3622 with data sharing clauses referencing variable sized vars. That
3623 is unnecessarily hard to support and very unlikely to result in
3624 vectorized code anyway. */
3626 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3627 switch (OMP_CLAUSE_CODE (c
))
3629 case OMP_CLAUSE_LINEAR
:
3630 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3633 case OMP_CLAUSE_REDUCTION
:
3634 case OMP_CLAUSE_PRIVATE
:
3635 case OMP_CLAUSE_FIRSTPRIVATE
:
3636 case OMP_CLAUSE_LASTPRIVATE
:
3637 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3644 /* Do all the fixed sized types in the first pass, and the variable sized
3645 types in the second pass. This makes sure that the scalar arguments to
3646 the variable sized types are processed before we use them in the
3647 variable sized operations. */
3648 for (pass
= 0; pass
< 2; ++pass
)
3650 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3652 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3655 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3659 case OMP_CLAUSE_PRIVATE
:
3660 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3663 case OMP_CLAUSE_SHARED
:
3664 /* Ignore shared directives in teams construct. */
3665 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3667 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3669 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3672 case OMP_CLAUSE_FIRSTPRIVATE
:
3673 case OMP_CLAUSE_COPYIN
:
3674 case OMP_CLAUSE_LINEAR
:
3676 case OMP_CLAUSE_REDUCTION
:
3677 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3678 reduction_omp_orig_ref
= true;
3680 case OMP_CLAUSE__LOOPTEMP_
:
3681 /* Handle _looptemp_ clauses only on parallel. */
3685 case OMP_CLAUSE_LASTPRIVATE
:
3686 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3688 lastprivate_firstprivate
= true;
3692 /* Even without corresponding firstprivate, if
3693 decl is Fortran allocatable, it needs outer var
3696 && lang_hooks
.decls
.omp_private_outer_ref
3697 (OMP_CLAUSE_DECL (c
)))
3698 lastprivate_firstprivate
= true;
3700 case OMP_CLAUSE_ALIGNED
:
3703 var
= OMP_CLAUSE_DECL (c
);
3704 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3705 && !is_global_var (var
))
3707 new_var
= maybe_lookup_decl (var
, ctx
);
3708 if (new_var
== NULL_TREE
)
3709 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3710 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3711 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3712 omp_clause_aligned_alignment (c
));
3713 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3714 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3715 gimplify_and_add (x
, ilist
);
3717 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3718 && is_global_var (var
))
3720 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3721 new_var
= lookup_decl (var
, ctx
);
3722 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3723 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3724 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3725 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3726 omp_clause_aligned_alignment (c
));
3727 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3728 x
= create_tmp_var (ptype
);
3729 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3730 gimplify_and_add (t
, ilist
);
3731 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3732 SET_DECL_VALUE_EXPR (new_var
, t
);
3733 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3740 new_var
= var
= OMP_CLAUSE_DECL (c
);
3741 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3742 new_var
= lookup_decl (var
, ctx
);
3744 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3749 else if (is_variable_sized (var
))
3751 /* For variable sized types, we need to allocate the
3752 actual storage here. Call alloca and store the
3753 result in the pointer decl that we created elsewhere. */
3757 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3762 ptr
= DECL_VALUE_EXPR (new_var
);
3763 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3764 ptr
= TREE_OPERAND (ptr
, 0);
3765 gcc_assert (DECL_P (ptr
));
3766 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3768 /* void *tmp = __builtin_alloca */
3769 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3770 stmt
= gimple_build_call (atmp
, 1, x
);
3771 tmp
= create_tmp_var_raw (ptr_type_node
);
3772 gimple_add_tmp_var (tmp
);
3773 gimple_call_set_lhs (stmt
, tmp
);
3775 gimple_seq_add_stmt (ilist
, stmt
);
3777 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3778 gimplify_assign (ptr
, x
, ilist
);
3781 else if (is_reference (var
))
3783 /* For references that are being privatized for Fortran,
3784 allocate new backing storage for the new pointer
3785 variable. This allows us to avoid changing all the
3786 code that expects a pointer to something that expects
3787 a direct variable. */
3791 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3792 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3794 x
= build_receiver_ref (var
, false, ctx
);
3795 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3797 else if (TREE_CONSTANT (x
))
3799 /* For reduction in SIMD loop, defer adding the
3800 initialization of the reference, because if we decide
3801 to use SIMD array for it, the initilization could cause
3803 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3807 const char *name
= NULL
;
3808 if (DECL_NAME (var
))
3809 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3811 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3813 gimple_add_tmp_var (x
);
3814 TREE_ADDRESSABLE (x
) = 1;
3815 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3820 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3821 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3826 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3827 gimplify_assign (new_var
, x
, ilist
);
3830 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3832 else if (c_kind
== OMP_CLAUSE_REDUCTION
3833 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3841 switch (OMP_CLAUSE_CODE (c
))
3843 case OMP_CLAUSE_SHARED
:
3844 /* Ignore shared directives in teams construct. */
3845 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3847 /* Shared global vars are just accessed directly. */
3848 if (is_global_var (new_var
))
3850 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3851 needs to be delayed until after fixup_child_record_type so
3852 that we get the correct type during the dereference. */
3853 by_ref
= use_pointer_for_field (var
, ctx
);
3854 x
= build_receiver_ref (var
, by_ref
, ctx
);
3855 SET_DECL_VALUE_EXPR (new_var
, x
);
3856 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3858 /* ??? If VAR is not passed by reference, and the variable
3859 hasn't been initialized yet, then we'll get a warning for
3860 the store into the omp_data_s structure. Ideally, we'd be
3861 able to notice this and not store anything at all, but
3862 we're generating code too early. Suppress the warning. */
3864 TREE_NO_WARNING (var
) = 1;
3867 case OMP_CLAUSE_LASTPRIVATE
:
3868 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3872 case OMP_CLAUSE_PRIVATE
:
3873 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3874 x
= build_outer_var_ref (var
, ctx
);
3875 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3877 if (is_task_ctx (ctx
))
3878 x
= build_receiver_ref (var
, false, ctx
);
3880 x
= build_outer_var_ref (var
, ctx
);
3886 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3889 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3890 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3891 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3892 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3893 idx
, lane
, ivar
, lvar
))
3896 x
= lang_hooks
.decls
.omp_clause_default_ctor
3897 (c
, unshare_expr (ivar
), x
);
3899 gimplify_and_add (x
, &llist
[0]);
3902 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3905 gimple_seq tseq
= NULL
;
3908 gimplify_stmt (&dtor
, &tseq
);
3909 gimple_seq_add_seq (&llist
[1], tseq
);
3916 gimplify_and_add (nx
, ilist
);
3920 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3923 gimple_seq tseq
= NULL
;
3926 gimplify_stmt (&dtor
, &tseq
);
3927 gimple_seq_add_seq (dlist
, tseq
);
3931 case OMP_CLAUSE_LINEAR
:
3932 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3933 goto do_firstprivate
;
3934 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3937 x
= build_outer_var_ref (var
, ctx
);
3940 case OMP_CLAUSE_FIRSTPRIVATE
:
3941 if (is_task_ctx (ctx
))
3943 if (is_reference (var
) || is_variable_sized (var
))
3945 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3947 || use_pointer_for_field (var
, NULL
))
3949 x
= build_receiver_ref (var
, false, ctx
);
3950 SET_DECL_VALUE_EXPR (new_var
, x
);
3951 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3956 x
= build_outer_var_ref (var
, ctx
);
3959 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3960 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3962 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3963 tree stept
= TREE_TYPE (t
);
3964 tree ct
= find_omp_clause (clauses
,
3965 OMP_CLAUSE__LOOPTEMP_
);
3967 tree l
= OMP_CLAUSE_DECL (ct
);
3968 tree n1
= fd
->loop
.n1
;
3969 tree step
= fd
->loop
.step
;
3970 tree itype
= TREE_TYPE (l
);
3971 if (POINTER_TYPE_P (itype
))
3972 itype
= signed_type_for (itype
);
3973 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3974 if (TYPE_UNSIGNED (itype
)
3975 && fd
->loop
.cond_code
== GT_EXPR
)
3976 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3977 fold_build1 (NEGATE_EXPR
, itype
, l
),
3978 fold_build1 (NEGATE_EXPR
,
3981 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3982 t
= fold_build2 (MULT_EXPR
, stept
,
3983 fold_convert (stept
, l
), t
);
3985 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3987 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3989 gimplify_and_add (x
, ilist
);
3993 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3994 x
= fold_build2 (POINTER_PLUS_EXPR
,
3995 TREE_TYPE (x
), x
, t
);
3997 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
4000 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
4001 || TREE_ADDRESSABLE (new_var
))
4002 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4003 idx
, lane
, ivar
, lvar
))
4005 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
4007 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
4008 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
4009 gimplify_and_add (x
, ilist
);
4010 gimple_stmt_iterator gsi
4011 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4013 = gimple_build_assign (unshare_expr (lvar
), iv
);
4014 gsi_insert_before_without_update (&gsi
, g
,
4016 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4017 enum tree_code code
= PLUS_EXPR
;
4018 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4019 code
= POINTER_PLUS_EXPR
;
4020 g
= gimple_build_assign (iv
, code
, iv
, t
);
4021 gsi_insert_before_without_update (&gsi
, g
,
4025 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4026 (c
, unshare_expr (ivar
), x
);
4027 gimplify_and_add (x
, &llist
[0]);
4028 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4031 gimple_seq tseq
= NULL
;
4034 gimplify_stmt (&dtor
, &tseq
);
4035 gimple_seq_add_seq (&llist
[1], tseq
);
4040 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
4041 gimplify_and_add (x
, ilist
);
4044 case OMP_CLAUSE__LOOPTEMP_
:
4045 gcc_assert (is_parallel_ctx (ctx
));
4046 x
= build_outer_var_ref (var
, ctx
);
4047 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4048 gimplify_and_add (x
, ilist
);
4051 case OMP_CLAUSE_COPYIN
:
4052 by_ref
= use_pointer_for_field (var
, NULL
);
4053 x
= build_receiver_ref (var
, by_ref
, ctx
);
4054 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4055 append_to_statement_list (x
, ©in_seq
);
4056 copyin_by_ref
|= by_ref
;
4059 case OMP_CLAUSE_REDUCTION
:
4060 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4062 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4064 x
= build_outer_var_ref (var
, ctx
);
4066 if (is_reference (var
)
4067 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4069 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4070 SET_DECL_VALUE_EXPR (placeholder
, x
);
4071 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4072 tree new_vard
= new_var
;
4073 if (is_reference (var
))
4075 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4076 new_vard
= TREE_OPERAND (new_var
, 0);
4077 gcc_assert (DECL_P (new_vard
));
4080 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4081 idx
, lane
, ivar
, lvar
))
4083 if (new_vard
== new_var
)
4085 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4086 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4090 SET_DECL_VALUE_EXPR (new_vard
,
4091 build_fold_addr_expr (ivar
));
4092 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4094 x
= lang_hooks
.decls
.omp_clause_default_ctor
4095 (c
, unshare_expr (ivar
),
4096 build_outer_var_ref (var
, ctx
));
4098 gimplify_and_add (x
, &llist
[0]);
4099 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4101 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4102 lower_omp (&tseq
, ctx
);
4103 gimple_seq_add_seq (&llist
[0], tseq
);
4105 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4106 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4107 lower_omp (&tseq
, ctx
);
4108 gimple_seq_add_seq (&llist
[1], tseq
);
4109 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4110 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4111 if (new_vard
== new_var
)
4112 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4114 SET_DECL_VALUE_EXPR (new_vard
,
4115 build_fold_addr_expr (lvar
));
4116 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4121 gimplify_stmt (&dtor
, &tseq
);
4122 gimple_seq_add_seq (&llist
[1], tseq
);
4126 /* If this is a reference to constant size reduction var
4127 with placeholder, we haven't emitted the initializer
4128 for it because it is undesirable if SIMD arrays are used.
4129 But if they aren't used, we need to emit the deferred
4130 initialization now. */
4131 else if (is_reference (var
) && is_simd
)
4132 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4133 x
= lang_hooks
.decls
.omp_clause_default_ctor
4134 (c
, unshare_expr (new_var
),
4135 build_outer_var_ref (var
, ctx
));
4137 gimplify_and_add (x
, ilist
);
4138 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4140 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4141 lower_omp (&tseq
, ctx
);
4142 gimple_seq_add_seq (ilist
, tseq
);
4144 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4147 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4148 lower_omp (&tseq
, ctx
);
4149 gimple_seq_add_seq (dlist
, tseq
);
4150 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4152 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4157 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4158 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4159 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4161 /* reduction(-:var) sums up the partial results, so it
4162 acts identically to reduction(+:var). */
4163 if (code
== MINUS_EXPR
)
4166 tree new_vard
= new_var
;
4167 if (is_simd
&& is_reference (var
))
4169 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4170 new_vard
= TREE_OPERAND (new_var
, 0);
4171 gcc_assert (DECL_P (new_vard
));
4174 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4175 idx
, lane
, ivar
, lvar
))
4177 tree ref
= build_outer_var_ref (var
, ctx
);
4179 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
4181 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
4182 ref
= build_outer_var_ref (var
, ctx
);
4183 gimplify_assign (ref
, x
, &llist
[1]);
4185 if (new_vard
!= new_var
)
4187 SET_DECL_VALUE_EXPR (new_vard
,
4188 build_fold_addr_expr (lvar
));
4189 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4194 if (is_reference (var
) && is_simd
)
4195 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4196 gimplify_assign (new_var
, x
, ilist
);
4199 tree ref
= build_outer_var_ref (var
, ctx
);
4201 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4202 ref
= build_outer_var_ref (var
, ctx
);
4203 gimplify_assign (ref
, x
, dlist
);
4217 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
4218 /* Don't want uninit warnings on simduid, it is always uninitialized,
4219 but we use it not for the value, but for the DECL_UID only. */
4220 TREE_NO_WARNING (uid
) = 1;
4222 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
4223 gimple_call_set_lhs (g
, lane
);
4224 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4225 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
4226 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
4227 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
4228 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4229 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4230 g
= gimple_build_assign (lane
, INTEGER_CST
,
4231 build_int_cst (unsigned_type_node
, 0));
4232 gimple_seq_add_stmt (ilist
, g
);
4233 for (int i
= 0; i
< 2; i
++)
4236 tree vf
= create_tmp_var (unsigned_type_node
);
4237 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
4238 gimple_call_set_lhs (g
, vf
);
4239 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
4240 gimple_seq_add_stmt (seq
, g
);
4241 tree t
= build_int_cst (unsigned_type_node
, 0);
4242 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
4243 gimple_seq_add_stmt (seq
, g
);
4244 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4245 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
4246 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4247 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
4248 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
4249 gimple_seq_add_seq (seq
, llist
[i
]);
4250 t
= build_int_cst (unsigned_type_node
, 1);
4251 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
4252 gimple_seq_add_stmt (seq
, g
);
4253 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
4254 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
4255 gimple_seq_add_stmt (seq
, g
);
4256 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
4260 /* The copyin sequence is not to be executed by the main thread, since
4261 that would result in self-copies. Perhaps not visible to scalars,
4262 but it certainly is to C++ operator=. */
4265 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
4267 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
4268 build_int_cst (TREE_TYPE (x
), 0));
4269 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
4270 gimplify_and_add (x
, ilist
);
4273 /* If any copyin variable is passed by reference, we must ensure the
4274 master thread doesn't modify it before it is copied over in all
4275 threads. Similarly for variables in both firstprivate and
4276 lastprivate clauses we need to ensure the lastprivate copying
4277 happens after firstprivate copying in all threads. And similarly
4278 for UDRs if initializer expression refers to omp_orig. */
4279 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
4281 /* Don't add any barrier for #pragma omp simd or
4282 #pragma omp distribute. */
4283 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
4284 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
4285 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
4288 /* If max_vf is non-zero, then we can use only a vectorization factor
4289 up to the max_vf we chose. So stick it into the safelen clause. */
4292 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4293 OMP_CLAUSE_SAFELEN
);
4295 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
4296 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4299 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
4300 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
4302 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4303 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4309 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4310 both parallel and workshare constructs. PREDICATE may be NULL if it's
4314 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
4317 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
4318 bool par_clauses
= false;
4319 tree simduid
= NULL
, lastlane
= NULL
;
4321 /* Early exit if there are no lastprivate or linear clauses. */
4322 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
4323 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
4324 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
4325 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
4327 if (clauses
== NULL
)
4329 /* If this was a workshare clause, see if it had been combined
4330 with its parallel. In that case, look for the clauses on the
4331 parallel statement itself. */
4332 if (is_parallel_ctx (ctx
))
4336 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4339 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4340 OMP_CLAUSE_LASTPRIVATE
);
4341 if (clauses
== NULL
)
4349 tree label_true
, arm1
, arm2
;
4351 label
= create_artificial_label (UNKNOWN_LOCATION
);
4352 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
4353 arm1
= TREE_OPERAND (predicate
, 0);
4354 arm2
= TREE_OPERAND (predicate
, 1);
4355 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4356 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4357 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
4359 gimple_seq_add_stmt (stmt_list
, stmt
);
4360 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
4363 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4364 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4366 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
4368 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
4371 for (c
= clauses
; c
;)
4374 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4376 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4377 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4378 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
4380 var
= OMP_CLAUSE_DECL (c
);
4381 new_var
= lookup_decl (var
, ctx
);
4383 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
4385 tree val
= DECL_VALUE_EXPR (new_var
);
4386 if (TREE_CODE (val
) == ARRAY_REF
4387 && VAR_P (TREE_OPERAND (val
, 0))
4388 && lookup_attribute ("omp simd array",
4389 DECL_ATTRIBUTES (TREE_OPERAND (val
,
4392 if (lastlane
== NULL
)
4394 lastlane
= create_tmp_var (unsigned_type_node
);
4396 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4398 TREE_OPERAND (val
, 1));
4399 gimple_call_set_lhs (g
, lastlane
);
4400 gimple_seq_add_stmt (stmt_list
, g
);
4402 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4403 TREE_OPERAND (val
, 0), lastlane
,
4404 NULL_TREE
, NULL_TREE
);
4408 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4409 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4411 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4412 gimple_seq_add_seq (stmt_list
,
4413 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4414 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4416 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4417 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4419 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4420 gimple_seq_add_seq (stmt_list
,
4421 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4422 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4425 x
= build_outer_var_ref (var
, ctx
);
4426 if (is_reference (var
))
4427 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4428 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4429 gimplify_and_add (x
, stmt_list
);
4431 c
= OMP_CLAUSE_CHAIN (c
);
4432 if (c
== NULL
&& !par_clauses
)
4434 /* If this was a workshare clause, see if it had been combined
4435 with its parallel. In that case, continue looking for the
4436 clauses also on the parallel statement itself. */
4437 if (is_parallel_ctx (ctx
))
4441 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4444 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4445 OMP_CLAUSE_LASTPRIVATE
);
4451 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4455 oacc_lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
,
4456 tree tid
, tree var
, tree new_var
)
4458 /* The atomic add at the end of the sum creates unnecessary
4459 write contention on accelerators. To work around this,
4460 create an array to store the partial reductions. Later, in
4461 lower_omp_for (for openacc), the values of array will be
4464 tree t
= NULL_TREE
, array
, x
;
4465 tree type
= get_base_type (var
);
4468 /* Now insert the partial reductions into the array. */
4470 /* Find the reduction array. */
4472 tree ptype
= build_pointer_type (type
);
4474 t
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
), ctx
);
4475 t
= build_receiver_ref (t
, false, ctx
->outer
);
4477 array
= create_tmp_var (ptype
);
4478 gimplify_assign (array
, t
, stmt_seqp
);
4480 tree ptr
= create_tmp_var (TREE_TYPE (array
));
4482 /* Find the reduction array. */
4484 /* testing a unary conversion. */
4485 tree offset
= create_tmp_var (sizetype
);
4486 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
4488 t
= create_tmp_var (sizetype
);
4489 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
4491 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, t
);
4492 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4494 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4495 of adding sizeof(var) to the array? */
4496 ptr
= create_tmp_var (ptype
);
4497 stmt
= gimple_build_assign (unshare_expr (ptr
), POINTER_PLUS_EXPR
, array
,
4499 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4501 /* Move the local sum to gfc$sum[i]. */
4502 x
= unshare_expr (build_simple_mem_ref (ptr
));
4503 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
4506 /* Generate code to implement the REDUCTION clauses. */
4509 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4511 gimple_seq sub_seq
= NULL
;
4513 tree x
, c
, tid
= NULL_TREE
;
4516 /* SIMD reductions are handled in lower_rec_input_clauses. */
4517 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4518 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4521 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4522 update in that case, otherwise use a lock. */
4523 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4524 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4526 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4528 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4538 /* Initialize thread info for OpenACC. */
4539 if (is_gimple_omp_oacc (ctx
->stmt
))
4541 /* Get the current thread id. */
4542 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
4543 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)));
4544 gimple stmt
= gimple_build_call (call
, 0);
4545 gimple_call_set_lhs (stmt
, tid
);
4546 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4549 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4551 tree var
, ref
, new_var
;
4552 enum tree_code code
;
4553 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4555 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4558 var
= OMP_CLAUSE_DECL (c
);
4559 new_var
= lookup_decl (var
, ctx
);
4560 if (is_reference (var
))
4561 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4562 ref
= build_outer_var_ref (var
, ctx
);
4563 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4565 /* reduction(-:var) sums up the partial results, so it acts
4566 identically to reduction(+:var). */
4567 if (code
== MINUS_EXPR
)
4570 if (is_gimple_omp_oacc (ctx
->stmt
))
4572 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
));
4574 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
4576 else if (count
== 1)
4578 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4580 addr
= save_expr (addr
);
4581 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4582 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4583 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4584 gimplify_and_add (x
, stmt_seqp
);
4587 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4589 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4591 if (is_reference (var
)
4592 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4594 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4595 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4596 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4597 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4598 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4599 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4600 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4604 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4605 ref
= build_outer_var_ref (var
, ctx
);
4606 gimplify_assign (ref
, x
, &sub_seq
);
4610 if (is_gimple_omp_oacc (ctx
->stmt
))
4613 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4615 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4617 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4619 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4621 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4625 /* Generate code to implement the COPYPRIVATE clauses. */
4628 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4633 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4635 tree var
, new_var
, ref
, x
;
4637 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4639 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4642 var
= OMP_CLAUSE_DECL (c
);
4643 by_ref
= use_pointer_for_field (var
, NULL
);
4645 ref
= build_sender_ref (var
, ctx
);
4646 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4649 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4650 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4652 gimplify_assign (ref
, x
, slist
);
4654 ref
= build_receiver_ref (var
, false, ctx
);
4657 ref
= fold_convert_loc (clause_loc
,
4658 build_pointer_type (TREE_TYPE (new_var
)),
4660 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4662 if (is_reference (var
))
4664 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4665 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4666 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4668 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4669 gimplify_and_add (x
, rlist
);
4674 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4675 and REDUCTION from the sender (aka parent) side. */
4678 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4683 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4685 tree val
, ref
, x
, var
;
4686 bool by_ref
, do_in
= false, do_out
= false;
4687 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4689 switch (OMP_CLAUSE_CODE (c
))
4691 case OMP_CLAUSE_PRIVATE
:
4692 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4695 case OMP_CLAUSE_FIRSTPRIVATE
:
4696 case OMP_CLAUSE_COPYIN
:
4697 case OMP_CLAUSE_LASTPRIVATE
:
4698 case OMP_CLAUSE_REDUCTION
:
4699 case OMP_CLAUSE__LOOPTEMP_
:
4705 val
= OMP_CLAUSE_DECL (c
);
4706 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4708 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4709 && is_global_var (var
))
4711 if (is_variable_sized (val
))
4713 by_ref
= use_pointer_for_field (val
, NULL
);
4715 switch (OMP_CLAUSE_CODE (c
))
4717 case OMP_CLAUSE_PRIVATE
:
4718 case OMP_CLAUSE_FIRSTPRIVATE
:
4719 case OMP_CLAUSE_COPYIN
:
4720 case OMP_CLAUSE__LOOPTEMP_
:
4724 case OMP_CLAUSE_LASTPRIVATE
:
4725 if (by_ref
|| is_reference (val
))
4727 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4734 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4739 case OMP_CLAUSE_REDUCTION
:
4741 do_out
= !(by_ref
|| is_reference (val
));
4750 ref
= build_sender_ref (val
, ctx
);
4751 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4752 gimplify_assign (ref
, x
, ilist
);
4753 if (is_task_ctx (ctx
))
4754 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4759 ref
= build_sender_ref (val
, ctx
);
4760 gimplify_assign (var
, ref
, olist
);
4765 /* Generate code to implement SHARED from the sender (aka parent)
4766 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4767 list things that got automatically shared. */
4770 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4772 tree var
, ovar
, nvar
, f
, x
, record_type
;
4774 if (ctx
->record_type
== NULL
)
4777 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4778 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4780 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4781 nvar
= maybe_lookup_decl (ovar
, ctx
);
4782 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4785 /* If CTX is a nested parallel directive. Find the immediately
4786 enclosing parallel or workshare construct that contains a
4787 mapping for OVAR. */
4788 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4790 if (use_pointer_for_field (ovar
, ctx
))
4792 x
= build_sender_ref (ovar
, ctx
);
4793 var
= build_fold_addr_expr (var
);
4794 gimplify_assign (x
, var
, ilist
);
4798 x
= build_sender_ref (ovar
, ctx
);
4799 gimplify_assign (x
, var
, ilist
);
4801 if (!TREE_READONLY (var
)
4802 /* We don't need to receive a new reference to a result
4803 or parm decl. In fact we may not store to it as we will
4804 invalidate any pending RSO and generate wrong gimple
4806 && !((TREE_CODE (var
) == RESULT_DECL
4807 || TREE_CODE (var
) == PARM_DECL
)
4808 && DECL_BY_REFERENCE (var
)))
4810 x
= build_sender_ref (ovar
, ctx
);
4811 gimplify_assign (var
, x
, olist
);
4818 /* A convenience function to build an empty GIMPLE_COND with just the
4822 gimple_build_cond_empty (tree cond
)
4824 enum tree_code pred_code
;
4827 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4828 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4832 /* Build the function calls to GOMP_parallel_start etc to actually
4833 generate the parallel operation. REGION is the parallel region
4834 being expanded. BB is the block where to insert the code. WS_ARGS
4835 will be set if this is a call to a combined parallel+workshare
4836 construct, it contains the list of additional arguments needed by
4837 the workshare construct. */
4840 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4841 gomp_parallel
*entry_stmt
,
4842 vec
<tree
, va_gc
> *ws_args
)
4844 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4845 gimple_stmt_iterator gsi
;
4847 enum built_in_function start_ix
;
4849 location_t clause_loc
;
4850 vec
<tree
, va_gc
> *args
;
4852 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4854 /* Determine what flavor of GOMP_parallel we will be
4856 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4857 if (is_combined_parallel (region
))
4859 switch (region
->inner
->type
)
4861 case GIMPLE_OMP_FOR
:
4862 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4863 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4864 + (region
->inner
->sched_kind
4865 == OMP_CLAUSE_SCHEDULE_RUNTIME
4866 ? 3 : region
->inner
->sched_kind
));
4867 start_ix
= (enum built_in_function
)start_ix2
;
4869 case GIMPLE_OMP_SECTIONS
:
4870 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4877 /* By default, the value of NUM_THREADS is zero (selected at run time)
4878 and there is no conditional. */
4880 val
= build_int_cst (unsigned_type_node
, 0);
4881 flags
= build_int_cst (unsigned_type_node
, 0);
4883 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4885 cond
= OMP_CLAUSE_IF_EXPR (c
);
4887 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4890 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4891 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4894 clause_loc
= gimple_location (entry_stmt
);
4896 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4898 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4900 /* Ensure 'val' is of the correct type. */
4901 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4903 /* If we found the clause 'if (cond)', build either
4904 (cond != 0) or (cond ? val : 1u). */
4907 cond
= gimple_boolify (cond
);
4909 if (integer_zerop (val
))
4910 val
= fold_build2_loc (clause_loc
,
4911 EQ_EXPR
, unsigned_type_node
, cond
,
4912 build_int_cst (TREE_TYPE (cond
), 0));
4915 basic_block cond_bb
, then_bb
, else_bb
;
4916 edge e
, e_then
, e_else
;
4917 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4919 tmp_var
= create_tmp_var (TREE_TYPE (val
));
4920 if (gimple_in_ssa_p (cfun
))
4922 tmp_then
= make_ssa_name (tmp_var
);
4923 tmp_else
= make_ssa_name (tmp_var
);
4924 tmp_join
= make_ssa_name (tmp_var
);
4933 e
= split_block (bb
, NULL
);
4938 then_bb
= create_empty_bb (cond_bb
);
4939 else_bb
= create_empty_bb (then_bb
);
4940 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4941 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4943 stmt
= gimple_build_cond_empty (cond
);
4944 gsi
= gsi_start_bb (cond_bb
);
4945 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4947 gsi
= gsi_start_bb (then_bb
);
4948 stmt
= gimple_build_assign (tmp_then
, val
);
4949 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4951 gsi
= gsi_start_bb (else_bb
);
4952 stmt
= gimple_build_assign
4953 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4954 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4956 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4957 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4958 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4959 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4960 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4961 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4963 if (gimple_in_ssa_p (cfun
))
4965 gphi
*phi
= create_phi_node (tmp_join
, bb
);
4966 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4967 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4973 gsi
= gsi_start_bb (bb
);
4974 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4975 false, GSI_CONTINUE_LINKING
);
4978 gsi
= gsi_last_bb (bb
);
4979 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4981 t1
= null_pointer_node
;
4983 t1
= build_fold_addr_expr (t
);
4984 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4986 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4987 args
->quick_push (t2
);
4988 args
->quick_push (t1
);
4989 args
->quick_push (val
);
4991 args
->splice (*ws_args
);
4992 args
->quick_push (flags
);
4994 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4995 builtin_decl_explicit (start_ix
), args
);
4997 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4998 false, GSI_CONTINUE_LINKING
);
5001 /* Insert a function call whose name is FUNC_NAME with the information from
5002 ENTRY_STMT into the basic_block BB. */
5005 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
5006 vec
<tree
, va_gc
> *ws_args
)
5009 gimple_stmt_iterator gsi
;
5010 vec
<tree
, va_gc
> *args
;
5012 gcc_assert (vec_safe_length (ws_args
) == 2);
5013 tree func_name
= (*ws_args
)[0];
5014 tree grain
= (*ws_args
)[1];
5016 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
5017 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
5018 gcc_assert (count
!= NULL_TREE
);
5019 count
= OMP_CLAUSE_OPERAND (count
, 0);
5021 gsi
= gsi_last_bb (bb
);
5022 t
= gimple_omp_parallel_data_arg (entry_stmt
);
5024 t1
= null_pointer_node
;
5026 t1
= build_fold_addr_expr (t
);
5027 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5029 vec_alloc (args
, 4);
5030 args
->quick_push (t2
);
5031 args
->quick_push (t1
);
5032 args
->quick_push (count
);
5033 args
->quick_push (grain
);
5034 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
5036 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
5037 GSI_CONTINUE_LINKING
);
5040 /* Build the function call to GOMP_task to actually
5041 generate the task operation. BB is the block where to insert the code. */
5044 expand_task_call (basic_block bb
, gomp_task
*entry_stmt
)
5046 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
5047 gimple_stmt_iterator gsi
;
5048 location_t loc
= gimple_location (entry_stmt
);
5050 clauses
= gimple_omp_task_clauses (entry_stmt
);
5052 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
5054 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
5056 cond
= boolean_true_node
;
5058 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
5059 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
5060 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
5061 flags
= build_int_cst (unsigned_type_node
,
5062 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
5064 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
5067 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
5068 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
5069 build_int_cst (unsigned_type_node
, 2),
5070 build_int_cst (unsigned_type_node
, 0));
5071 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
5074 depend
= OMP_CLAUSE_DECL (depend
);
5076 depend
= build_int_cst (ptr_type_node
, 0);
5078 gsi
= gsi_last_bb (bb
);
5079 t
= gimple_omp_task_data_arg (entry_stmt
);
5081 t2
= null_pointer_node
;
5083 t2
= build_fold_addr_expr_loc (loc
, t
);
5084 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
5085 t
= gimple_omp_task_copy_fn (entry_stmt
);
5087 t3
= null_pointer_node
;
5089 t3
= build_fold_addr_expr_loc (loc
, t
);
5091 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
5093 gimple_omp_task_arg_size (entry_stmt
),
5094 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
5097 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5098 false, GSI_CONTINUE_LINKING
);
5102 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5103 catch handler and return it. This prevents programs from violating the
5104 structured block semantics with throws. */
5107 maybe_catch_exception (gimple_seq body
)
5112 if (!flag_exceptions
)
5115 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
5116 decl
= lang_hooks
.eh_protect_cleanup_actions ();
5118 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
5120 g
= gimple_build_eh_must_not_throw (decl
);
5121 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
5124 return gimple_seq_alloc_with_stmt (g
);
5127 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5130 vec2chain (vec
<tree
, va_gc
> *v
)
5132 tree chain
= NULL_TREE
, t
;
5135 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
5137 DECL_CHAIN (t
) = chain
;
5145 /* Remove barriers in REGION->EXIT's block. Note that this is only
5146 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5147 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5148 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5152 remove_exit_barrier (struct omp_region
*region
)
5154 gimple_stmt_iterator gsi
;
5155 basic_block exit_bb
;
5159 int any_addressable_vars
= -1;
5161 exit_bb
= region
->exit
;
5163 /* If the parallel region doesn't return, we don't have REGION->EXIT
5168 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5169 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5170 statements that can appear in between are extremely limited -- no
5171 memory operations at all. Here, we allow nothing at all, so the
5172 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5173 gsi
= gsi_last_bb (exit_bb
);
5174 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5176 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
5179 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
5181 gsi
= gsi_last_bb (e
->src
);
5182 if (gsi_end_p (gsi
))
5184 stmt
= gsi_stmt (gsi
);
5185 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
5186 && !gimple_omp_return_nowait_p (stmt
))
5188 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5189 in many cases. If there could be tasks queued, the barrier
5190 might be needed to let the tasks run before some local
5191 variable of the parallel that the task uses as shared
5192 runs out of scope. The task can be spawned either
5193 from within current function (this would be easy to check)
5194 or from some function it calls and gets passed an address
5195 of such a variable. */
5196 if (any_addressable_vars
< 0)
5198 gomp_parallel
*parallel_stmt
5199 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
5200 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
5201 tree local_decls
, block
, decl
;
5204 any_addressable_vars
= 0;
5205 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
5206 if (TREE_ADDRESSABLE (decl
))
5208 any_addressable_vars
= 1;
5211 for (block
= gimple_block (stmt
);
5212 !any_addressable_vars
5214 && TREE_CODE (block
) == BLOCK
;
5215 block
= BLOCK_SUPERCONTEXT (block
))
5217 for (local_decls
= BLOCK_VARS (block
);
5219 local_decls
= DECL_CHAIN (local_decls
))
5220 if (TREE_ADDRESSABLE (local_decls
))
5222 any_addressable_vars
= 1;
5225 if (block
== gimple_block (parallel_stmt
))
5229 if (!any_addressable_vars
)
5230 gimple_omp_return_set_nowait (stmt
);
5236 remove_exit_barriers (struct omp_region
*region
)
5238 if (region
->type
== GIMPLE_OMP_PARALLEL
)
5239 remove_exit_barrier (region
);
5243 region
= region
->inner
;
5244 remove_exit_barriers (region
);
5245 while (region
->next
)
5247 region
= region
->next
;
5248 remove_exit_barriers (region
);
5253 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5254 calls. These can't be declared as const functions, but
5255 within one parallel body they are constant, so they can be
5256 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5257 which are declared const. Similarly for task body, except
5258 that in untied task omp_get_thread_num () can change at any task
5259 scheduling point. */
5262 optimize_omp_library_calls (gimple entry_stmt
)
5265 gimple_stmt_iterator gsi
;
5266 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5267 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
5268 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5269 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
5270 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
5271 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
5272 OMP_CLAUSE_UNTIED
) != NULL
);
5274 FOR_EACH_BB_FN (bb
, cfun
)
5275 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5277 gimple call
= gsi_stmt (gsi
);
5280 if (is_gimple_call (call
)
5281 && (decl
= gimple_call_fndecl (call
))
5282 && DECL_EXTERNAL (decl
)
5283 && TREE_PUBLIC (decl
)
5284 && DECL_INITIAL (decl
) == NULL
)
5288 if (DECL_NAME (decl
) == thr_num_id
)
5290 /* In #pragma omp task untied omp_get_thread_num () can change
5291 during the execution of the task region. */
5294 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5296 else if (DECL_NAME (decl
) == num_thr_id
)
5297 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5301 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
5302 || gimple_call_num_args (call
) != 0)
5305 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
5308 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
5309 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
5310 TREE_TYPE (TREE_TYPE (built_in
))))
5313 gimple_call_set_fndecl (call
, built_in
);
5318 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5322 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
5326 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5327 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
5330 if (TREE_CODE (t
) == ADDR_EXPR
)
5331 recompute_tree_invariant_for_addr_expr (t
);
5333 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
5337 /* Prepend TO = FROM assignment before *GSI_P. */
5340 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
5342 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
5343 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
5344 true, GSI_SAME_STMT
);
5345 gimple stmt
= gimple_build_assign (to
, from
);
5346 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
5347 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
5348 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
5350 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
5351 gimple_regimplify_operands (stmt
, &gsi
);
5355 /* Expand the OpenMP parallel or task directive starting at REGION. */
5358 expand_omp_taskreg (struct omp_region
*region
)
5360 basic_block entry_bb
, exit_bb
, new_bb
;
5361 struct function
*child_cfun
;
5362 tree child_fn
, block
, t
;
5363 gimple_stmt_iterator gsi
;
5364 gimple entry_stmt
, stmt
;
5366 vec
<tree
, va_gc
> *ws_args
;
5368 entry_stmt
= last_stmt (region
->entry
);
5369 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
5370 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
5372 entry_bb
= region
->entry
;
5373 exit_bb
= region
->exit
;
5377 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
5378 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
5379 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
5382 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5383 and the inner statement contains the name of the built-in function
5385 ws_args
= region
->inner
->ws_args
;
5386 else if (is_combined_parallel (region
))
5387 ws_args
= region
->ws_args
;
5391 if (child_cfun
->cfg
)
5393 /* Due to inlining, it may happen that we have already outlined
5394 the region, in which case all we need to do is make the
5395 sub-graph unreachable and emit the parallel call. */
5396 edge entry_succ_e
, exit_succ_e
;
5398 entry_succ_e
= single_succ_edge (entry_bb
);
5400 gsi
= gsi_last_bb (entry_bb
);
5401 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
5402 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
5403 gsi_remove (&gsi
, true);
5408 exit_succ_e
= single_succ_edge (exit_bb
);
5409 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
5411 remove_edge_and_dominated_blocks (entry_succ_e
);
5415 unsigned srcidx
, dstidx
, num
;
5417 /* If the parallel region needs data sent from the parent
5418 function, then the very first statement (except possible
5419 tree profile counter updates) of the parallel body
5420 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5421 &.OMP_DATA_O is passed as an argument to the child function,
5422 we need to replace it with the argument as seen by the child
5425 In most cases, this will end up being the identity assignment
5426 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5427 a function call that has been inlined, the original PARM_DECL
5428 .OMP_DATA_I may have been converted into a different local
5429 variable. In which case, we need to keep the assignment. */
5430 if (gimple_omp_taskreg_data_arg (entry_stmt
))
5432 basic_block entry_succ_bb
= single_succ (entry_bb
);
5434 gimple parcopy_stmt
= NULL
;
5436 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
5440 gcc_assert (!gsi_end_p (gsi
));
5441 stmt
= gsi_stmt (gsi
);
5442 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
5445 if (gimple_num_ops (stmt
) == 2)
5447 tree arg
= gimple_assign_rhs1 (stmt
);
5449 /* We're ignore the subcode because we're
5450 effectively doing a STRIP_NOPS. */
5452 if (TREE_CODE (arg
) == ADDR_EXPR
5453 && TREE_OPERAND (arg
, 0)
5454 == gimple_omp_taskreg_data_arg (entry_stmt
))
5456 parcopy_stmt
= stmt
;
5462 gcc_assert (parcopy_stmt
!= NULL
);
5463 arg
= DECL_ARGUMENTS (child_fn
);
5465 if (!gimple_in_ssa_p (cfun
))
5467 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5468 gsi_remove (&gsi
, true);
5471 /* ?? Is setting the subcode really necessary ?? */
5472 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5473 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5478 /* If we are in ssa form, we must load the value from the default
5479 definition of the argument. That should not be defined now,
5480 since the argument is not used uninitialized. */
5481 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5482 narg
= make_ssa_name (arg
, gimple_build_nop ());
5483 set_ssa_default_def (cfun
, arg
, narg
);
5484 /* ?? Is setting the subcode really necessary ?? */
5485 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5486 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5487 update_stmt (parcopy_stmt
);
5491 /* Declare local variables needed in CHILD_CFUN. */
5492 block
= DECL_INITIAL (child_fn
);
5493 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5494 /* The gimplifier could record temporaries in parallel/task block
5495 rather than in containing function's local_decls chain,
5496 which would mean cgraph missed finalizing them. Do it now. */
5497 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5498 if (TREE_CODE (t
) == VAR_DECL
5500 && !DECL_EXTERNAL (t
))
5501 varpool_node::finalize_decl (t
);
5502 DECL_SAVED_TREE (child_fn
) = NULL
;
5503 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5504 gimple_set_body (child_fn
, NULL
);
5505 TREE_USED (block
) = 1;
5507 /* Reset DECL_CONTEXT on function arguments. */
5508 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5509 DECL_CONTEXT (t
) = child_fn
;
5511 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5512 so that it can be moved to the child function. */
5513 gsi
= gsi_last_bb (entry_bb
);
5514 stmt
= gsi_stmt (gsi
);
5515 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5516 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5517 gsi_remove (&gsi
, true);
5518 e
= split_block (entry_bb
, stmt
);
5520 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5522 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5525 gsi
= gsi_last_bb (exit_bb
);
5526 gcc_assert (!gsi_end_p (gsi
)
5527 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5528 stmt
= gimple_build_return (NULL
);
5529 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5530 gsi_remove (&gsi
, true);
5533 /* Move the parallel region into CHILD_CFUN. */
5535 if (gimple_in_ssa_p (cfun
))
5537 init_tree_ssa (child_cfun
);
5538 init_ssa_operands (child_cfun
);
5539 child_cfun
->gimple_df
->in_ssa_p
= true;
5543 block
= gimple_block (entry_stmt
);
5545 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5547 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5548 /* When the OMP expansion process cannot guarantee an up-to-date
5549 loop tree arrange for the child function to fixup loops. */
5550 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5551 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5553 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5554 num
= vec_safe_length (child_cfun
->local_decls
);
5555 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5557 t
= (*child_cfun
->local_decls
)[srcidx
];
5558 if (DECL_CONTEXT (t
) == cfun
->decl
)
5560 if (srcidx
!= dstidx
)
5561 (*child_cfun
->local_decls
)[dstidx
] = t
;
5565 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5567 /* Inform the callgraph about the new function. */
5568 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5569 cgraph_node::add_new_function (child_fn
, true);
5571 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5572 fixed in a following pass. */
5573 push_cfun (child_cfun
);
5575 optimize_omp_library_calls (entry_stmt
);
5576 cgraph_edge::rebuild_edges ();
5578 /* Some EH regions might become dead, see PR34608. If
5579 pass_cleanup_cfg isn't the first pass to happen with the
5580 new child, these dead EH edges might cause problems.
5581 Clean them up now. */
5582 if (flag_exceptions
)
5585 bool changed
= false;
5587 FOR_EACH_BB_FN (bb
, cfun
)
5588 changed
|= gimple_purge_dead_eh_edges (bb
);
5590 cleanup_tree_cfg ();
5592 if (gimple_in_ssa_p (cfun
))
5593 update_ssa (TODO_update_ssa
);
5597 /* Emit a library call to launch the children threads. */
5599 expand_cilk_for_call (new_bb
,
5600 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5601 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5602 expand_parallel_call (region
, new_bb
,
5603 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5605 expand_task_call (new_bb
, as_a
<gomp_task
*> (entry_stmt
));
5606 if (gimple_in_ssa_p (cfun
))
5607 update_ssa (TODO_update_ssa_only_virtuals
);
5611 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5612 of the combined collapse > 1 loop constructs, generate code like:
5613 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5618 count3 = (adj + N32 - N31) / STEP3;
5619 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5624 count2 = (adj + N22 - N21) / STEP2;
5625 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5630 count1 = (adj + N12 - N11) / STEP1;
5631 count = count1 * count2 * count3;
5632 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5634 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5635 of the combined loop constructs, just initialize COUNTS array
5636 from the _looptemp_ clauses. */
5638 /* NOTE: It *could* be better to moosh all of the BBs together,
5639 creating one larger BB with all the computation and the unexpected
5640 jump at the end. I.e.
5642 bool zero3, zero2, zero1, zero;
5645 count3 = (N32 - N31) /[cl] STEP3;
5647 count2 = (N22 - N21) /[cl] STEP2;
5649 count1 = (N12 - N11) /[cl] STEP1;
5650 zero = zero3 || zero2 || zero1;
5651 count = count1 * count2 * count3;
5652 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5654 After all, we expect the zero=false, and thus we expect to have to
5655 evaluate all of the comparison expressions, so short-circuiting
5656 oughtn't be a win. Since the condition isn't protecting a
5657 denominator, we're not concerned about divide-by-zero, so we can
5658 fully evaluate count even if a numerator turned out to be wrong.
5660 It seems like putting this all together would create much better
5661 scheduling opportunities, and less pressure on the chip's branch
5665 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5666 basic_block
&entry_bb
, tree
*counts
,
5667 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5668 basic_block
&l2_dom_bb
)
5670 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5674 /* Collapsed loops need work for expansion into SSA form. */
5675 gcc_assert (!gimple_in_ssa_p (cfun
));
5677 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5678 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5680 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5681 isn't supposed to be handled, as the inner loop doesn't
5683 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5684 OMP_CLAUSE__LOOPTEMP_
);
5685 gcc_assert (innerc
);
5686 for (i
= 0; i
< fd
->collapse
; i
++)
5688 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5689 OMP_CLAUSE__LOOPTEMP_
);
5690 gcc_assert (innerc
);
5692 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5694 counts
[0] = NULL_TREE
;
5699 for (i
= 0; i
< fd
->collapse
; i
++)
5701 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5703 if (SSA_VAR_P (fd
->loop
.n2
)
5704 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5705 fold_convert (itype
, fd
->loops
[i
].n1
),
5706 fold_convert (itype
, fd
->loops
[i
].n2
)))
5707 == NULL_TREE
|| !integer_onep (t
)))
5711 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5712 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5713 true, GSI_SAME_STMT
);
5714 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5715 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5716 true, GSI_SAME_STMT
);
5717 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5718 NULL_TREE
, NULL_TREE
);
5719 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
5720 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
5721 expand_omp_regimplify_p
, NULL
, NULL
)
5722 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
5723 expand_omp_regimplify_p
, NULL
, NULL
))
5725 *gsi
= gsi_for_stmt (cond_stmt
);
5726 gimple_regimplify_operands (cond_stmt
, gsi
);
5728 e
= split_block (entry_bb
, cond_stmt
);
5729 if (zero_iter_bb
== NULL
)
5731 gassign
*assign_stmt
;
5732 first_zero_iter
= i
;
5733 zero_iter_bb
= create_empty_bb (entry_bb
);
5734 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5735 *gsi
= gsi_after_labels (zero_iter_bb
);
5736 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
5737 build_zero_cst (type
));
5738 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
5739 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5742 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5743 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5744 e
->flags
= EDGE_TRUE_VALUE
;
5745 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5746 if (l2_dom_bb
== NULL
)
5747 l2_dom_bb
= entry_bb
;
5749 *gsi
= gsi_last_bb (entry_bb
);
5752 if (POINTER_TYPE_P (itype
))
5753 itype
= signed_type_for (itype
);
5754 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5756 t
= fold_build2 (PLUS_EXPR
, itype
,
5757 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5758 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5759 fold_convert (itype
, fd
->loops
[i
].n2
));
5760 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5761 fold_convert (itype
, fd
->loops
[i
].n1
));
5762 /* ?? We could probably use CEIL_DIV_EXPR instead of
5763 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5764 generate the same code in the end because generically we
5765 don't know that the values involved must be negative for
5767 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5768 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5769 fold_build1 (NEGATE_EXPR
, itype
, t
),
5770 fold_build1 (NEGATE_EXPR
, itype
,
5771 fold_convert (itype
,
5772 fd
->loops
[i
].step
)));
5774 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5775 fold_convert (itype
, fd
->loops
[i
].step
));
5776 t
= fold_convert (type
, t
);
5777 if (TREE_CODE (t
) == INTEGER_CST
)
5781 counts
[i
] = create_tmp_reg (type
, ".count");
5782 expand_omp_build_assign (gsi
, counts
[i
], t
);
5784 if (SSA_VAR_P (fd
->loop
.n2
))
5789 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5790 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5796 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5798 V3 = N31 + (T % count3) * STEP3;
5800 V2 = N21 + (T % count2) * STEP2;
5802 V1 = N11 + T * STEP1;
5803 if this loop doesn't have an inner loop construct combined with it.
5804 If it does have an inner loop construct combined with it and the
5805 iteration count isn't known constant, store values from counts array
5806 into its _looptemp_ temporaries instead. */
5809 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5810 tree
*counts
, gimple inner_stmt
, tree startvar
)
5813 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5815 /* If fd->loop.n2 is constant, then no propagation of the counts
5816 is needed, they are constant. */
5817 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5820 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5821 ? gimple_omp_parallel_clauses (inner_stmt
)
5822 : gimple_omp_for_clauses (inner_stmt
);
5823 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5824 isn't supposed to be handled, as the inner loop doesn't
5826 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5827 gcc_assert (innerc
);
5828 for (i
= 0; i
< fd
->collapse
; i
++)
5830 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5831 OMP_CLAUSE__LOOPTEMP_
);
5832 gcc_assert (innerc
);
5835 tree tem
= OMP_CLAUSE_DECL (innerc
);
5836 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5837 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5838 false, GSI_CONTINUE_LINKING
);
5839 gassign
*stmt
= gimple_build_assign (tem
, t
);
5840 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5846 tree type
= TREE_TYPE (fd
->loop
.v
);
5847 tree tem
= create_tmp_reg (type
, ".tem");
5848 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
5849 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5851 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5853 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5855 if (POINTER_TYPE_P (vtype
))
5856 itype
= signed_type_for (vtype
);
5858 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5861 t
= fold_convert (itype
, t
);
5862 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5863 fold_convert (itype
, fd
->loops
[i
].step
));
5864 if (POINTER_TYPE_P (vtype
))
5865 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5867 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5868 t
= force_gimple_operand_gsi (gsi
, t
,
5869 DECL_P (fd
->loops
[i
].v
)
5870 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5872 GSI_CONTINUE_LINKING
);
5873 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5874 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5877 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5878 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5879 false, GSI_CONTINUE_LINKING
);
5880 stmt
= gimple_build_assign (tem
, t
);
5881 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5887 /* Helper function for expand_omp_for_*. Generate code like:
5890 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5894 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5901 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5902 basic_block body_bb
)
5904 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5906 gimple_stmt_iterator gsi
;
5912 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5914 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5916 bb
= create_empty_bb (last_bb
);
5917 add_bb_to_loop (bb
, last_bb
->loop_father
);
5918 gsi
= gsi_start_bb (bb
);
5920 if (i
< fd
->collapse
- 1)
5922 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5923 e
->probability
= REG_BR_PROB_BASE
/ 8;
5925 t
= fd
->loops
[i
+ 1].n1
;
5926 t
= force_gimple_operand_gsi (&gsi
, t
,
5927 DECL_P (fd
->loops
[i
+ 1].v
)
5928 && TREE_ADDRESSABLE (fd
->loops
[i
5931 GSI_CONTINUE_LINKING
);
5932 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5933 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5938 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5940 if (POINTER_TYPE_P (vtype
))
5941 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5943 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5944 t
= force_gimple_operand_gsi (&gsi
, t
,
5945 DECL_P (fd
->loops
[i
].v
)
5946 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5947 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5948 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5949 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5953 t
= fd
->loops
[i
].n2
;
5954 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5955 false, GSI_CONTINUE_LINKING
);
5956 tree v
= fd
->loops
[i
].v
;
5957 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5958 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5959 false, GSI_CONTINUE_LINKING
);
5960 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5961 stmt
= gimple_build_cond_empty (t
);
5962 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5963 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5964 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5967 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5975 /* A subroutine of expand_omp_for. Generate code for a parallel
5976 loop with any schedule. Given parameters:
5978 for (V = N1; V cond N2; V += STEP) BODY;
5980 where COND is "<" or ">", we generate pseudocode
5982 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5983 if (more) goto L0; else goto L3;
5990 if (V cond iend) goto L1; else goto L2;
5992 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5995 If this is a combined omp parallel loop, instead of the call to
5996 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5997 If this is gimple_omp_for_combined_p loop, then instead of assigning
5998 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5999 inner GIMPLE_OMP_FOR and V += STEP; and
6000 if (V cond iend) goto L1; else goto L2; are removed.
6002 For collapsed loops, given parameters:
6004 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6005 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6006 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6009 we generate pseudocode
6011 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6016 count3 = (adj + N32 - N31) / STEP3;
6017 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6022 count2 = (adj + N22 - N21) / STEP2;
6023 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6028 count1 = (adj + N12 - N11) / STEP1;
6029 count = count1 * count2 * count3;
6034 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6035 if (more) goto L0; else goto L3;
6039 V3 = N31 + (T % count3) * STEP3;
6041 V2 = N21 + (T % count2) * STEP2;
6043 V1 = N11 + T * STEP1;
6048 if (V < iend) goto L10; else goto L2;
6051 if (V3 cond3 N32) goto L1; else goto L11;
6055 if (V2 cond2 N22) goto L1; else goto L12;
6061 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6067 expand_omp_for_generic (struct omp_region
*region
,
6068 struct omp_for_data
*fd
,
6069 enum built_in_function start_fn
,
6070 enum built_in_function next_fn
,
6073 tree type
, istart0
, iend0
, iend
;
6074 tree t
, vmain
, vback
, bias
= NULL_TREE
;
6075 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
6076 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
6077 gimple_stmt_iterator gsi
;
6078 gassign
*assign_stmt
;
6079 bool in_combined_parallel
= is_combined_parallel (region
);
6080 bool broken_loop
= region
->cont
== NULL
;
6082 tree
*counts
= NULL
;
6085 gcc_assert (!broken_loop
|| !in_combined_parallel
);
6086 gcc_assert (fd
->iter_type
== long_integer_type_node
6087 || !in_combined_parallel
);
6089 type
= TREE_TYPE (fd
->loop
.v
);
6090 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
6091 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
6092 TREE_ADDRESSABLE (istart0
) = 1;
6093 TREE_ADDRESSABLE (iend0
) = 1;
6095 /* See if we need to bias by LLONG_MIN. */
6096 if (fd
->iter_type
== long_long_unsigned_type_node
6097 && TREE_CODE (type
) == INTEGER_TYPE
6098 && !TYPE_UNSIGNED (type
))
6102 if (fd
->loop
.cond_code
== LT_EXPR
)
6105 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6109 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6112 if (TREE_CODE (n1
) != INTEGER_CST
6113 || TREE_CODE (n2
) != INTEGER_CST
6114 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
6115 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
6118 entry_bb
= region
->entry
;
6119 cont_bb
= region
->cont
;
6121 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6122 gcc_assert (broken_loop
6123 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6124 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6125 l1_bb
= single_succ (l0_bb
);
6128 l2_bb
= create_empty_bb (cont_bb
);
6129 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
6130 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6134 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6135 exit_bb
= region
->exit
;
6137 gsi
= gsi_last_bb (entry_bb
);
6139 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6140 if (fd
->collapse
> 1)
6142 int first_zero_iter
= -1;
6143 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
6145 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6146 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6147 zero_iter_bb
, first_zero_iter
,
6152 /* Some counts[i] vars might be uninitialized if
6153 some loop has zero iterations. But the body shouldn't
6154 be executed in that case, so just avoid uninit warnings. */
6155 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
6156 if (SSA_VAR_P (counts
[i
]))
6157 TREE_NO_WARNING (counts
[i
]) = 1;
6159 e
= split_block (entry_bb
, gsi_stmt (gsi
));
6161 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
6162 gsi
= gsi_last_bb (entry_bb
);
6163 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
6164 get_immediate_dominator (CDI_DOMINATORS
,
6168 if (in_combined_parallel
)
6170 /* In a combined parallel loop, emit a call to
6171 GOMP_loop_foo_next. */
6172 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6173 build_fold_addr_expr (istart0
),
6174 build_fold_addr_expr (iend0
));
6178 tree t0
, t1
, t2
, t3
, t4
;
6179 /* If this is not a combined parallel loop, emit a call to
6180 GOMP_loop_foo_start in ENTRY_BB. */
6181 t4
= build_fold_addr_expr (iend0
);
6182 t3
= build_fold_addr_expr (istart0
);
6183 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
6186 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6188 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6189 OMP_CLAUSE__LOOPTEMP_
);
6190 gcc_assert (innerc
);
6191 t0
= OMP_CLAUSE_DECL (innerc
);
6192 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6193 OMP_CLAUSE__LOOPTEMP_
);
6194 gcc_assert (innerc
);
6195 t1
= OMP_CLAUSE_DECL (innerc
);
6197 if (POINTER_TYPE_P (TREE_TYPE (t0
))
6198 && TYPE_PRECISION (TREE_TYPE (t0
))
6199 != TYPE_PRECISION (fd
->iter_type
))
6201 /* Avoid casting pointers to integer of a different size. */
6202 tree itype
= signed_type_for (type
);
6203 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
6204 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
6208 t1
= fold_convert (fd
->iter_type
, t1
);
6209 t0
= fold_convert (fd
->iter_type
, t0
);
6213 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
6214 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
6216 if (fd
->iter_type
== long_integer_type_node
)
6220 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6221 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6222 6, t0
, t1
, t2
, t
, t3
, t4
);
6225 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6226 5, t0
, t1
, t2
, t3
, t4
);
6234 /* The GOMP_loop_ull_*start functions have additional boolean
6235 argument, true for < loops and false for > loops.
6236 In Fortran, the C bool type can be different from
6237 boolean_type_node. */
6238 bfn_decl
= builtin_decl_explicit (start_fn
);
6239 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
6240 t5
= build_int_cst (c_bool_type
,
6241 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
6244 tree bfn_decl
= builtin_decl_explicit (start_fn
);
6245 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6246 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
6249 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6250 6, t5
, t0
, t1
, t2
, t3
, t4
);
6253 if (TREE_TYPE (t
) != boolean_type_node
)
6254 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6255 t
, build_int_cst (TREE_TYPE (t
), 0));
6256 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6257 true, GSI_SAME_STMT
);
6258 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6260 /* Remove the GIMPLE_OMP_FOR statement. */
6261 gsi_remove (&gsi
, true);
6263 /* Iteration setup for sequential loop goes in L0_BB. */
6264 tree startvar
= fd
->loop
.v
;
6265 tree endvar
= NULL_TREE
;
6267 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6269 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
6270 && gimple_omp_for_kind (inner_stmt
)
6271 == GF_OMP_FOR_KIND_SIMD
);
6272 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
6273 OMP_CLAUSE__LOOPTEMP_
);
6274 gcc_assert (innerc
);
6275 startvar
= OMP_CLAUSE_DECL (innerc
);
6276 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6277 OMP_CLAUSE__LOOPTEMP_
);
6278 gcc_assert (innerc
);
6279 endvar
= OMP_CLAUSE_DECL (innerc
);
6282 gsi
= gsi_start_bb (l0_bb
);
6285 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6286 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6287 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6288 t
= fold_convert (TREE_TYPE (startvar
), t
);
6289 t
= force_gimple_operand_gsi (&gsi
, t
,
6291 && TREE_ADDRESSABLE (startvar
),
6292 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6293 assign_stmt
= gimple_build_assign (startvar
, t
);
6294 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6298 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6299 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6300 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6301 t
= fold_convert (TREE_TYPE (startvar
), t
);
6302 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6303 false, GSI_CONTINUE_LINKING
);
6306 assign_stmt
= gimple_build_assign (endvar
, iend
);
6307 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6308 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
6309 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
6311 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
6312 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6314 if (fd
->collapse
> 1)
6315 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6319 /* Code to control the increment and predicate for the sequential
6320 loop goes in the CONT_BB. */
6321 gsi
= gsi_last_bb (cont_bb
);
6322 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6323 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6324 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6325 vback
= gimple_omp_continue_control_def (cont_stmt
);
6327 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6329 if (POINTER_TYPE_P (type
))
6330 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
6332 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
6333 t
= force_gimple_operand_gsi (&gsi
, t
,
6335 && TREE_ADDRESSABLE (vback
),
6336 NULL_TREE
, true, GSI_SAME_STMT
);
6337 assign_stmt
= gimple_build_assign (vback
, t
);
6338 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6340 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6341 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
6343 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6344 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6347 /* Remove GIMPLE_OMP_CONTINUE. */
6348 gsi_remove (&gsi
, true);
6350 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6351 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
6353 /* Emit code to get the next parallel iteration in L2_BB. */
6354 gsi
= gsi_start_bb (l2_bb
);
6356 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6357 build_fold_addr_expr (istart0
),
6358 build_fold_addr_expr (iend0
));
6359 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6360 false, GSI_CONTINUE_LINKING
);
6361 if (TREE_TYPE (t
) != boolean_type_node
)
6362 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6363 t
, build_int_cst (TREE_TYPE (t
), 0));
6364 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6365 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
6368 /* Add the loop cleanup function. */
6369 gsi
= gsi_last_bb (exit_bb
);
6370 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6371 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
6372 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6373 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
6375 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
6376 gcall
*call_stmt
= gimple_build_call (t
, 0);
6377 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6378 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
6379 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
6380 gsi_remove (&gsi
, true);
6382 /* Connect the new blocks. */
6383 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
6384 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
6390 e
= find_edge (cont_bb
, l3_bb
);
6391 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
6393 phis
= phi_nodes (l3_bb
);
6394 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6396 gimple phi
= gsi_stmt (gsi
);
6397 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
6398 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
6402 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
6403 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
6404 e
= find_edge (cont_bb
, l1_bb
);
6405 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6410 else if (fd
->collapse
> 1)
6413 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6416 e
->flags
= EDGE_TRUE_VALUE
;
6419 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6420 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
6424 e
= find_edge (cont_bb
, l2_bb
);
6425 e
->flags
= EDGE_FALLTHRU
;
6427 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6429 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
6430 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
6431 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
6432 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
6433 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
6434 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
6435 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
6436 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
6438 struct loop
*outer_loop
= alloc_loop ();
6439 outer_loop
->header
= l0_bb
;
6440 outer_loop
->latch
= l2_bb
;
6441 add_loop (outer_loop
, l0_bb
->loop_father
);
6443 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6445 struct loop
*loop
= alloc_loop ();
6446 loop
->header
= l1_bb
;
6447 /* The loop may have multiple latches. */
6448 add_loop (loop
, outer_loop
);
6454 /* A subroutine of expand_omp_for. Generate code for a parallel
6455 loop with static schedule and no specified chunk size. Given
6458 for (V = N1; V cond N2; V += STEP) BODY;
6460 where COND is "<" or ">", we generate pseudocode
6462 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6467 if ((__typeof (V)) -1 > 0 && cond is >)
6468 n = -(adj + N2 - N1) / -STEP;
6470 n = (adj + N2 - N1) / STEP;
6473 if (threadid < tt) goto L3; else goto L4;
6478 s0 = q * threadid + tt;
6481 if (s0 >= e0) goto L2; else goto L0;
6487 if (V cond e) goto L1;
6492 expand_omp_for_static_nochunk (struct omp_region
*region
,
6493 struct omp_for_data
*fd
,
6496 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6497 tree type
, itype
, vmain
, vback
;
6498 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6499 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6501 gimple_stmt_iterator gsi
;
6503 bool broken_loop
= region
->cont
== NULL
;
6504 tree
*counts
= NULL
;
6507 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6508 != GF_OMP_FOR_KIND_OACC_LOOP
)
6511 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6512 if (POINTER_TYPE_P (type
))
6513 itype
= signed_type_for (type
);
6515 entry_bb
= region
->entry
;
6516 cont_bb
= region
->cont
;
6517 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6518 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6519 gcc_assert (broken_loop
6520 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6521 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6522 body_bb
= single_succ (seq_start_bb
);
6525 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6526 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6528 exit_bb
= region
->exit
;
6530 /* Iteration space partitioning goes in ENTRY_BB. */
6531 gsi
= gsi_last_bb (entry_bb
);
6532 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6534 if (fd
->collapse
> 1)
6536 int first_zero_iter
= -1;
6537 basic_block l2_dom_bb
= NULL
;
6539 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6540 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6541 fin_bb
, first_zero_iter
,
6545 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6546 t
= integer_one_node
;
6548 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6549 fold_convert (type
, fd
->loop
.n1
),
6550 fold_convert (type
, fd
->loop
.n2
));
6551 if (fd
->collapse
== 1
6552 && TYPE_UNSIGNED (type
)
6553 && (t
== NULL_TREE
|| !integer_onep (t
)))
6555 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6556 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6557 true, GSI_SAME_STMT
);
6558 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6559 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6560 true, GSI_SAME_STMT
);
6561 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6562 NULL_TREE
, NULL_TREE
);
6563 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6564 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6565 expand_omp_regimplify_p
, NULL
, NULL
)
6566 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6567 expand_omp_regimplify_p
, NULL
, NULL
))
6569 gsi
= gsi_for_stmt (cond_stmt
);
6570 gimple_regimplify_operands (cond_stmt
, &gsi
);
6572 ep
= split_block (entry_bb
, cond_stmt
);
6573 ep
->flags
= EDGE_TRUE_VALUE
;
6574 entry_bb
= ep
->dest
;
6575 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6576 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6577 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6578 if (gimple_in_ssa_p (cfun
))
6580 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6581 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6582 !gsi_end_p (gpi
); gsi_next (&gpi
))
6584 gphi
*phi
= gpi
.phi ();
6585 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6586 ep
, UNKNOWN_LOCATION
);
6589 gsi
= gsi_last_bb (entry_bb
);
6592 switch (gimple_omp_for_kind (fd
->for_stmt
))
6594 case GF_OMP_FOR_KIND_FOR
:
6595 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6596 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6598 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6599 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6600 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6602 case GF_OMP_FOR_KIND_OACC_LOOP
:
6603 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6604 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
6609 nthreads
= build_call_expr (nthreads
, 0);
6610 nthreads
= fold_convert (itype
, nthreads
);
6611 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
6612 true, GSI_SAME_STMT
);
6613 threadid
= build_call_expr (threadid
, 0);
6614 threadid
= fold_convert (itype
, threadid
);
6615 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
6616 true, GSI_SAME_STMT
);
6620 step
= fd
->loop
.step
;
6621 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6623 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6624 OMP_CLAUSE__LOOPTEMP_
);
6625 gcc_assert (innerc
);
6626 n1
= OMP_CLAUSE_DECL (innerc
);
6627 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6628 OMP_CLAUSE__LOOPTEMP_
);
6629 gcc_assert (innerc
);
6630 n2
= OMP_CLAUSE_DECL (innerc
);
6632 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6633 true, NULL_TREE
, true, GSI_SAME_STMT
);
6634 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6635 true, NULL_TREE
, true, GSI_SAME_STMT
);
6636 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6637 true, NULL_TREE
, true, GSI_SAME_STMT
);
6639 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6640 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6641 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6642 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6643 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6644 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6645 fold_build1 (NEGATE_EXPR
, itype
, t
),
6646 fold_build1 (NEGATE_EXPR
, itype
, step
));
6648 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6649 t
= fold_convert (itype
, t
);
6650 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6652 q
= create_tmp_reg (itype
, "q");
6653 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6654 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6655 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6657 tt
= create_tmp_reg (itype
, "tt");
6658 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6659 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6660 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6662 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6663 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6664 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6666 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
6667 gsi
= gsi_last_bb (second_bb
);
6668 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6670 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6672 gassign
*assign_stmt
6673 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
6674 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6676 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
6677 gsi
= gsi_last_bb (third_bb
);
6678 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6680 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6681 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6682 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6684 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6685 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6687 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6688 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6690 /* Remove the GIMPLE_OMP_FOR statement. */
6691 gsi_remove (&gsi
, true);
6693 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6694 gsi
= gsi_start_bb (seq_start_bb
);
6696 tree startvar
= fd
->loop
.v
;
6697 tree endvar
= NULL_TREE
;
6699 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6701 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6702 ? gimple_omp_parallel_clauses (inner_stmt
)
6703 : gimple_omp_for_clauses (inner_stmt
);
6704 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6705 gcc_assert (innerc
);
6706 startvar
= OMP_CLAUSE_DECL (innerc
);
6707 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6708 OMP_CLAUSE__LOOPTEMP_
);
6709 gcc_assert (innerc
);
6710 endvar
= OMP_CLAUSE_DECL (innerc
);
6712 t
= fold_convert (itype
, s0
);
6713 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6714 if (POINTER_TYPE_P (type
))
6715 t
= fold_build_pointer_plus (n1
, t
);
6717 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6718 t
= fold_convert (TREE_TYPE (startvar
), t
);
6719 t
= force_gimple_operand_gsi (&gsi
, t
,
6721 && TREE_ADDRESSABLE (startvar
),
6722 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6723 assign_stmt
= gimple_build_assign (startvar
, t
);
6724 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6726 t
= fold_convert (itype
, e0
);
6727 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6728 if (POINTER_TYPE_P (type
))
6729 t
= fold_build_pointer_plus (n1
, t
);
6731 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6732 t
= fold_convert (TREE_TYPE (startvar
), t
);
6733 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6734 false, GSI_CONTINUE_LINKING
);
6737 assign_stmt
= gimple_build_assign (endvar
, e
);
6738 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6739 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6740 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6742 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
6743 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6745 if (fd
->collapse
> 1)
6746 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6750 /* The code controlling the sequential loop replaces the
6751 GIMPLE_OMP_CONTINUE. */
6752 gsi
= gsi_last_bb (cont_bb
);
6753 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6754 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6755 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6756 vback
= gimple_omp_continue_control_def (cont_stmt
);
6758 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6760 if (POINTER_TYPE_P (type
))
6761 t
= fold_build_pointer_plus (vmain
, step
);
6763 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6764 t
= force_gimple_operand_gsi (&gsi
, t
,
6766 && TREE_ADDRESSABLE (vback
),
6767 NULL_TREE
, true, GSI_SAME_STMT
);
6768 assign_stmt
= gimple_build_assign (vback
, t
);
6769 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6771 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6772 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6774 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6777 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6778 gsi_remove (&gsi
, true);
6780 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6781 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6784 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6785 gsi
= gsi_last_bb (exit_bb
);
6786 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6788 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6789 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
6790 gcc_checking_assert (t
== NULL_TREE
);
6792 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6794 gsi_remove (&gsi
, true);
6796 /* Connect all the blocks. */
6797 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6798 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6799 ep
= find_edge (entry_bb
, second_bb
);
6800 ep
->flags
= EDGE_TRUE_VALUE
;
6801 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6802 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6803 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6807 ep
= find_edge (cont_bb
, body_bb
);
6808 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6813 else if (fd
->collapse
> 1)
6816 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6819 ep
->flags
= EDGE_TRUE_VALUE
;
6820 find_edge (cont_bb
, fin_bb
)->flags
6821 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6824 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6825 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6826 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6828 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6829 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6830 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6831 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6833 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6835 struct loop
*loop
= alloc_loop ();
6836 loop
->header
= body_bb
;
6837 if (collapse_bb
== NULL
)
6838 loop
->latch
= cont_bb
;
6839 add_loop (loop
, body_bb
->loop_father
);
6844 /* A subroutine of expand_omp_for. Generate code for a parallel
6845 loop with static schedule and a specified chunk size. Given
6848 for (V = N1; V cond N2; V += STEP) BODY;
6850 where COND is "<" or ">", we generate pseudocode
6852 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6857 if ((__typeof (V)) -1 > 0 && cond is >)
6858 n = -(adj + N2 - N1) / -STEP;
6860 n = (adj + N2 - N1) / STEP;
6862 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6863 here so that V is defined
6864 if the loop is not entered
6866 s0 = (trip * nthreads + threadid) * CHUNK;
6867 e0 = min(s0 + CHUNK, n);
6868 if (s0 < n) goto L1; else goto L4;
6875 if (V cond e) goto L2; else goto L3;
6883 expand_omp_for_static_chunk (struct omp_region
*region
,
6884 struct omp_for_data
*fd
, gimple inner_stmt
)
6886 tree n
, s0
, e0
, e
, t
;
6887 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6888 tree type
, itype
, vmain
, vback
, vextra
;
6889 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6890 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6891 gimple_stmt_iterator gsi
;
6893 bool broken_loop
= region
->cont
== NULL
;
6894 tree
*counts
= NULL
;
6897 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6898 != GF_OMP_FOR_KIND_OACC_LOOP
)
6901 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6902 if (POINTER_TYPE_P (type
))
6903 itype
= signed_type_for (type
);
6905 entry_bb
= region
->entry
;
6906 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6908 iter_part_bb
= se
->dest
;
6909 cont_bb
= region
->cont
;
6910 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6911 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6912 gcc_assert (broken_loop
6913 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6914 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6915 body_bb
= single_succ (seq_start_bb
);
6918 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6919 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6920 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6922 exit_bb
= region
->exit
;
6924 /* Trip and adjustment setup goes in ENTRY_BB. */
6925 gsi
= gsi_last_bb (entry_bb
);
6926 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6928 if (fd
->collapse
> 1)
6930 int first_zero_iter
= -1;
6931 basic_block l2_dom_bb
= NULL
;
6933 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6934 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6935 fin_bb
, first_zero_iter
,
6939 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6940 t
= integer_one_node
;
6942 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6943 fold_convert (type
, fd
->loop
.n1
),
6944 fold_convert (type
, fd
->loop
.n2
));
6945 if (fd
->collapse
== 1
6946 && TYPE_UNSIGNED (type
)
6947 && (t
== NULL_TREE
|| !integer_onep (t
)))
6949 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6950 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6951 true, GSI_SAME_STMT
);
6952 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6953 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6954 true, GSI_SAME_STMT
);
6955 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6956 NULL_TREE
, NULL_TREE
);
6957 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6958 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6959 expand_omp_regimplify_p
, NULL
, NULL
)
6960 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6961 expand_omp_regimplify_p
, NULL
, NULL
))
6963 gsi
= gsi_for_stmt (cond_stmt
);
6964 gimple_regimplify_operands (cond_stmt
, &gsi
);
6966 se
= split_block (entry_bb
, cond_stmt
);
6967 se
->flags
= EDGE_TRUE_VALUE
;
6968 entry_bb
= se
->dest
;
6969 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6970 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6971 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6972 if (gimple_in_ssa_p (cfun
))
6974 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6975 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6976 !gsi_end_p (gpi
); gsi_next (&gpi
))
6978 gphi
*phi
= gpi
.phi ();
6979 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6980 se
, UNKNOWN_LOCATION
);
6983 gsi
= gsi_last_bb (entry_bb
);
6986 switch (gimple_omp_for_kind (fd
->for_stmt
))
6988 case GF_OMP_FOR_KIND_FOR
:
6989 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6990 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6992 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6993 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6994 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6996 case GF_OMP_FOR_KIND_OACC_LOOP
:
6997 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6998 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
7003 nthreads
= build_call_expr (nthreads
, 0);
7004 nthreads
= fold_convert (itype
, nthreads
);
7005 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
7006 true, GSI_SAME_STMT
);
7007 threadid
= build_call_expr (threadid
, 0);
7008 threadid
= fold_convert (itype
, threadid
);
7009 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
7010 true, GSI_SAME_STMT
);
7014 step
= fd
->loop
.step
;
7015 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7017 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7018 OMP_CLAUSE__LOOPTEMP_
);
7019 gcc_assert (innerc
);
7020 n1
= OMP_CLAUSE_DECL (innerc
);
7021 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7022 OMP_CLAUSE__LOOPTEMP_
);
7023 gcc_assert (innerc
);
7024 n2
= OMP_CLAUSE_DECL (innerc
);
7026 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
7027 true, NULL_TREE
, true, GSI_SAME_STMT
);
7028 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
7029 true, NULL_TREE
, true, GSI_SAME_STMT
);
7030 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
7031 true, NULL_TREE
, true, GSI_SAME_STMT
);
7033 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
7034 true, NULL_TREE
, true, GSI_SAME_STMT
);
7036 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
7037 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
7038 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
7039 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
7040 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
7041 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7042 fold_build1 (NEGATE_EXPR
, itype
, t
),
7043 fold_build1 (NEGATE_EXPR
, itype
, step
));
7045 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
7046 t
= fold_convert (itype
, t
);
7047 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7048 true, GSI_SAME_STMT
);
7050 trip_var
= create_tmp_reg (itype
, ".trip");
7051 if (gimple_in_ssa_p (cfun
))
7053 trip_init
= make_ssa_name (trip_var
);
7054 trip_main
= make_ssa_name (trip_var
);
7055 trip_back
= make_ssa_name (trip_var
);
7059 trip_init
= trip_var
;
7060 trip_main
= trip_var
;
7061 trip_back
= trip_var
;
7064 gassign
*assign_stmt
7065 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
7066 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7068 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
7069 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7070 if (POINTER_TYPE_P (type
))
7071 t
= fold_build_pointer_plus (n1
, t
);
7073 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7074 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7075 true, GSI_SAME_STMT
);
7077 /* Remove the GIMPLE_OMP_FOR. */
7078 gsi_remove (&gsi
, true);
7080 /* Iteration space partitioning goes in ITER_PART_BB. */
7081 gsi
= gsi_last_bb (iter_part_bb
);
7083 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
7084 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
7085 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
7086 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7087 false, GSI_CONTINUE_LINKING
);
7089 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
7090 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
7091 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7092 false, GSI_CONTINUE_LINKING
);
7094 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
7095 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
7097 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7098 gsi
= gsi_start_bb (seq_start_bb
);
7100 tree startvar
= fd
->loop
.v
;
7101 tree endvar
= NULL_TREE
;
7103 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7105 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
7106 ? gimple_omp_parallel_clauses (inner_stmt
)
7107 : gimple_omp_for_clauses (inner_stmt
);
7108 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7109 gcc_assert (innerc
);
7110 startvar
= OMP_CLAUSE_DECL (innerc
);
7111 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7112 OMP_CLAUSE__LOOPTEMP_
);
7113 gcc_assert (innerc
);
7114 endvar
= OMP_CLAUSE_DECL (innerc
);
7117 t
= fold_convert (itype
, s0
);
7118 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7119 if (POINTER_TYPE_P (type
))
7120 t
= fold_build_pointer_plus (n1
, t
);
7122 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7123 t
= fold_convert (TREE_TYPE (startvar
), t
);
7124 t
= force_gimple_operand_gsi (&gsi
, t
,
7126 && TREE_ADDRESSABLE (startvar
),
7127 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7128 assign_stmt
= gimple_build_assign (startvar
, t
);
7129 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7131 t
= fold_convert (itype
, e0
);
7132 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7133 if (POINTER_TYPE_P (type
))
7134 t
= fold_build_pointer_plus (n1
, t
);
7136 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7137 t
= fold_convert (TREE_TYPE (startvar
), t
);
7138 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7139 false, GSI_CONTINUE_LINKING
);
7142 assign_stmt
= gimple_build_assign (endvar
, e
);
7143 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7144 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
7145 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
7147 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
7148 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7150 if (fd
->collapse
> 1)
7151 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
7155 /* The code controlling the sequential loop goes in CONT_BB,
7156 replacing the GIMPLE_OMP_CONTINUE. */
7157 gsi
= gsi_last_bb (cont_bb
);
7158 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
7159 vmain
= gimple_omp_continue_control_use (cont_stmt
);
7160 vback
= gimple_omp_continue_control_def (cont_stmt
);
7162 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7164 if (POINTER_TYPE_P (type
))
7165 t
= fold_build_pointer_plus (vmain
, step
);
7167 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
7168 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
7169 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7170 true, GSI_SAME_STMT
);
7171 assign_stmt
= gimple_build_assign (vback
, t
);
7172 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7174 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
7175 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
7177 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
7180 /* Remove GIMPLE_OMP_CONTINUE. */
7181 gsi_remove (&gsi
, true);
7183 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
7184 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
7186 /* Trip update code goes into TRIP_UPDATE_BB. */
7187 gsi
= gsi_start_bb (trip_update_bb
);
7189 t
= build_int_cst (itype
, 1);
7190 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
7191 assign_stmt
= gimple_build_assign (trip_back
, t
);
7192 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7195 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7196 gsi
= gsi_last_bb (exit_bb
);
7197 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
7199 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
7200 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
7201 gcc_checking_assert (t
== NULL_TREE
);
7203 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
7205 gsi_remove (&gsi
, true);
7207 /* Connect the new blocks. */
7208 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
7209 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
7213 se
= find_edge (cont_bb
, body_bb
);
7214 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7219 else if (fd
->collapse
> 1)
7222 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
7225 se
->flags
= EDGE_TRUE_VALUE
;
7226 find_edge (cont_bb
, trip_update_bb
)->flags
7227 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
7229 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
7232 if (gimple_in_ssa_p (cfun
))
7240 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
7242 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7243 remove arguments of the phi nodes in fin_bb. We need to create
7244 appropriate phi nodes in iter_part_bb instead. */
7245 se
= single_pred_edge (fin_bb
);
7246 re
= single_succ_edge (trip_update_bb
);
7247 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
7248 ene
= single_succ_edge (entry_bb
);
7250 psi
= gsi_start_phis (fin_bb
);
7251 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
7252 gsi_next (&psi
), ++i
)
7255 source_location locus
;
7258 t
= gimple_phi_result (phi
);
7259 gcc_assert (t
== redirect_edge_var_map_result (vm
));
7260 nphi
= create_phi_node (t
, iter_part_bb
);
7262 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
7263 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
7265 /* A special case -- fd->loop.v is not yet computed in
7266 iter_part_bb, we need to use vextra instead. */
7267 if (t
== fd
->loop
.v
)
7269 add_phi_arg (nphi
, t
, ene
, locus
);
7270 locus
= redirect_edge_var_map_location (vm
);
7271 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
7273 gcc_assert (gsi_end_p (psi
) && i
== head
->length ());
7274 redirect_edge_var_map_clear (re
);
7277 psi
= gsi_start_phis (fin_bb
);
7278 if (gsi_end_p (psi
))
7280 remove_phi_node (&psi
, false);
7283 /* Make phi node for trip. */
7284 phi
= create_phi_node (trip_main
, iter_part_bb
);
7285 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
7287 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
7292 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
7293 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
7294 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
7295 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
7296 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
7297 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
7298 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
7299 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
7300 recompute_dominator (CDI_DOMINATORS
, body_bb
));
7304 struct loop
*trip_loop
= alloc_loop ();
7305 trip_loop
->header
= iter_part_bb
;
7306 trip_loop
->latch
= trip_update_bb
;
7307 add_loop (trip_loop
, iter_part_bb
->loop_father
);
7309 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7311 struct loop
*loop
= alloc_loop ();
7312 loop
->header
= body_bb
;
7313 if (collapse_bb
== NULL
)
7314 loop
->latch
= cont_bb
;
7315 add_loop (loop
, trip_loop
);
7320 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7322 for (V = N1; V cond N2; V += STEP) BODY;
7324 where COND is "<" or ">" or "!=", we generate pseudocode
7326 for (ind_var = low; ind_var < high; ind_var++)
7328 V = n1 + (ind_var * STEP)
7333 In the above pseudocode, low and high are function parameters of the
7334 child function. In the function below, we are inserting a temp.
7335 variable that will be making a call to two OMP functions that will not be
7336 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7337 with _Cilk_for). These functions are replaced with low and high
7338 by the function that handles taskreg. */
7342 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
7344 bool broken_loop
= region
->cont
== NULL
;
7345 basic_block entry_bb
= region
->entry
;
7346 basic_block cont_bb
= region
->cont
;
7348 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7349 gcc_assert (broken_loop
7350 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7351 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7352 basic_block l1_bb
, l2_bb
;
7356 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7357 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7358 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7359 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7363 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7364 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7365 l2_bb
= single_succ (l1_bb
);
7367 basic_block exit_bb
= region
->exit
;
7368 basic_block l2_dom_bb
= NULL
;
7370 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
7372 /* Below statements until the "tree high_val = ..." are pseudo statements
7373 used to pass information to be used by expand_omp_taskreg.
7374 low_val and high_val will be replaced by the __low and __high
7375 parameter from the child function.
7377 The call_exprs part is a place-holder, it is mainly used
7378 to distinctly identify to the top-level part that this is
7379 where we should put low and high (reasoning given in header
7383 = gimple_omp_parallel_child_fn (
7384 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
7385 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
7386 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
7388 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
7390 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
7393 gcc_assert (low_val
&& high_val
);
7395 tree type
= TREE_TYPE (low_val
);
7396 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
7397 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7399 /* Not needed in SSA form right now. */
7400 gcc_assert (!gimple_in_ssa_p (cfun
));
7401 if (l2_dom_bb
== NULL
)
7407 gimple stmt
= gimple_build_assign (ind_var
, n1
);
7409 /* Replace the GIMPLE_OMP_FOR statement. */
7410 gsi_replace (&gsi
, stmt
, true);
7414 /* Code to control the increment goes in the CONT_BB. */
7415 gsi
= gsi_last_bb (cont_bb
);
7416 stmt
= gsi_stmt (gsi
);
7417 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7418 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
7419 build_one_cst (type
));
7421 /* Replace GIMPLE_OMP_CONTINUE. */
7422 gsi_replace (&gsi
, stmt
, true);
7425 /* Emit the condition in L1_BB. */
7426 gsi
= gsi_after_labels (l1_bb
);
7427 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
7428 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
7430 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
7431 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7432 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7434 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7435 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
7436 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
7437 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7439 /* The condition is always '<' since the runtime will fill in the low
7441 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
7442 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7444 /* Remove GIMPLE_OMP_RETURN. */
7445 gsi
= gsi_last_bb (exit_bb
);
7446 gsi_remove (&gsi
, true);
7448 /* Connect the new blocks. */
7449 remove_edge (FALLTHRU_EDGE (entry_bb
));
7454 remove_edge (BRANCH_EDGE (entry_bb
));
7455 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7457 e
= BRANCH_EDGE (l1_bb
);
7458 ne
= FALLTHRU_EDGE (l1_bb
);
7459 e
->flags
= EDGE_TRUE_VALUE
;
7463 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7465 ne
= single_succ_edge (l1_bb
);
7466 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7469 ne
->flags
= EDGE_FALSE_VALUE
;
7470 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7471 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7473 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7474 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7475 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7479 struct loop
*loop
= alloc_loop ();
7480 loop
->header
= l1_bb
;
7481 loop
->latch
= cont_bb
;
7482 add_loop (loop
, l1_bb
->loop_father
);
7483 loop
->safelen
= INT_MAX
;
7486 /* Pick the correct library function based on the precision of the
7487 induction variable type. */
7488 tree lib_fun
= NULL_TREE
;
7489 if (TYPE_PRECISION (type
) == 32)
7490 lib_fun
= cilk_for_32_fndecl
;
7491 else if (TYPE_PRECISION (type
) == 64)
7492 lib_fun
= cilk_for_64_fndecl
;
7496 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7498 /* WS_ARGS contains the library function flavor to call:
7499 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7500 user-defined grain value. If the user does not define one, then zero
7501 is passed in by the parser. */
7502 vec_alloc (region
->ws_args
, 2);
7503 region
->ws_args
->quick_push (lib_fun
);
7504 region
->ws_args
->quick_push (fd
->chunk_size
);
7507 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7508 loop. Given parameters:
7510 for (V = N1; V cond N2; V += STEP) BODY;
7512 where COND is "<" or ">", we generate pseudocode
7520 if (V cond N2) goto L0; else goto L2;
7523 For collapsed loops, given parameters:
7525 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7526 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7527 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7530 we generate pseudocode
7536 count3 = (adj + N32 - N31) / STEP3;
7541 count2 = (adj + N22 - N21) / STEP2;
7546 count1 = (adj + N12 - N11) / STEP1;
7547 count = count1 * count2 * count3;
7557 V2 += (V3 cond3 N32) ? 0 : STEP2;
7558 V3 = (V3 cond3 N32) ? V3 : N31;
7559 V1 += (V2 cond2 N22) ? 0 : STEP1;
7560 V2 = (V2 cond2 N22) ? V2 : N21;
7562 if (V < count) goto L0; else goto L2;
7568 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7571 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7572 gimple_stmt_iterator gsi
;
7575 bool broken_loop
= region
->cont
== NULL
;
7577 tree
*counts
= NULL
;
7579 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7580 OMP_CLAUSE_SAFELEN
);
7581 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7582 OMP_CLAUSE__SIMDUID_
);
7585 type
= TREE_TYPE (fd
->loop
.v
);
7586 entry_bb
= region
->entry
;
7587 cont_bb
= region
->cont
;
7588 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7589 gcc_assert (broken_loop
7590 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7591 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7594 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7595 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7596 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7597 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7601 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7602 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7603 l2_bb
= single_succ (l1_bb
);
7605 exit_bb
= region
->exit
;
7608 gsi
= gsi_last_bb (entry_bb
);
7610 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7611 /* Not needed in SSA form right now. */
7612 gcc_assert (!gimple_in_ssa_p (cfun
));
7613 if (fd
->collapse
> 1)
7615 int first_zero_iter
= -1;
7616 basic_block zero_iter_bb
= l2_bb
;
7618 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7619 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7620 zero_iter_bb
, first_zero_iter
,
7623 if (l2_dom_bb
== NULL
)
7628 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7630 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7631 OMP_CLAUSE__LOOPTEMP_
);
7632 gcc_assert (innerc
);
7633 n1
= OMP_CLAUSE_DECL (innerc
);
7634 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7635 OMP_CLAUSE__LOOPTEMP_
);
7636 gcc_assert (innerc
);
7637 n2
= OMP_CLAUSE_DECL (innerc
);
7638 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7639 fold_convert (type
, n1
));
7640 if (fd
->collapse
> 1)
7643 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7649 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7650 fold_convert (type
, fd
->loop
.n1
));
7651 if (fd
->collapse
> 1)
7652 for (i
= 0; i
< fd
->collapse
; i
++)
7654 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7655 if (POINTER_TYPE_P (itype
))
7656 itype
= signed_type_for (itype
);
7657 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7658 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7662 /* Remove the GIMPLE_OMP_FOR statement. */
7663 gsi_remove (&gsi
, true);
7667 /* Code to control the increment goes in the CONT_BB. */
7668 gsi
= gsi_last_bb (cont_bb
);
7669 stmt
= gsi_stmt (gsi
);
7670 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7672 if (POINTER_TYPE_P (type
))
7673 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7675 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7676 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7678 if (fd
->collapse
> 1)
7680 i
= fd
->collapse
- 1;
7681 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7683 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7684 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7688 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7690 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7693 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7695 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7697 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7698 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7699 if (POINTER_TYPE_P (itype2
))
7700 itype2
= signed_type_for (itype2
);
7701 t
= build3 (COND_EXPR
, itype2
,
7702 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7704 fold_convert (itype
, fd
->loops
[i
].n2
)),
7705 build_int_cst (itype2
, 0),
7706 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7707 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7708 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7710 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7711 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7713 t
= build3 (COND_EXPR
, itype
,
7714 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7716 fold_convert (itype
, fd
->loops
[i
].n2
)),
7718 fold_convert (itype
, fd
->loops
[i
].n1
));
7719 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7723 /* Remove GIMPLE_OMP_CONTINUE. */
7724 gsi_remove (&gsi
, true);
7727 /* Emit the condition in L1_BB. */
7728 gsi
= gsi_start_bb (l1_bb
);
7730 t
= fold_convert (type
, n2
);
7731 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7732 false, GSI_CONTINUE_LINKING
);
7733 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7734 cond_stmt
= gimple_build_cond_empty (t
);
7735 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
7736 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7738 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7741 gsi
= gsi_for_stmt (cond_stmt
);
7742 gimple_regimplify_operands (cond_stmt
, &gsi
);
7745 /* Remove GIMPLE_OMP_RETURN. */
7746 gsi
= gsi_last_bb (exit_bb
);
7747 gsi_remove (&gsi
, true);
7749 /* Connect the new blocks. */
7750 remove_edge (FALLTHRU_EDGE (entry_bb
));
7754 remove_edge (BRANCH_EDGE (entry_bb
));
7755 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7757 e
= BRANCH_EDGE (l1_bb
);
7758 ne
= FALLTHRU_EDGE (l1_bb
);
7759 e
->flags
= EDGE_TRUE_VALUE
;
7763 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7765 ne
= single_succ_edge (l1_bb
);
7766 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7769 ne
->flags
= EDGE_FALSE_VALUE
;
7770 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7771 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7773 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7774 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7775 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7779 struct loop
*loop
= alloc_loop ();
7780 loop
->header
= l1_bb
;
7781 loop
->latch
= cont_bb
;
7782 add_loop (loop
, l1_bb
->loop_father
);
7783 if (safelen
== NULL_TREE
)
7784 loop
->safelen
= INT_MAX
;
7787 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7788 if (TREE_CODE (safelen
) != INTEGER_CST
)
7790 else if (!tree_fits_uhwi_p (safelen
)
7791 || tree_to_uhwi (safelen
) > INT_MAX
)
7792 loop
->safelen
= INT_MAX
;
7794 loop
->safelen
= tree_to_uhwi (safelen
);
7795 if (loop
->safelen
== 1)
7800 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7801 cfun
->has_simduid_loops
= true;
7803 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7805 if ((flag_tree_loop_vectorize
7806 || (!global_options_set
.x_flag_tree_loop_vectorize
7807 && !global_options_set
.x_flag_tree_vectorize
))
7808 && flag_tree_loop_optimize
7809 && loop
->safelen
> 1)
7811 loop
->force_vectorize
= true;
7812 cfun
->has_force_vectorize_loops
= true;
7818 /* Expand the OMP loop defined by REGION. */
7821 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7823 struct omp_for_data fd
;
7824 struct omp_for_data_loop
*loops
;
7827 = (struct omp_for_data_loop
*)
7828 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7829 * sizeof (struct omp_for_data_loop
));
7830 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
7832 region
->sched_kind
= fd
.sched_kind
;
7834 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7835 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7836 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7839 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7840 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7841 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7844 /* If there isn't a continue then this is a degerate case where
7845 the introduction of abnormal edges during lowering will prevent
7846 original loops from being detected. Fix that up. */
7847 loops_state_set (LOOPS_NEED_FIXUP
);
7849 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7850 expand_omp_simd (region
, &fd
);
7851 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7852 expand_cilk_for (region
, &fd
);
7853 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7854 && !fd
.have_ordered
)
7856 if (fd
.chunk_size
== NULL
)
7857 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7859 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7863 int fn_index
, start_ix
, next_ix
;
7865 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7866 == GF_OMP_FOR_KIND_FOR
);
7867 if (fd
.chunk_size
== NULL
7868 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7869 fd
.chunk_size
= integer_zero_node
;
7870 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7871 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7872 ? 3 : fd
.sched_kind
;
7873 fn_index
+= fd
.have_ordered
* 4;
7874 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7875 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7876 if (fd
.iter_type
== long_long_unsigned_type_node
)
7878 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7879 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7880 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7881 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7883 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7884 (enum built_in_function
) next_ix
, inner_stmt
);
7887 if (gimple_in_ssa_p (cfun
))
7888 update_ssa (TODO_update_ssa_only_virtuals
);
7892 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7894 v = GOMP_sections_start (n);
7911 v = GOMP_sections_next ();
7916 If this is a combined parallel sections, replace the call to
7917 GOMP_sections_start with call to GOMP_sections_next. */
7920 expand_omp_sections (struct omp_region
*region
)
7922 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7924 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7925 gimple_stmt_iterator si
, switch_si
;
7926 gomp_sections
*sections_stmt
;
7928 gomp_continue
*cont
;
7931 struct omp_region
*inner
;
7933 bool exit_reachable
= region
->cont
!= NULL
;
7935 gcc_assert (region
->exit
!= NULL
);
7936 entry_bb
= region
->entry
;
7937 l0_bb
= single_succ (entry_bb
);
7938 l1_bb
= region
->cont
;
7939 l2_bb
= region
->exit
;
7940 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7941 l2
= gimple_block_label (l2_bb
);
7944 /* This can happen if there are reductions. */
7945 len
= EDGE_COUNT (l0_bb
->succs
);
7946 gcc_assert (len
> 0);
7947 e
= EDGE_SUCC (l0_bb
, len
- 1);
7948 si
= gsi_last_bb (e
->dest
);
7951 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7952 l2
= gimple_block_label (e
->dest
);
7954 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7956 si
= gsi_last_bb (e
->dest
);
7958 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7960 l2
= gimple_block_label (e
->dest
);
7966 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7968 default_bb
= create_empty_bb (l0_bb
);
7970 /* We will build a switch() with enough cases for all the
7971 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7972 and a default case to abort if something goes wrong. */
7973 len
= EDGE_COUNT (l0_bb
->succs
);
7975 /* Use vec::quick_push on label_vec throughout, since we know the size
7977 auto_vec
<tree
> label_vec (len
);
7979 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7980 GIMPLE_OMP_SECTIONS statement. */
7981 si
= gsi_last_bb (entry_bb
);
7982 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
7983 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7984 vin
= gimple_omp_sections_control (sections_stmt
);
7985 if (!is_combined_parallel (region
))
7987 /* If we are not inside a combined parallel+sections region,
7988 call GOMP_sections_start. */
7989 t
= build_int_cst (unsigned_type_node
, len
- 1);
7990 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7991 stmt
= gimple_build_call (u
, 1, t
);
7995 /* Otherwise, call GOMP_sections_next. */
7996 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7997 stmt
= gimple_build_call (u
, 0);
7999 gimple_call_set_lhs (stmt
, vin
);
8000 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8001 gsi_remove (&si
, true);
8003 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8005 switch_si
= gsi_last_bb (l0_bb
);
8006 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
8009 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
8010 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
8011 vmain
= gimple_omp_continue_control_use (cont
);
8012 vnext
= gimple_omp_continue_control_def (cont
);
8020 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
8021 label_vec
.quick_push (t
);
8024 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8025 for (inner
= region
->inner
, casei
= 1;
8027 inner
= inner
->next
, i
++, casei
++)
8029 basic_block s_entry_bb
, s_exit_bb
;
8031 /* Skip optional reduction region. */
8032 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
8039 s_entry_bb
= inner
->entry
;
8040 s_exit_bb
= inner
->exit
;
8042 t
= gimple_block_label (s_entry_bb
);
8043 u
= build_int_cst (unsigned_type_node
, casei
);
8044 u
= build_case_label (u
, NULL
, t
);
8045 label_vec
.quick_push (u
);
8047 si
= gsi_last_bb (s_entry_bb
);
8048 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
8049 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
8050 gsi_remove (&si
, true);
8051 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
8053 if (s_exit_bb
== NULL
)
8056 si
= gsi_last_bb (s_exit_bb
);
8057 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8058 gsi_remove (&si
, true);
8060 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
8063 /* Error handling code goes in DEFAULT_BB. */
8064 t
= gimple_block_label (default_bb
);
8065 u
= build_case_label (NULL
, NULL
, t
);
8066 make_edge (l0_bb
, default_bb
, 0);
8067 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
8069 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
8070 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
8071 gsi_remove (&switch_si
, true);
8073 si
= gsi_start_bb (default_bb
);
8074 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
8075 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
8081 /* Code to get the next section goes in L1_BB. */
8082 si
= gsi_last_bb (l1_bb
);
8083 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
8085 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8086 stmt
= gimple_build_call (bfn_decl
, 0);
8087 gimple_call_set_lhs (stmt
, vnext
);
8088 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8089 gsi_remove (&si
, true);
8091 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
8094 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8095 si
= gsi_last_bb (l2_bb
);
8096 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
8097 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
8098 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
8099 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
8101 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
8102 stmt
= gimple_build_call (t
, 0);
8103 if (gimple_omp_return_lhs (gsi_stmt (si
)))
8104 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
8105 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8106 gsi_remove (&si
, true);
8108 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
8112 /* Expand code for an OpenMP single directive. We've already expanded
8113 much of the code, here we simply place the GOMP_barrier call. */
8116 expand_omp_single (struct omp_region
*region
)
8118 basic_block entry_bb
, exit_bb
;
8119 gimple_stmt_iterator si
;
8121 entry_bb
= region
->entry
;
8122 exit_bb
= region
->exit
;
8124 si
= gsi_last_bb (entry_bb
);
8125 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
8126 gsi_remove (&si
, true);
8127 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8129 si
= gsi_last_bb (exit_bb
);
8130 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
8132 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
8133 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
8135 gsi_remove (&si
, true);
8136 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8140 /* Generic expansion for OpenMP synchronization directives: master,
8141 ordered and critical. All we need to do here is remove the entry
8142 and exit markers for REGION. */
8145 expand_omp_synch (struct omp_region
*region
)
8147 basic_block entry_bb
, exit_bb
;
8148 gimple_stmt_iterator si
;
8150 entry_bb
= region
->entry
;
8151 exit_bb
= region
->exit
;
8153 si
= gsi_last_bb (entry_bb
);
8154 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
8155 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
8156 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
8157 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
8158 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
8159 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
8160 gsi_remove (&si
, true);
8161 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8165 si
= gsi_last_bb (exit_bb
);
8166 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8167 gsi_remove (&si
, true);
8168 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8172 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8173 operation as a normal volatile load. */
8176 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
8177 tree loaded_val
, int index
)
8179 enum built_in_function tmpbase
;
8180 gimple_stmt_iterator gsi
;
8181 basic_block store_bb
;
8184 tree decl
, call
, type
, itype
;
8186 gsi
= gsi_last_bb (load_bb
);
8187 stmt
= gsi_stmt (gsi
);
8188 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8189 loc
= gimple_location (stmt
);
8191 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8192 is smaller than word size, then expand_atomic_load assumes that the load
8193 is atomic. We could avoid the builtin entirely in this case. */
8195 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8196 decl
= builtin_decl_explicit (tmpbase
);
8197 if (decl
== NULL_TREE
)
8200 type
= TREE_TYPE (loaded_val
);
8201 itype
= TREE_TYPE (TREE_TYPE (decl
));
8203 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
8204 build_int_cst (NULL
,
8205 gimple_omp_atomic_seq_cst_p (stmt
)
8207 : MEMMODEL_RELAXED
));
8208 if (!useless_type_conversion_p (type
, itype
))
8209 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8210 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8212 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8213 gsi_remove (&gsi
, true);
8215 store_bb
= single_succ (load_bb
);
8216 gsi
= gsi_last_bb (store_bb
);
8217 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8218 gsi_remove (&gsi
, true);
8220 if (gimple_in_ssa_p (cfun
))
8221 update_ssa (TODO_update_ssa_no_phi
);
8226 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8227 operation as a normal volatile store. */
8230 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
8231 tree loaded_val
, tree stored_val
, int index
)
8233 enum built_in_function tmpbase
;
8234 gimple_stmt_iterator gsi
;
8235 basic_block store_bb
= single_succ (load_bb
);
8238 tree decl
, call
, type
, itype
;
8242 gsi
= gsi_last_bb (load_bb
);
8243 stmt
= gsi_stmt (gsi
);
8244 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8246 /* If the load value is needed, then this isn't a store but an exchange. */
8247 exchange
= gimple_omp_atomic_need_value_p (stmt
);
8249 gsi
= gsi_last_bb (store_bb
);
8250 stmt
= gsi_stmt (gsi
);
8251 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
8252 loc
= gimple_location (stmt
);
8254 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8255 is smaller than word size, then expand_atomic_store assumes that the store
8256 is atomic. We could avoid the builtin entirely in this case. */
8258 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
8259 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
8260 decl
= builtin_decl_explicit (tmpbase
);
8261 if (decl
== NULL_TREE
)
8264 type
= TREE_TYPE (stored_val
);
8266 /* Dig out the type of the function's second argument. */
8267 itype
= TREE_TYPE (decl
);
8268 itype
= TYPE_ARG_TYPES (itype
);
8269 itype
= TREE_CHAIN (itype
);
8270 itype
= TREE_VALUE (itype
);
8271 imode
= TYPE_MODE (itype
);
8273 if (exchange
&& !can_atomic_exchange_p (imode
, true))
8276 if (!useless_type_conversion_p (itype
, type
))
8277 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
8278 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
8279 build_int_cst (NULL
,
8280 gimple_omp_atomic_seq_cst_p (stmt
)
8282 : MEMMODEL_RELAXED
));
8285 if (!useless_type_conversion_p (type
, itype
))
8286 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8287 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8290 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8291 gsi_remove (&gsi
, true);
8293 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8294 gsi
= gsi_last_bb (load_bb
);
8295 gsi_remove (&gsi
, true);
8297 if (gimple_in_ssa_p (cfun
))
8298 update_ssa (TODO_update_ssa_no_phi
);
8303 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8304 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8305 size of the data type, and thus usable to find the index of the builtin
8306 decl. Returns false if the expression is not of the proper form. */
8309 expand_omp_atomic_fetch_op (basic_block load_bb
,
8310 tree addr
, tree loaded_val
,
8311 tree stored_val
, int index
)
8313 enum built_in_function oldbase
, newbase
, tmpbase
;
8314 tree decl
, itype
, call
;
8316 basic_block store_bb
= single_succ (load_bb
);
8317 gimple_stmt_iterator gsi
;
8320 enum tree_code code
;
8321 bool need_old
, need_new
;
8325 /* We expect to find the following sequences:
8328 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8331 val = tmp OP something; (or: something OP tmp)
8332 GIMPLE_OMP_STORE (val)
8334 ???FIXME: Allow a more flexible sequence.
8335 Perhaps use data flow to pick the statements.
8339 gsi
= gsi_after_labels (store_bb
);
8340 stmt
= gsi_stmt (gsi
);
8341 loc
= gimple_location (stmt
);
8342 if (!is_gimple_assign (stmt
))
8345 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
8347 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
8348 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
8349 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
8350 gcc_checking_assert (!need_old
|| !need_new
);
8352 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
8355 /* Check for one of the supported fetch-op operations. */
8356 code
= gimple_assign_rhs_code (stmt
);
8360 case POINTER_PLUS_EXPR
:
8361 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
8362 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
8365 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
8366 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
8369 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
8370 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
8373 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
8374 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
8377 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
8378 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
8384 /* Make sure the expression is of the proper form. */
8385 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
8386 rhs
= gimple_assign_rhs2 (stmt
);
8387 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
8388 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
8389 rhs
= gimple_assign_rhs1 (stmt
);
8393 tmpbase
= ((enum built_in_function
)
8394 ((need_new
? newbase
: oldbase
) + index
+ 1));
8395 decl
= builtin_decl_explicit (tmpbase
);
8396 if (decl
== NULL_TREE
)
8398 itype
= TREE_TYPE (TREE_TYPE (decl
));
8399 imode
= TYPE_MODE (itype
);
8401 /* We could test all of the various optabs involved, but the fact of the
8402 matter is that (with the exception of i486 vs i586 and xadd) all targets
8403 that support any atomic operaton optab also implements compare-and-swap.
8404 Let optabs.c take care of expanding any compare-and-swap loop. */
8405 if (!can_compare_and_swap_p (imode
, true))
8408 gsi
= gsi_last_bb (load_bb
);
8409 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8411 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8412 It only requires that the operation happen atomically. Thus we can
8413 use the RELAXED memory model. */
8414 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
8415 fold_convert_loc (loc
, itype
, rhs
),
8416 build_int_cst (NULL
,
8417 seq_cst
? MEMMODEL_SEQ_CST
8418 : MEMMODEL_RELAXED
));
8420 if (need_old
|| need_new
)
8422 lhs
= need_old
? loaded_val
: stored_val
;
8423 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
8424 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
8427 call
= fold_convert_loc (loc
, void_type_node
, call
);
8428 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8429 gsi_remove (&gsi
, true);
8431 gsi
= gsi_last_bb (store_bb
);
8432 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8433 gsi_remove (&gsi
, true);
8434 gsi
= gsi_last_bb (store_bb
);
8435 gsi_remove (&gsi
, true);
8437 if (gimple_in_ssa_p (cfun
))
8438 update_ssa (TODO_update_ssa_no_phi
);
8443 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8447 newval = rhs; // with oldval replacing *addr in rhs
8448 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8449 if (oldval != newval)
8452 INDEX is log2 of the size of the data type, and thus usable to find the
8453 index of the builtin decl. */
8456 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
8457 tree addr
, tree loaded_val
, tree stored_val
,
8460 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
8461 tree type
, itype
, cmpxchg
, iaddr
;
8462 gimple_stmt_iterator si
;
8463 basic_block loop_header
= single_succ (load_bb
);
8466 enum built_in_function fncode
;
8468 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8469 order to use the RELAXED memory model effectively. */
8470 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8472 cmpxchg
= builtin_decl_explicit (fncode
);
8473 if (cmpxchg
== NULL_TREE
)
8475 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8476 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
8478 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
8481 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8482 si
= gsi_last_bb (load_bb
);
8483 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8485 /* For floating-point values, we'll need to view-convert them to integers
8486 so that we can perform the atomic compare and swap. Simplify the
8487 following code by always setting up the "i"ntegral variables. */
8488 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
8492 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
8495 = force_gimple_operand_gsi (&si
,
8496 fold_convert (TREE_TYPE (iaddr
), addr
),
8497 false, NULL_TREE
, true, GSI_SAME_STMT
);
8498 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
8499 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8500 loadedi
= create_tmp_var (itype
);
8501 if (gimple_in_ssa_p (cfun
))
8502 loadedi
= make_ssa_name (loadedi
);
8507 loadedi
= loaded_val
;
8510 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8511 tree loaddecl
= builtin_decl_explicit (fncode
);
8514 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8515 build_call_expr (loaddecl
, 2, iaddr
,
8516 build_int_cst (NULL_TREE
,
8517 MEMMODEL_RELAXED
)));
8519 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8520 build_int_cst (TREE_TYPE (iaddr
), 0));
8523 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8526 /* Move the value to the LOADEDI temporary. */
8527 if (gimple_in_ssa_p (cfun
))
8529 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8530 phi
= create_phi_node (loadedi
, loop_header
);
8531 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8535 gsi_insert_before (&si
,
8536 gimple_build_assign (loadedi
, initial
),
8538 if (loadedi
!= loaded_val
)
8540 gimple_stmt_iterator gsi2
;
8543 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8544 gsi2
= gsi_start_bb (loop_header
);
8545 if (gimple_in_ssa_p (cfun
))
8548 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8549 true, GSI_SAME_STMT
);
8550 stmt
= gimple_build_assign (loaded_val
, x
);
8551 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8555 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8556 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8557 true, GSI_SAME_STMT
);
8560 gsi_remove (&si
, true);
8562 si
= gsi_last_bb (store_bb
);
8563 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8566 storedi
= stored_val
;
8569 force_gimple_operand_gsi (&si
,
8570 build1 (VIEW_CONVERT_EXPR
, itype
,
8571 stored_val
), true, NULL_TREE
, true,
8574 /* Build the compare&swap statement. */
8575 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8576 new_storedi
= force_gimple_operand_gsi (&si
,
8577 fold_convert (TREE_TYPE (loadedi
),
8580 true, GSI_SAME_STMT
);
8582 if (gimple_in_ssa_p (cfun
))
8586 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
8587 stmt
= gimple_build_assign (old_vali
, loadedi
);
8588 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8590 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8591 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8594 /* Note that we always perform the comparison as an integer, even for
8595 floating point. This allows the atomic operation to properly
8596 succeed even with NaNs and -0.0. */
8597 stmt
= gimple_build_cond_empty
8598 (build2 (NE_EXPR
, boolean_type_node
,
8599 new_storedi
, old_vali
));
8600 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8603 e
= single_succ_edge (store_bb
);
8604 e
->flags
&= ~EDGE_FALLTHRU
;
8605 e
->flags
|= EDGE_FALSE_VALUE
;
8607 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8609 /* Copy the new value to loadedi (we already did that before the condition
8610 if we are not in SSA). */
8611 if (gimple_in_ssa_p (cfun
))
8613 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8614 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8617 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8618 gsi_remove (&si
, true);
8620 struct loop
*loop
= alloc_loop ();
8621 loop
->header
= loop_header
;
8622 loop
->latch
= store_bb
;
8623 add_loop (loop
, loop_header
->loop_father
);
8625 if (gimple_in_ssa_p (cfun
))
8626 update_ssa (TODO_update_ssa_no_phi
);
8631 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8633 GOMP_atomic_start ();
8637 The result is not globally atomic, but works so long as all parallel
8638 references are within #pragma omp atomic directives. According to
8639 responses received from omp@openmp.org, appears to be within spec.
8640 Which makes sense, since that's how several other compilers handle
8641 this situation as well.
8642 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8643 expanding. STORED_VAL is the operand of the matching
8644 GIMPLE_OMP_ATOMIC_STORE.
8647 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8651 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8656 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8657 tree addr
, tree loaded_val
, tree stored_val
)
8659 gimple_stmt_iterator si
;
8663 si
= gsi_last_bb (load_bb
);
8664 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8666 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8667 t
= build_call_expr (t
, 0);
8668 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8670 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8671 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8672 gsi_remove (&si
, true);
8674 si
= gsi_last_bb (store_bb
);
8675 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8677 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8679 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8681 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8682 t
= build_call_expr (t
, 0);
8683 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8684 gsi_remove (&si
, true);
8686 if (gimple_in_ssa_p (cfun
))
8687 update_ssa (TODO_update_ssa_no_phi
);
8691 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8692 using expand_omp_atomic_fetch_op. If it failed, we try to
8693 call expand_omp_atomic_pipeline, and if it fails too, the
8694 ultimate fallback is wrapping the operation in a mutex
8695 (expand_omp_atomic_mutex). REGION is the atomic region built
8696 by build_omp_regions_1(). */
8699 expand_omp_atomic (struct omp_region
*region
)
8701 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8702 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
8703 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
8704 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8705 tree addr
= gimple_omp_atomic_load_rhs (load
);
8706 tree stored_val
= gimple_omp_atomic_store_val (store
);
8707 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8708 HOST_WIDE_INT index
;
8710 /* Make sure the type is one of the supported sizes. */
8711 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8712 index
= exact_log2 (index
);
8713 if (index
>= 0 && index
<= 4)
8715 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8717 /* __sync builtins require strict data alignment. */
8718 if (exact_log2 (align
) >= index
)
8721 if (loaded_val
== stored_val
8722 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8723 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8724 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8725 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8729 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8730 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8731 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8732 && store_bb
== single_succ (load_bb
)
8733 && first_stmt (store_bb
) == store
8734 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8738 /* When possible, use specialized atomic update functions. */
8739 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8740 && store_bb
== single_succ (load_bb
)
8741 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8742 loaded_val
, stored_val
, index
))
8745 /* If we don't have specialized __sync builtins, try and implement
8746 as a compare and swap loop. */
8747 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8748 loaded_val
, stored_val
, index
))
8753 /* The ultimate fallback is wrapping the operation in a mutex. */
8754 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8758 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8761 expand_omp_target (struct omp_region
*region
)
8763 basic_block entry_bb
, exit_bb
, new_bb
;
8764 struct function
*child_cfun
;
8765 tree child_fn
, block
, t
;
8766 gimple_stmt_iterator gsi
;
8767 gomp_target
*entry_stmt
;
8770 bool offloaded
, data_region
;
8772 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
8773 new_bb
= region
->entry
;
8775 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
8776 switch (gimple_omp_target_kind (entry_stmt
))
8778 case GF_OMP_TARGET_KIND_REGION
:
8779 case GF_OMP_TARGET_KIND_UPDATE
:
8780 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8781 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8782 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8783 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8784 data_region
= false;
8786 case GF_OMP_TARGET_KIND_DATA
:
8787 case GF_OMP_TARGET_KIND_OACC_DATA
:
8794 child_fn
= NULL_TREE
;
8798 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8799 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8802 /* Supported by expand_omp_taskreg, but not here. */
8803 if (child_cfun
!= NULL
)
8804 gcc_checking_assert (!child_cfun
->cfg
);
8805 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
8807 entry_bb
= region
->entry
;
8808 exit_bb
= region
->exit
;
8812 unsigned srcidx
, dstidx
, num
;
8814 /* If the offloading region needs data sent from the parent
8815 function, then the very first statement (except possible
8816 tree profile counter updates) of the offloading body
8817 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8818 &.OMP_DATA_O is passed as an argument to the child function,
8819 we need to replace it with the argument as seen by the child
8822 In most cases, this will end up being the identity assignment
8823 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8824 a function call that has been inlined, the original PARM_DECL
8825 .OMP_DATA_I may have been converted into a different local
8826 variable. In which case, we need to keep the assignment. */
8827 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
8830 basic_block entry_succ_bb
= single_succ (entry_bb
);
8831 gimple_stmt_iterator gsi
;
8833 gimple tgtcopy_stmt
= NULL
;
8834 tree sender
= TREE_VEC_ELT (data_arg
, 0);
8836 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8838 gcc_assert (!gsi_end_p (gsi
));
8839 stmt
= gsi_stmt (gsi
);
8840 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8843 if (gimple_num_ops (stmt
) == 2)
8845 tree arg
= gimple_assign_rhs1 (stmt
);
8847 /* We're ignoring the subcode because we're
8848 effectively doing a STRIP_NOPS. */
8850 if (TREE_CODE (arg
) == ADDR_EXPR
8851 && TREE_OPERAND (arg
, 0) == sender
)
8853 tgtcopy_stmt
= stmt
;
8859 gcc_assert (tgtcopy_stmt
!= NULL
);
8860 arg
= DECL_ARGUMENTS (child_fn
);
8862 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8863 gsi_remove (&gsi
, true);
8866 /* Declare local variables needed in CHILD_CFUN. */
8867 block
= DECL_INITIAL (child_fn
);
8868 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8869 /* The gimplifier could record temporaries in the offloading block
8870 rather than in containing function's local_decls chain,
8871 which would mean cgraph missed finalizing them. Do it now. */
8872 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8873 if (TREE_CODE (t
) == VAR_DECL
8875 && !DECL_EXTERNAL (t
))
8876 varpool_node::finalize_decl (t
);
8877 DECL_SAVED_TREE (child_fn
) = NULL
;
8878 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8879 gimple_set_body (child_fn
, NULL
);
8880 TREE_USED (block
) = 1;
8882 /* Reset DECL_CONTEXT on function arguments. */
8883 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8884 DECL_CONTEXT (t
) = child_fn
;
8886 /* Split ENTRY_BB at GIMPLE_*,
8887 so that it can be moved to the child function. */
8888 gsi
= gsi_last_bb (entry_bb
);
8889 stmt
= gsi_stmt (gsi
);
8891 && gimple_code (stmt
) == gimple_code (entry_stmt
));
8892 gsi_remove (&gsi
, true);
8893 e
= split_block (entry_bb
, stmt
);
8895 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8897 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8900 gsi
= gsi_last_bb (exit_bb
);
8901 gcc_assert (!gsi_end_p (gsi
)
8902 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8903 stmt
= gimple_build_return (NULL
);
8904 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8905 gsi_remove (&gsi
, true);
8908 /* Move the offloading region into CHILD_CFUN. */
8910 block
= gimple_block (entry_stmt
);
8912 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8914 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8915 /* When the OMP expansion process cannot guarantee an up-to-date
8916 loop tree arrange for the child function to fixup loops. */
8917 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8918 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8920 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8921 num
= vec_safe_length (child_cfun
->local_decls
);
8922 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8924 t
= (*child_cfun
->local_decls
)[srcidx
];
8925 if (DECL_CONTEXT (t
) == cfun
->decl
)
8927 if (srcidx
!= dstidx
)
8928 (*child_cfun
->local_decls
)[dstidx
] = t
;
8932 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8934 /* Inform the callgraph about the new function. */
8935 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8936 cgraph_node::add_new_function (child_fn
, true);
8938 #ifdef ENABLE_OFFLOADING
8939 /* Add the new function to the offload table. */
8940 vec_safe_push (offload_funcs
, child_fn
);
8943 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8944 fixed in a following pass. */
8945 push_cfun (child_cfun
);
8946 cgraph_edge::rebuild_edges ();
8948 #ifdef ENABLE_OFFLOADING
8949 /* Prevent IPA from removing child_fn as unreachable, since there are no
8950 refs from the parent function to child_fn in offload LTO mode. */
8951 struct cgraph_node
*node
= cgraph_node::get (child_fn
);
8952 node
->mark_force_output ();
8955 /* Some EH regions might become dead, see PR34608. If
8956 pass_cleanup_cfg isn't the first pass to happen with the
8957 new child, these dead EH edges might cause problems.
8958 Clean them up now. */
8959 if (flag_exceptions
)
8962 bool changed
= false;
8964 FOR_EACH_BB_FN (bb
, cfun
)
8965 changed
|= gimple_purge_dead_eh_edges (bb
);
8967 cleanup_tree_cfg ();
8972 /* Emit a library call to launch the offloading region, or do data
8974 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8975 enum built_in_function start_ix
;
8976 location_t clause_loc
;
8978 switch (gimple_omp_target_kind (entry_stmt
))
8980 case GF_OMP_TARGET_KIND_REGION
:
8981 start_ix
= BUILT_IN_GOMP_TARGET
;
8983 case GF_OMP_TARGET_KIND_DATA
:
8984 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8986 case GF_OMP_TARGET_KIND_UPDATE
:
8987 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8989 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8990 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8991 start_ix
= BUILT_IN_GOACC_PARALLEL
;
8993 case GF_OMP_TARGET_KIND_OACC_DATA
:
8994 start_ix
= BUILT_IN_GOACC_DATA_START
;
8996 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8997 start_ix
= BUILT_IN_GOACC_UPDATE
;
8999 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9000 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
9006 clauses
= gimple_omp_target_clauses (entry_stmt
);
9008 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
9009 library choose) and there is no conditional. */
9011 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
9013 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
9015 cond
= OMP_CLAUSE_IF_EXPR (c
);
9017 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
9020 /* Even if we pass it to all library function calls, it is currently only
9021 defined/used for the OpenMP target ones. */
9022 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
9023 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
9024 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
);
9026 device
= OMP_CLAUSE_DEVICE_ID (c
);
9027 clause_loc
= OMP_CLAUSE_LOCATION (c
);
9030 clause_loc
= gimple_location (entry_stmt
);
9032 /* Ensure 'device' is of the correct type. */
9033 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
9035 /* If we found the clause 'if (cond)', build
9036 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
9039 cond
= gimple_boolify (cond
);
9041 basic_block cond_bb
, then_bb
, else_bb
;
9045 tmp_var
= create_tmp_var (TREE_TYPE (device
));
9047 e
= split_block (new_bb
, NULL
);
9050 gsi
= gsi_last_bb (new_bb
);
9052 e
= split_block (new_bb
, gsi_stmt (gsi
));
9058 then_bb
= create_empty_bb (cond_bb
);
9059 else_bb
= create_empty_bb (then_bb
);
9060 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
9061 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
9063 stmt
= gimple_build_cond_empty (cond
);
9064 gsi
= gsi_last_bb (cond_bb
);
9065 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9067 gsi
= gsi_start_bb (then_bb
);
9068 stmt
= gimple_build_assign (tmp_var
, device
);
9069 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9071 gsi
= gsi_start_bb (else_bb
);
9072 stmt
= gimple_build_assign (tmp_var
,
9073 build_int_cst (integer_type_node
,
9074 GOMP_DEVICE_HOST_FALLBACK
));
9075 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9077 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
9078 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
9079 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
9080 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
9081 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
9082 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
9087 gsi
= gsi_last_bb (new_bb
);
9088 t
= gimple_omp_target_data_arg (entry_stmt
);
9091 t1
= size_zero_node
;
9092 t2
= build_zero_cst (ptr_type_node
);
9098 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
9099 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
9100 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
9101 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
9102 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
9107 /* The maximum number used by any start_ix, without varargs. */
9108 unsigned int argcnt
= 11;
9110 vec_alloc (args
, argcnt
);
9111 args
->quick_push (device
);
9113 args
->quick_push (build_fold_addr_expr (child_fn
));
9116 case BUILT_IN_GOMP_TARGET
:
9117 case BUILT_IN_GOMP_TARGET_DATA
:
9118 case BUILT_IN_GOMP_TARGET_UPDATE
:
9119 /* This const void * is part of the current ABI, but we're not actually
9121 args
->quick_push (build_zero_cst (ptr_type_node
));
9123 case BUILT_IN_GOACC_DATA_START
:
9124 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9125 case BUILT_IN_GOACC_PARALLEL
:
9126 case BUILT_IN_GOACC_UPDATE
:
9131 args
->quick_push (t1
);
9132 args
->quick_push (t2
);
9133 args
->quick_push (t3
);
9134 args
->quick_push (t4
);
9137 case BUILT_IN_GOACC_DATA_START
:
9138 case BUILT_IN_GOMP_TARGET
:
9139 case BUILT_IN_GOMP_TARGET_DATA
:
9140 case BUILT_IN_GOMP_TARGET_UPDATE
:
9142 case BUILT_IN_GOACC_PARALLEL
:
9144 tree t_num_gangs
, t_num_workers
, t_vector_length
;
9146 /* Default values for num_gangs, num_workers, and vector_length. */
9147 t_num_gangs
= t_num_workers
= t_vector_length
9148 = fold_convert_loc (gimple_location (entry_stmt
),
9149 integer_type_node
, integer_one_node
);
9150 /* ..., but if present, use the value specified by the respective
9151 clause, making sure that are of the correct type. */
9152 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
9154 t_num_gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9156 OMP_CLAUSE_NUM_GANGS_EXPR (c
));
9157 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_WORKERS
);
9159 t_num_workers
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9161 OMP_CLAUSE_NUM_WORKERS_EXPR (c
));
9162 c
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
9164 t_vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9166 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c
));
9167 args
->quick_push (t_num_gangs
);
9168 args
->quick_push (t_num_workers
);
9169 args
->quick_push (t_vector_length
);
9172 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9173 case BUILT_IN_GOACC_UPDATE
:
9178 /* Default values for t_async. */
9179 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
9181 build_int_cst (integer_type_node
,
9183 /* ..., but if present, use the value specified by the respective
9184 clause, making sure that is of the correct type. */
9185 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
9187 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9189 OMP_CLAUSE_ASYNC_EXPR (c
));
9191 args
->quick_push (t_async
);
9192 /* Save the index, and... */
9193 t_wait_idx
= args
->length ();
9194 /* ... push a default value. */
9195 args
->quick_push (fold_convert_loc (gimple_location (entry_stmt
),
9197 integer_zero_node
));
9198 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
9203 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
9205 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
9207 args
->safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9209 OMP_CLAUSE_WAIT_EXPR (c
)));
9214 /* Now that we know the number, replace the default value. */
9215 args
->ordered_remove (t_wait_idx
);
9216 args
->quick_insert (t_wait_idx
,
9217 fold_convert_loc (gimple_location (entry_stmt
),
9219 build_int_cst (integer_type_node
, n
)));
9227 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), *args
);
9229 gimple_set_location (g
, gimple_location (entry_stmt
));
9230 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
9234 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
9235 gsi_remove (&gsi
, true);
9240 gsi
= gsi_last_bb (region
->exit
);
9242 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
9243 gsi_remove (&gsi
, true);
9248 /* Expand the parallel region tree rooted at REGION. Expansion
9249 proceeds in depth-first order. Innermost regions are expanded
9250 first. This way, parallel regions that require a new function to
9251 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9252 internal dependencies in their body. */
9255 expand_omp (struct omp_region
*region
)
9259 location_t saved_location
;
9260 gimple inner_stmt
= NULL
;
9262 /* First, determine whether this is a combined parallel+workshare
9264 if (region
->type
== GIMPLE_OMP_PARALLEL
)
9265 determine_parallel_type (region
);
9267 if (region
->type
== GIMPLE_OMP_FOR
9268 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
9269 inner_stmt
= last_stmt (region
->inner
->entry
);
9272 expand_omp (region
->inner
);
9274 saved_location
= input_location
;
9275 if (gimple_has_location (last_stmt (region
->entry
)))
9276 input_location
= gimple_location (last_stmt (region
->entry
));
9278 switch (region
->type
)
9280 case GIMPLE_OMP_PARALLEL
:
9281 case GIMPLE_OMP_TASK
:
9282 expand_omp_taskreg (region
);
9285 case GIMPLE_OMP_FOR
:
9286 expand_omp_for (region
, inner_stmt
);
9289 case GIMPLE_OMP_SECTIONS
:
9290 expand_omp_sections (region
);
9293 case GIMPLE_OMP_SECTION
:
9294 /* Individual omp sections are handled together with their
9295 parent GIMPLE_OMP_SECTIONS region. */
9298 case GIMPLE_OMP_SINGLE
:
9299 expand_omp_single (region
);
9302 case GIMPLE_OMP_MASTER
:
9303 case GIMPLE_OMP_TASKGROUP
:
9304 case GIMPLE_OMP_ORDERED
:
9305 case GIMPLE_OMP_CRITICAL
:
9306 case GIMPLE_OMP_TEAMS
:
9307 expand_omp_synch (region
);
9310 case GIMPLE_OMP_ATOMIC_LOAD
:
9311 expand_omp_atomic (region
);
9314 case GIMPLE_OMP_TARGET
:
9315 expand_omp_target (region
);
9322 input_location
= saved_location
;
9323 region
= region
->next
;
9328 /* Helper for build_omp_regions. Scan the dominator tree starting at
9329 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9330 true, the function ends once a single tree is built (otherwise, whole
9331 forest of OMP constructs may be built). */
9334 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
9337 gimple_stmt_iterator gsi
;
9341 gsi
= gsi_last_bb (bb
);
9342 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
9344 struct omp_region
*region
;
9345 enum gimple_code code
;
9347 stmt
= gsi_stmt (gsi
);
9348 code
= gimple_code (stmt
);
9349 if (code
== GIMPLE_OMP_RETURN
)
9351 /* STMT is the return point out of region PARENT. Mark it
9352 as the exit point and make PARENT the immediately
9353 enclosing region. */
9354 gcc_assert (parent
);
9357 parent
= parent
->outer
;
9359 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
9361 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9362 GIMPLE_OMP_RETURN, but matches with
9363 GIMPLE_OMP_ATOMIC_LOAD. */
9364 gcc_assert (parent
);
9365 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
9368 parent
= parent
->outer
;
9370 else if (code
== GIMPLE_OMP_CONTINUE
)
9372 gcc_assert (parent
);
9375 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
9377 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9378 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9382 region
= new_omp_region (bb
, code
, parent
);
9384 if (code
== GIMPLE_OMP_TARGET
)
9386 switch (gimple_omp_target_kind (stmt
))
9388 case GF_OMP_TARGET_KIND_REGION
:
9389 case GF_OMP_TARGET_KIND_DATA
:
9390 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9391 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9392 case GF_OMP_TARGET_KIND_OACC_DATA
:
9394 case GF_OMP_TARGET_KIND_UPDATE
:
9395 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9396 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9397 /* ..., other than for those stand-alone directives... */
9404 /* ..., this directive becomes the parent for a new region. */
9410 if (single_tree
&& !parent
)
9413 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
9415 son
= next_dom_son (CDI_DOMINATORS
, son
))
9416 build_omp_regions_1 (son
, parent
, single_tree
);
9419 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9423 build_omp_regions_root (basic_block root
)
9425 gcc_assert (root_omp_region
== NULL
);
9426 build_omp_regions_1 (root
, NULL
, true);
9427 gcc_assert (root_omp_region
!= NULL
);
9430 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9433 omp_expand_local (basic_block head
)
9435 build_omp_regions_root (head
);
9436 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
9438 fprintf (dump_file
, "\nOMP region tree\n\n");
9439 dump_omp_region (dump_file
, root_omp_region
, 0);
9440 fprintf (dump_file
, "\n");
9443 remove_exit_barriers (root_omp_region
);
9444 expand_omp (root_omp_region
);
9446 free_omp_regions ();
9449 /* Scan the CFG and build a tree of OMP regions. Return the root of
9450 the OMP region tree. */
9453 build_omp_regions (void)
9455 gcc_assert (root_omp_region
== NULL
);
9456 calculate_dominance_info (CDI_DOMINATORS
);
9457 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
9460 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9463 execute_expand_omp (void)
9465 build_omp_regions ();
9467 if (!root_omp_region
)
9472 fprintf (dump_file
, "\nOMP region tree\n\n");
9473 dump_omp_region (dump_file
, root_omp_region
, 0);
9474 fprintf (dump_file
, "\n");
9477 remove_exit_barriers (root_omp_region
);
9479 expand_omp (root_omp_region
);
9481 cleanup_tree_cfg ();
9483 free_omp_regions ();
9488 /* OMP expansion -- the default pass, run before creation of SSA form. */
9492 const pass_data pass_data_expand_omp
=
9494 GIMPLE_PASS
, /* type */
9495 "ompexp", /* name */
9496 OPTGROUP_NONE
, /* optinfo_flags */
9497 TV_NONE
, /* tv_id */
9498 PROP_gimple_any
, /* properties_required */
9499 PROP_gimple_eomp
, /* properties_provided */
9500 0, /* properties_destroyed */
9501 0, /* todo_flags_start */
9502 0, /* todo_flags_finish */
9505 class pass_expand_omp
: public gimple_opt_pass
9508 pass_expand_omp (gcc::context
*ctxt
)
9509 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
9512 /* opt_pass methods: */
9513 virtual unsigned int execute (function
*)
9515 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
9516 || flag_openmp_simd
!= 0)
9519 /* This pass always runs, to provide PROP_gimple_eomp.
9520 But often, there is nothing to do. */
9524 return execute_expand_omp ();
9527 }; // class pass_expand_omp
9532 make_pass_expand_omp (gcc::context
*ctxt
)
9534 return new pass_expand_omp (ctxt
);
9539 const pass_data pass_data_expand_omp_ssa
=
9541 GIMPLE_PASS
, /* type */
9542 "ompexpssa", /* name */
9543 OPTGROUP_NONE
, /* optinfo_flags */
9544 TV_NONE
, /* tv_id */
9545 PROP_cfg
| PROP_ssa
, /* properties_required */
9546 PROP_gimple_eomp
, /* properties_provided */
9547 0, /* properties_destroyed */
9548 0, /* todo_flags_start */
9549 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
9552 class pass_expand_omp_ssa
: public gimple_opt_pass
9555 pass_expand_omp_ssa (gcc::context
*ctxt
)
9556 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
9559 /* opt_pass methods: */
9560 virtual bool gate (function
*fun
)
9562 return !(fun
->curr_properties
& PROP_gimple_eomp
);
9564 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
9566 }; // class pass_expand_omp_ssa
9571 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
9573 return new pass_expand_omp_ssa (ctxt
);
9576 /* Routines to lower OMP directives into OMP-GIMPLE. */
9578 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9579 convert it to gimple. */
9581 oacc_gimple_assign (tree dest
, tree_code op
, tree src
, gimple_seq
*seq
)
9585 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
9587 stmt
= gimple_build_assign (dest
, op
, dest
, src
);
9588 gimple_seq_add_stmt (seq
, stmt
);
9592 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9593 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9594 gimplify_assign (t
, rdest
, seq
);
9597 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9598 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9599 gimplify_assign (t
, idest
, seq
);
9602 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9603 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9604 gimplify_assign (t
, rsrc
, seq
);
9607 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9608 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9609 gimplify_assign (t
, isrc
, seq
);
9612 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9613 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9616 if (op
== PLUS_EXPR
)
9618 stmt
= gimple_build_assign (r
, op
, rdest
, rsrc
);
9619 gimple_seq_add_stmt (seq
, stmt
);
9621 stmt
= gimple_build_assign (i
, op
, idest
, isrc
);
9622 gimple_seq_add_stmt (seq
, stmt
);
9624 else if (op
== MULT_EXPR
)
9626 /* Let x = a + ib = dest, y = c + id = src.
9627 x * y = (ac - bd) + i(ad + bc) */
9628 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9629 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9630 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9631 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9633 stmt
= gimple_build_assign (ac
, MULT_EXPR
, rdest
, rsrc
);
9634 gimple_seq_add_stmt (seq
, stmt
);
9636 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, isrc
);
9637 gimple_seq_add_stmt (seq
, stmt
);
9639 stmt
= gimple_build_assign (r
, MINUS_EXPR
, ac
, bd
);
9640 gimple_seq_add_stmt (seq
, stmt
);
9642 stmt
= gimple_build_assign (ad
, MULT_EXPR
, rdest
, isrc
);
9643 gimple_seq_add_stmt (seq
, stmt
);
9645 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, rsrc
);
9646 gimple_seq_add_stmt (seq
, stmt
);
9648 stmt
= gimple_build_assign (i
, PLUS_EXPR
, ad
, bc
);
9649 gimple_seq_add_stmt (seq
, stmt
);
9654 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
9655 gimplify_assign (dest
, result
, seq
);
9658 /* Helper function to initialize local data for the reduction arrays.
9659 The reduction arrays need to be placed inside the calling function
9660 for accelerators, or else the host won't be able to preform the final
9664 oacc_initialize_reduction_data (tree clauses
, tree nthreads
,
9665 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9671 /* Find the innermost OpenACC parallel context. */
9672 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
9673 && (gimple_omp_target_kind (ctx
->stmt
)
9674 == GF_OMP_TARGET_KIND_OACC_PARALLEL
))
9678 gcc_checking_assert (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
9679 && (gimple_omp_target_kind (octx
->stmt
)
9680 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
9682 /* Extract the clauses. */
9683 oc
= gimple_omp_target_clauses (octx
->stmt
);
9685 /* Find the last outer clause. */
9686 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
9689 /* Allocate arrays for each reduction variable. */
9690 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9692 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9695 tree var
= OMP_CLAUSE_DECL (c
);
9696 tree type
= get_base_type (var
);
9697 tree array
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
),
9701 /* Calculate size of the reduction array. */
9702 t
= create_tmp_var (TREE_TYPE (nthreads
));
9703 stmt
= gimple_build_assign (t
, MULT_EXPR
, nthreads
,
9704 fold_convert (TREE_TYPE (nthreads
),
9705 TYPE_SIZE_UNIT (type
)));
9706 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9708 size
= create_tmp_var (sizetype
);
9709 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
9711 /* Now allocate memory for it. */
9712 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
9713 stmt
= gimple_build_call (call
, 1, size
);
9714 gimple_call_set_lhs (stmt
, array
);
9715 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9717 /* Map this array into the accelerator. */
9719 /* Add the reduction array to the list of clauses. */
9721 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
9722 OMP_CLAUSE_SET_MAP_KIND (t
, GOMP_MAP_FORCE_FROM
);
9723 OMP_CLAUSE_DECL (t
) = x
;
9724 OMP_CLAUSE_CHAIN (t
) = NULL
;
9726 OMP_CLAUSE_CHAIN (oc
) = t
;
9728 gimple_omp_target_set_clauses (as_a
<gomp_target
*> (octx
->stmt
), t
);
9729 OMP_CLAUSE_SIZE (t
) = size
;
9734 /* Helper function to process the array of partial reductions. Nthreads
9735 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9736 cannot be used here, because nthreads on the host may be different than
9737 on the accelerator. */
9740 oacc_finalize_reduction_data (tree clauses
, tree nthreads
,
9741 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9743 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
9748 let var = the original reduction variable
9749 let array = reduction variable array
9751 for (i = 0; i < nthreads; i++)
9755 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
9756 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
9757 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
9759 /* Create and initialize an index variable. */
9760 tree ix
= create_tmp_var (sizetype
);
9761 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
9764 /* Insert the loop header label here. */
9765 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
9767 /* Exit loop if ix >= nthreads. */
9768 x
= create_tmp_var (sizetype
);
9769 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
9770 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
9771 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9773 /* Insert the loop body label here. */
9774 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
9776 /* Collapse each reduction array, one element at a time. */
9777 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9779 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9782 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
9784 /* reduction(-:var) sums up the partial results, so it acts
9785 identically to reduction(+:var). */
9786 if (reduction_code
== MINUS_EXPR
)
9787 reduction_code
= PLUS_EXPR
;
9789 /* Set up reduction variable var. */
9790 var
= OMP_CLAUSE_DECL (c
);
9791 type
= get_base_type (var
);
9792 array
= lookup_oacc_reduction (oacc_get_reduction_array_id
9793 (OMP_CLAUSE_DECL (c
)), ctx
);
9795 /* Calculate the array offset. */
9796 tree offset
= create_tmp_var (sizetype
);
9797 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
9798 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, ix
);
9799 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9801 tree ptr
= create_tmp_var (TREE_TYPE (array
));
9802 stmt
= gimple_build_assign (ptr
, POINTER_PLUS_EXPR
, array
, offset
);
9803 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9805 /* Extract array[ix] into mem. */
9806 tree mem
= create_tmp_var (type
);
9807 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
9809 /* Find the original reduction variable. */
9810 if (is_reference (var
))
9811 var
= build_simple_mem_ref (var
);
9813 tree t
= create_tmp_var (type
);
9815 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
9816 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9818 /* var = var op mem */
9819 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
9821 case TRUTH_ANDIF_EXPR
:
9822 case TRUTH_ORIF_EXPR
:
9823 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
9825 gimplify_and_add (t
, stmt_seqp
);
9828 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9829 oacc_gimple_assign (t
, OMP_CLAUSE_REDUCTION_CODE (c
), mem
,
9833 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
9834 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
9835 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9838 /* Increment the induction variable. */
9839 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
9840 stmt
= gimple_build_assign (ix
, PLUS_EXPR
, ix
, one
);
9841 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9843 /* Go back to the top of the loop. */
9844 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
9846 /* Place the loop exit label here. */
9847 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
9850 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9851 scan that for reductions. */
9854 oacc_process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
9855 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
9857 gimple_stmt_iterator gsi
;
9858 gimple_seq inner
= NULL
;
9860 /* A collapse clause may have inserted a new bind block. */
9861 gsi
= gsi_start (*body
);
9862 while (!gsi_end_p (gsi
))
9864 gimple stmt
= gsi_stmt (gsi
);
9865 if (gbind
*bind_stmt
= dyn_cast
<gbind
*> (stmt
))
9867 inner
= gimple_bind_body (bind_stmt
);
9869 gsi
= gsi_start (*body
);
9871 else if (dyn_cast
<gomp_for
*> (stmt
))
9877 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9879 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
9881 bool reduction_found
= false;
9883 gimple stmt
= gsi_stmt (gsi
);
9885 switch (gimple_code (stmt
))
9887 case GIMPLE_OMP_FOR
:
9888 clauses
= gimple_omp_for_clauses (stmt
);
9890 /* Search for a reduction clause. */
9891 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9892 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
9894 reduction_found
= true;
9898 if (!reduction_found
)
9901 ctx
= maybe_lookup_ctx (stmt
);
9904 /* Extract the number of threads. */
9905 nthreads
= create_tmp_var (sizetype
);
9906 t
= oacc_max_threads (ctx
);
9907 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
9909 /* Determine if this is kernel will be executed on the host. */
9910 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
9911 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
9912 stmt
= gimple_build_call (call
, 0);
9913 gimple_call_set_lhs (stmt
, acc_device
);
9914 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9916 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9917 acc_device_host
= create_tmp_var (integer_type_node
,
9918 ".acc_device_host");
9919 gimplify_assign (acc_device_host
,
9920 build_int_cst (integer_type_node
,
9924 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9925 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9927 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9929 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9930 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9931 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9934 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9936 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9937 gimplify_assign (acc_device_host
,
9938 build_int_cst (integer_type_node
,
9939 GOMP_DEVICE_HOST_NONSHM
),
9942 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9943 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9945 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9947 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9948 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9949 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9952 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9954 oacc_initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
,
9956 oacc_finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
9959 // Scan for other directives which support reduction here.
9965 /* If ctx is a worksharing context inside of a cancellable parallel
9966 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
9967 and conditional branch to parallel's cancel_label to handle
9968 cancellation in the implicit barrier. */
9971 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
9973 gimple omp_return
= gimple_seq_last_stmt (*body
);
9974 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
9975 if (gimple_omp_return_nowait_p (omp_return
))
9978 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
9979 && ctx
->outer
->cancellable
)
9981 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
9982 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
9983 tree lhs
= create_tmp_var (c_bool_type
);
9984 gimple_omp_return_set_lhs (omp_return
, lhs
);
9985 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
9986 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
9987 fold_convert (c_bool_type
,
9988 boolean_false_node
),
9989 ctx
->outer
->cancel_label
, fallthru_label
);
9990 gimple_seq_add_stmt (body
, g
);
9991 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
9995 /* Lower the OpenMP sections directive in the current statement in GSI_P.
9996 CTX is the enclosing OMP context for the current statement. */
9999 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10001 tree block
, control
;
10002 gimple_stmt_iterator tgsi
;
10003 gomp_sections
*stmt
;
10005 gbind
*new_stmt
, *bind
;
10006 gimple_seq ilist
, dlist
, olist
, new_body
;
10008 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
10010 push_gimplify_context ();
10014 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
10015 &ilist
, &dlist
, ctx
, NULL
);
10017 new_body
= gimple_omp_body (stmt
);
10018 gimple_omp_set_body (stmt
, NULL
);
10019 tgsi
= gsi_start (new_body
);
10020 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
10025 sec_start
= gsi_stmt (tgsi
);
10026 sctx
= maybe_lookup_ctx (sec_start
);
10029 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
10030 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
10031 GSI_CONTINUE_LINKING
);
10032 gimple_omp_set_body (sec_start
, NULL
);
10034 if (gsi_one_before_end_p (tgsi
))
10036 gimple_seq l
= NULL
;
10037 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
10039 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
10040 gimple_omp_section_set_last (sec_start
);
10043 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
10044 GSI_CONTINUE_LINKING
);
10047 block
= make_node (BLOCK
);
10048 bind
= gimple_build_bind (NULL
, new_body
, block
);
10051 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
10053 block
= make_node (BLOCK
);
10054 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10055 gsi_replace (gsi_p
, new_stmt
, true);
10057 pop_gimplify_context (new_stmt
);
10058 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10059 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10060 if (BLOCK_VARS (block
))
10061 TREE_USED (block
) = 1;
10064 gimple_seq_add_seq (&new_body
, ilist
);
10065 gimple_seq_add_stmt (&new_body
, stmt
);
10066 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
10067 gimple_seq_add_stmt (&new_body
, bind
);
10069 control
= create_tmp_var (unsigned_type_node
, ".section");
10070 t
= gimple_build_omp_continue (control
, control
);
10071 gimple_omp_sections_set_control (stmt
, control
);
10072 gimple_seq_add_stmt (&new_body
, t
);
10074 gimple_seq_add_seq (&new_body
, olist
);
10075 if (ctx
->cancellable
)
10076 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10077 gimple_seq_add_seq (&new_body
, dlist
);
10079 new_body
= maybe_catch_exception (new_body
);
10081 t
= gimple_build_omp_return
10082 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
10083 OMP_CLAUSE_NOWAIT
));
10084 gimple_seq_add_stmt (&new_body
, t
);
10085 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
10087 gimple_bind_set_body (new_stmt
, new_body
);
10091 /* A subroutine of lower_omp_single. Expand the simple form of
10092 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10094 if (GOMP_single_start ())
10096 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10098 FIXME. It may be better to delay expanding the logic of this until
10099 pass_expand_omp. The expanded logic may make the job more difficult
10100 to a synchronization analysis pass. */
10103 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
10105 location_t loc
= gimple_location (single_stmt
);
10106 tree tlabel
= create_artificial_label (loc
);
10107 tree flabel
= create_artificial_label (loc
);
10111 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
10112 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
10113 call
= gimple_build_call (decl
, 0);
10114 gimple_call_set_lhs (call
, lhs
);
10115 gimple_seq_add_stmt (pre_p
, call
);
10117 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
10118 fold_convert_loc (loc
, TREE_TYPE (lhs
),
10119 boolean_true_node
),
10121 gimple_seq_add_stmt (pre_p
, cond
);
10122 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
10123 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10124 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
10128 /* A subroutine of lower_omp_single. Expand the simple form of
10129 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10131 #pragma omp single copyprivate (a, b, c)
10133 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10136 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10142 GOMP_single_copy_end (©out);
10153 FIXME. It may be better to delay expanding the logic of this until
10154 pass_expand_omp. The expanded logic may make the job more difficult
10155 to a synchronization analysis pass. */
10158 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
10161 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
10162 gimple_seq copyin_seq
;
10163 location_t loc
= gimple_location (single_stmt
);
10165 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
10167 ptr_type
= build_pointer_type (ctx
->record_type
);
10168 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
10170 l0
= create_artificial_label (loc
);
10171 l1
= create_artificial_label (loc
);
10172 l2
= create_artificial_label (loc
);
10174 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
10175 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
10176 t
= fold_convert_loc (loc
, ptr_type
, t
);
10177 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
10179 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
10180 build_int_cst (ptr_type
, 0));
10181 t
= build3 (COND_EXPR
, void_type_node
, t
,
10182 build_and_jump (&l0
), build_and_jump (&l1
));
10183 gimplify_and_add (t
, pre_p
);
10185 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
10187 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10190 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
10193 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10194 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
10195 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
10196 gimplify_and_add (t
, pre_p
);
10198 t
= build_and_jump (&l2
);
10199 gimplify_and_add (t
, pre_p
);
10201 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
10203 gimple_seq_add_seq (pre_p
, copyin_seq
);
10205 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
10209 /* Expand code for an OpenMP single directive. */
10212 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10216 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
10218 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
10220 push_gimplify_context ();
10222 block
= make_node (BLOCK
);
10223 bind
= gimple_build_bind (NULL
, NULL
, block
);
10224 gsi_replace (gsi_p
, bind
, true);
10227 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
10228 &bind_body
, &dlist
, ctx
, NULL
);
10229 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
10231 gimple_seq_add_stmt (&bind_body
, single_stmt
);
10233 if (ctx
->record_type
)
10234 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
10236 lower_omp_single_simple (single_stmt
, &bind_body
);
10238 gimple_omp_set_body (single_stmt
, NULL
);
10240 gimple_seq_add_seq (&bind_body
, dlist
);
10242 bind_body
= maybe_catch_exception (bind_body
);
10244 t
= gimple_build_omp_return
10245 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
10246 OMP_CLAUSE_NOWAIT
));
10247 gimple_seq_add_stmt (&bind_body_tail
, t
);
10248 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
10249 if (ctx
->record_type
)
10251 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
10252 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10253 TREE_THIS_VOLATILE (clobber
) = 1;
10254 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
10255 clobber
), GSI_SAME_STMT
);
10257 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
10258 gimple_bind_set_body (bind
, bind_body
);
10260 pop_gimplify_context (bind
);
10262 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10263 BLOCK_VARS (block
) = ctx
->block_vars
;
10264 if (BLOCK_VARS (block
))
10265 TREE_USED (block
) = 1;
10269 /* Expand code for an OpenMP master directive. */
10272 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10274 tree block
, lab
= NULL
, x
, bfn_decl
;
10275 gimple stmt
= gsi_stmt (*gsi_p
);
10277 location_t loc
= gimple_location (stmt
);
10280 push_gimplify_context ();
10282 block
= make_node (BLOCK
);
10283 bind
= gimple_build_bind (NULL
, NULL
, block
);
10284 gsi_replace (gsi_p
, bind
, true);
10285 gimple_bind_add_stmt (bind
, stmt
);
10287 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
10288 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
10289 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
10290 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
10292 gimplify_and_add (x
, &tseq
);
10293 gimple_bind_add_seq (bind
, tseq
);
10295 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10296 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10297 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10298 gimple_omp_set_body (stmt
, NULL
);
10300 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
10302 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10304 pop_gimplify_context (bind
);
10306 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10307 BLOCK_VARS (block
) = ctx
->block_vars
;
10311 /* Expand code for an OpenMP taskgroup directive. */
10314 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10316 gimple stmt
= gsi_stmt (*gsi_p
);
10319 tree block
= make_node (BLOCK
);
10321 bind
= gimple_build_bind (NULL
, NULL
, block
);
10322 gsi_replace (gsi_p
, bind
, true);
10323 gimple_bind_add_stmt (bind
, stmt
);
10325 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
10327 gimple_bind_add_stmt (bind
, x
);
10329 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10330 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10331 gimple_omp_set_body (stmt
, NULL
);
10333 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10335 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10336 BLOCK_VARS (block
) = ctx
->block_vars
;
10340 /* Expand code for an OpenMP ordered directive. */
10343 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10346 gimple stmt
= gsi_stmt (*gsi_p
);
10350 push_gimplify_context ();
10352 block
= make_node (BLOCK
);
10353 bind
= gimple_build_bind (NULL
, NULL
, block
);
10354 gsi_replace (gsi_p
, bind
, true);
10355 gimple_bind_add_stmt (bind
, stmt
);
10357 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
10359 gimple_bind_add_stmt (bind
, x
);
10361 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10362 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10363 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10364 gimple_omp_set_body (stmt
, NULL
);
10366 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
10367 gimple_bind_add_stmt (bind
, x
);
10369 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10371 pop_gimplify_context (bind
);
10373 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10374 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10378 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10379 substitution of a couple of function calls. But in the NAMED case,
10380 requires that languages coordinate a symbol name. It is therefore
10381 best put here in common code. */
10383 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
10386 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10389 tree name
, lock
, unlock
;
10390 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
10392 location_t loc
= gimple_location (stmt
);
10395 name
= gimple_omp_critical_name (stmt
);
10400 if (!critical_name_mutexes
)
10401 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
10403 tree
*n
= critical_name_mutexes
->get (name
);
10408 decl
= create_tmp_var_raw (ptr_type_node
);
10410 new_str
= ACONCAT ((".gomp_critical_user_",
10411 IDENTIFIER_POINTER (name
), NULL
));
10412 DECL_NAME (decl
) = get_identifier (new_str
);
10413 TREE_PUBLIC (decl
) = 1;
10414 TREE_STATIC (decl
) = 1;
10415 DECL_COMMON (decl
) = 1;
10416 DECL_ARTIFICIAL (decl
) = 1;
10417 DECL_IGNORED_P (decl
) = 1;
10419 varpool_node::finalize_decl (decl
);
10421 critical_name_mutexes
->put (name
, decl
);
10426 /* If '#pragma omp critical' is inside offloaded region or
10427 inside function marked as offloadable, the symbol must be
10428 marked as offloadable too. */
10430 if (cgraph_node::get (current_function_decl
)->offloadable
)
10431 varpool_node::get_create (decl
)->offloadable
= 1;
10433 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
10434 if (is_gimple_omp_offloaded (octx
->stmt
))
10436 varpool_node::get_create (decl
)->offloadable
= 1;
10440 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
10441 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
10443 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
10444 unlock
= build_call_expr_loc (loc
, unlock
, 1,
10445 build_fold_addr_expr_loc (loc
, decl
));
10449 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
10450 lock
= build_call_expr_loc (loc
, lock
, 0);
10452 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
10453 unlock
= build_call_expr_loc (loc
, unlock
, 0);
10456 push_gimplify_context ();
10458 block
= make_node (BLOCK
);
10459 bind
= gimple_build_bind (NULL
, NULL
, block
);
10460 gsi_replace (gsi_p
, bind
, true);
10461 gimple_bind_add_stmt (bind
, stmt
);
10463 tbody
= gimple_bind_body (bind
);
10464 gimplify_and_add (lock
, &tbody
);
10465 gimple_bind_set_body (bind
, tbody
);
10467 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10468 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10469 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10470 gimple_omp_set_body (stmt
, NULL
);
10472 tbody
= gimple_bind_body (bind
);
10473 gimplify_and_add (unlock
, &tbody
);
10474 gimple_bind_set_body (bind
, tbody
);
10476 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10478 pop_gimplify_context (bind
);
10479 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10480 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10484 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10485 for a lastprivate clause. Given a loop control predicate of (V
10486 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10487 is appended to *DLIST, iterator initialization is appended to
10491 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
10492 gimple_seq
*dlist
, struct omp_context
*ctx
)
10494 tree clauses
, cond
, vinit
;
10495 enum tree_code cond_code
;
10498 cond_code
= fd
->loop
.cond_code
;
10499 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
10501 /* When possible, use a strict equality expression. This can let VRP
10502 type optimizations deduce the value and remove a copy. */
10503 if (tree_fits_shwi_p (fd
->loop
.step
))
10505 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
10506 if (step
== 1 || step
== -1)
10507 cond_code
= EQ_EXPR
;
10510 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
10512 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
10514 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
10515 if (!gimple_seq_empty_p (stmts
))
10517 gimple_seq_add_seq (&stmts
, *dlist
);
10520 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10521 vinit
= fd
->loop
.n1
;
10522 if (cond_code
== EQ_EXPR
10523 && tree_fits_shwi_p (fd
->loop
.n2
)
10524 && ! integer_zerop (fd
->loop
.n2
))
10525 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
10527 vinit
= unshare_expr (vinit
);
10529 /* Initialize the iterator variable, so that threads that don't execute
10530 any iterations don't execute the lastprivate clauses by accident. */
10531 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
10536 /* Lower code for an OMP loop directive. */
10539 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10541 tree
*rhs_p
, block
;
10542 struct omp_for_data fd
, *fdp
= NULL
;
10543 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
10545 gimple_seq omp_for_body
, body
, dlist
;
10548 push_gimplify_context ();
10550 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
10552 block
= make_node (BLOCK
);
10553 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10554 /* Replace at gsi right away, so that 'stmt' is no member
10555 of a sequence anymore as we're going to add to to a different
10557 gsi_replace (gsi_p
, new_stmt
, true);
10559 /* Move declaration of temporaries in the loop body before we make
10561 omp_for_body
= gimple_omp_body (stmt
);
10562 if (!gimple_seq_empty_p (omp_for_body
)
10563 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
10566 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
10567 tree vars
= gimple_bind_vars (inner_bind
);
10568 gimple_bind_append_vars (new_stmt
, vars
);
10569 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10570 keep them on the inner_bind and it's block. */
10571 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
10572 if (gimple_bind_block (inner_bind
))
10573 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
10576 if (gimple_omp_for_combined_into_p (stmt
))
10578 extract_omp_for_data (stmt
, &fd
, NULL
);
10581 /* We need two temporaries with fd.loop.v type (istart/iend)
10582 and then (fd.collapse - 1) temporaries with the same
10583 type for count2 ... countN-1 vars if not constant. */
10585 tree type
= fd
.iter_type
;
10586 if (fd
.collapse
> 1
10587 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
10588 count
+= fd
.collapse
- 1;
10589 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
10590 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
10591 tree clauses
= *pc
;
10594 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
10595 OMP_CLAUSE__LOOPTEMP_
);
10596 for (i
= 0; i
< count
; i
++)
10601 gcc_assert (outerc
);
10602 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
10603 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
10604 OMP_CLAUSE__LOOPTEMP_
);
10608 temp
= create_tmp_var (type
);
10609 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
10611 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
10612 OMP_CLAUSE_DECL (*pc
) = temp
;
10613 pc
= &OMP_CLAUSE_CHAIN (*pc
);
10618 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10621 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
10623 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
10625 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10627 /* Lower the header expressions. At this point, we can assume that
10628 the header is of the form:
10630 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10632 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10633 using the .omp_data_s mapping, if needed. */
10634 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
10636 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
10637 if (!is_gimple_min_invariant (*rhs_p
))
10638 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10640 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
10641 if (!is_gimple_min_invariant (*rhs_p
))
10642 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10644 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
10645 if (!is_gimple_min_invariant (*rhs_p
))
10646 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10649 /* Once lowered, extract the bounds and clauses. */
10650 extract_omp_for_data (stmt
, &fd
, NULL
);
10652 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
10654 gimple_seq_add_stmt (&body
, stmt
);
10655 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
10657 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
10660 /* After the loop, add exit clauses. */
10661 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
10663 if (ctx
->cancellable
)
10664 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
10666 gimple_seq_add_seq (&body
, dlist
);
10668 body
= maybe_catch_exception (body
);
10670 /* Region exit marker goes at the end of the loop body. */
10671 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
10672 maybe_add_implicit_barrier_cancel (ctx
, &body
);
10673 pop_gimplify_context (new_stmt
);
10675 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10676 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
10677 if (BLOCK_VARS (block
))
10678 TREE_USED (block
) = 1;
10680 gimple_bind_set_body (new_stmt
, body
);
10681 gimple_omp_set_body (stmt
, NULL
);
10682 gimple_omp_for_set_pre_body (stmt
, NULL
);
10685 /* Callback for walk_stmts. Check if the current statement only contains
10686 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10689 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
10690 bool *handled_ops_p
,
10691 struct walk_stmt_info
*wi
)
10693 int *info
= (int *) wi
->info
;
10694 gimple stmt
= gsi_stmt (*gsi_p
);
10696 *handled_ops_p
= true;
10697 switch (gimple_code (stmt
))
10701 case GIMPLE_OMP_FOR
:
10702 case GIMPLE_OMP_SECTIONS
:
10703 *info
= *info
== 0 ? 1 : -1;
10712 struct omp_taskcopy_context
10714 /* This field must be at the beginning, as we do "inheritance": Some
10715 callback functions for tree-inline.c (e.g., omp_copy_decl)
10716 receive a copy_body_data pointer that is up-casted to an
10717 omp_context pointer. */
10723 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
10725 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
10727 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
10728 return create_tmp_var (TREE_TYPE (var
));
10734 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
10736 tree name
, new_fields
= NULL
, type
, f
;
10738 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
10739 name
= DECL_NAME (TYPE_NAME (orig_type
));
10740 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
10741 TYPE_DECL
, name
, type
);
10742 TYPE_NAME (type
) = name
;
10744 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
10746 tree new_f
= copy_node (f
);
10747 DECL_CONTEXT (new_f
) = type
;
10748 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
10749 TREE_CHAIN (new_f
) = new_fields
;
10750 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10751 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10752 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
10754 new_fields
= new_f
;
10755 tcctx
->cb
.decl_map
->put (f
, new_f
);
10757 TYPE_FIELDS (type
) = nreverse (new_fields
);
10758 layout_type (type
);
10762 /* Create task copyfn. */
10765 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
10767 struct function
*child_cfun
;
10768 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
10769 tree record_type
, srecord_type
, bind
, list
;
10770 bool record_needs_remap
= false, srecord_needs_remap
= false;
10772 struct omp_taskcopy_context tcctx
;
10773 location_t loc
= gimple_location (task_stmt
);
10775 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
10776 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
10777 gcc_assert (child_cfun
->cfg
== NULL
);
10778 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
10780 /* Reset DECL_CONTEXT on function arguments. */
10781 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
10782 DECL_CONTEXT (t
) = child_fn
;
10784 /* Populate the function. */
10785 push_gimplify_context ();
10786 push_cfun (child_cfun
);
10788 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
10789 TREE_SIDE_EFFECTS (bind
) = 1;
10791 DECL_SAVED_TREE (child_fn
) = bind
;
10792 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
10794 /* Remap src and dst argument types if needed. */
10795 record_type
= ctx
->record_type
;
10796 srecord_type
= ctx
->srecord_type
;
10797 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
10798 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10800 record_needs_remap
= true;
10803 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
10804 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10806 srecord_needs_remap
= true;
10810 if (record_needs_remap
|| srecord_needs_remap
)
10812 memset (&tcctx
, '\0', sizeof (tcctx
));
10813 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
10814 tcctx
.cb
.dst_fn
= child_fn
;
10815 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
10816 gcc_checking_assert (tcctx
.cb
.src_node
);
10817 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
10818 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
10819 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
10820 tcctx
.cb
.eh_lp_nr
= 0;
10821 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
10822 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
10825 if (record_needs_remap
)
10826 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
10827 if (srecord_needs_remap
)
10828 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
10831 tcctx
.cb
.decl_map
= NULL
;
10833 arg
= DECL_ARGUMENTS (child_fn
);
10834 TREE_TYPE (arg
) = build_pointer_type (record_type
);
10835 sarg
= DECL_CHAIN (arg
);
10836 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
10838 /* First pass: initialize temporaries used in record_type and srecord_type
10839 sizes and field offsets. */
10840 if (tcctx
.cb
.decl_map
)
10841 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10842 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10846 decl
= OMP_CLAUSE_DECL (c
);
10847 p
= tcctx
.cb
.decl_map
->get (decl
);
10850 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10851 sf
= (tree
) n
->value
;
10852 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10853 src
= build_simple_mem_ref_loc (loc
, sarg
);
10854 src
= omp_build_component_ref (src
, sf
);
10855 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
10856 append_to_statement_list (t
, &list
);
10859 /* Second pass: copy shared var pointers and copy construct non-VLA
10860 firstprivate vars. */
10861 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10862 switch (OMP_CLAUSE_CODE (c
))
10864 case OMP_CLAUSE_SHARED
:
10865 decl
= OMP_CLAUSE_DECL (c
);
10866 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10869 f
= (tree
) n
->value
;
10870 if (tcctx
.cb
.decl_map
)
10871 f
= *tcctx
.cb
.decl_map
->get (f
);
10872 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10873 sf
= (tree
) n
->value
;
10874 if (tcctx
.cb
.decl_map
)
10875 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10876 src
= build_simple_mem_ref_loc (loc
, sarg
);
10877 src
= omp_build_component_ref (src
, sf
);
10878 dst
= build_simple_mem_ref_loc (loc
, arg
);
10879 dst
= omp_build_component_ref (dst
, f
);
10880 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10881 append_to_statement_list (t
, &list
);
10883 case OMP_CLAUSE_FIRSTPRIVATE
:
10884 decl
= OMP_CLAUSE_DECL (c
);
10885 if (is_variable_sized (decl
))
10887 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10890 f
= (tree
) n
->value
;
10891 if (tcctx
.cb
.decl_map
)
10892 f
= *tcctx
.cb
.decl_map
->get (f
);
10893 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10896 sf
= (tree
) n
->value
;
10897 if (tcctx
.cb
.decl_map
)
10898 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10899 src
= build_simple_mem_ref_loc (loc
, sarg
);
10900 src
= omp_build_component_ref (src
, sf
);
10901 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
10902 src
= build_simple_mem_ref_loc (loc
, src
);
10906 dst
= build_simple_mem_ref_loc (loc
, arg
);
10907 dst
= omp_build_component_ref (dst
, f
);
10908 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10909 append_to_statement_list (t
, &list
);
10911 case OMP_CLAUSE_PRIVATE
:
10912 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
10914 decl
= OMP_CLAUSE_DECL (c
);
10915 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10916 f
= (tree
) n
->value
;
10917 if (tcctx
.cb
.decl_map
)
10918 f
= *tcctx
.cb
.decl_map
->get (f
);
10919 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10922 sf
= (tree
) n
->value
;
10923 if (tcctx
.cb
.decl_map
)
10924 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10925 src
= build_simple_mem_ref_loc (loc
, sarg
);
10926 src
= omp_build_component_ref (src
, sf
);
10927 if (use_pointer_for_field (decl
, NULL
))
10928 src
= build_simple_mem_ref_loc (loc
, src
);
10932 dst
= build_simple_mem_ref_loc (loc
, arg
);
10933 dst
= omp_build_component_ref (dst
, f
);
10934 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10935 append_to_statement_list (t
, &list
);
10941 /* Last pass: handle VLA firstprivates. */
10942 if (tcctx
.cb
.decl_map
)
10943 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10944 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10948 decl
= OMP_CLAUSE_DECL (c
);
10949 if (!is_variable_sized (decl
))
10951 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10954 f
= (tree
) n
->value
;
10955 f
= *tcctx
.cb
.decl_map
->get (f
);
10956 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
10957 ind
= DECL_VALUE_EXPR (decl
);
10958 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
10959 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
10960 n
= splay_tree_lookup (ctx
->sfield_map
,
10961 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10962 sf
= (tree
) n
->value
;
10963 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10964 src
= build_simple_mem_ref_loc (loc
, sarg
);
10965 src
= omp_build_component_ref (src
, sf
);
10966 src
= build_simple_mem_ref_loc (loc
, src
);
10967 dst
= build_simple_mem_ref_loc (loc
, arg
);
10968 dst
= omp_build_component_ref (dst
, f
);
10969 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10970 append_to_statement_list (t
, &list
);
10971 n
= splay_tree_lookup (ctx
->field_map
,
10972 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10973 df
= (tree
) n
->value
;
10974 df
= *tcctx
.cb
.decl_map
->get (df
);
10975 ptr
= build_simple_mem_ref_loc (loc
, arg
);
10976 ptr
= omp_build_component_ref (ptr
, df
);
10977 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
10978 build_fold_addr_expr_loc (loc
, dst
));
10979 append_to_statement_list (t
, &list
);
10982 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
10983 append_to_statement_list (t
, &list
);
10985 if (tcctx
.cb
.decl_map
)
10986 delete tcctx
.cb
.decl_map
;
10987 pop_gimplify_context (NULL
);
10988 BIND_EXPR_BODY (bind
) = list
;
10993 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
10997 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
10999 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
11000 OMP_CLAUSE_DEPEND
);
11001 gcc_assert (clauses
);
11002 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11003 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
11004 switch (OMP_CLAUSE_DEPEND_KIND (c
))
11006 case OMP_CLAUSE_DEPEND_IN
:
11009 case OMP_CLAUSE_DEPEND_OUT
:
11010 case OMP_CLAUSE_DEPEND_INOUT
:
11014 gcc_unreachable ();
11016 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
11017 tree array
= create_tmp_var (type
);
11018 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
11020 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
11021 gimple_seq_add_stmt (iseq
, g
);
11022 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
11024 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
11025 gimple_seq_add_stmt (iseq
, g
);
11026 for (i
= 0; i
< 2; i
++)
11028 if ((i
? n_in
: n_out
) == 0)
11030 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11031 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
11032 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
11034 tree t
= OMP_CLAUSE_DECL (c
);
11035 t
= fold_convert (ptr_type_node
, t
);
11036 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
11037 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
11038 NULL_TREE
, NULL_TREE
);
11039 g
= gimple_build_assign (r
, t
);
11040 gimple_seq_add_stmt (iseq
, g
);
11043 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
11044 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
11045 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
11046 OMP_CLAUSE_CHAIN (c
) = *p
;
11048 tree clobber
= build_constructor (type
, NULL
);
11049 TREE_THIS_VOLATILE (clobber
) = 1;
11050 g
= gimple_build_assign (array
, clobber
);
11051 gimple_seq_add_stmt (oseq
, g
);
11054 /* Lower the OpenMP parallel or task directive in the current statement
11055 in GSI_P. CTX holds context information for the directive. */
11058 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11062 gimple stmt
= gsi_stmt (*gsi_p
);
11063 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
11064 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
11065 location_t loc
= gimple_location (stmt
);
11067 clauses
= gimple_omp_taskreg_clauses (stmt
);
11069 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
11070 par_body
= gimple_bind_body (par_bind
);
11071 child_fn
= ctx
->cb
.dst_fn
;
11072 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11073 && !gimple_omp_parallel_combined_p (stmt
))
11075 struct walk_stmt_info wi
;
11078 memset (&wi
, 0, sizeof (wi
));
11080 wi
.val_only
= true;
11081 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
11083 gimple_omp_parallel_set_combined_p (stmt
, true);
11085 gimple_seq dep_ilist
= NULL
;
11086 gimple_seq dep_olist
= NULL
;
11087 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11088 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
11090 push_gimplify_context ();
11091 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11092 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
11095 if (ctx
->srecord_type
)
11096 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
11098 push_gimplify_context ();
11103 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
11104 lower_omp (&par_body
, ctx
);
11105 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
11106 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
11108 /* Declare all the variables created by mapping and the variables
11109 declared in the scope of the parallel body. */
11110 record_vars_into (ctx
->block_vars
, child_fn
);
11111 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
11113 if (ctx
->record_type
)
11116 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
11117 : ctx
->record_type
, ".omp_data_o");
11118 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11119 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11120 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
11125 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
11126 lower_send_shared_vars (&ilist
, &olist
, ctx
);
11128 if (ctx
->record_type
)
11130 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
11131 TREE_THIS_VOLATILE (clobber
) = 1;
11132 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11136 /* Once all the expansions are done, sequence all the different
11137 fragments inside gimple_omp_body. */
11141 if (ctx
->record_type
)
11143 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11144 /* fixup_child_record_type might have changed receiver_decl's type. */
11145 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11146 gimple_seq_add_stmt (&new_body
,
11147 gimple_build_assign (ctx
->receiver_decl
, t
));
11150 gimple_seq_add_seq (&new_body
, par_ilist
);
11151 gimple_seq_add_seq (&new_body
, par_body
);
11152 gimple_seq_add_seq (&new_body
, par_rlist
);
11153 if (ctx
->cancellable
)
11154 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
11155 gimple_seq_add_seq (&new_body
, par_olist
);
11156 new_body
= maybe_catch_exception (new_body
);
11157 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11158 gimple_omp_set_body (stmt
, new_body
);
11160 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
11161 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
11162 gimple_bind_add_seq (bind
, ilist
);
11163 gimple_bind_add_stmt (bind
, stmt
);
11164 gimple_bind_add_seq (bind
, olist
);
11166 pop_gimplify_context (NULL
);
11170 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11171 gimple_bind_add_stmt (dep_bind
, bind
);
11172 gimple_bind_add_seq (dep_bind
, dep_olist
);
11173 pop_gimplify_context (dep_bind
);
11177 /* Lower the GIMPLE_OMP_TARGET in the current statement
11178 in GSI_P. CTX holds context information for the directive. */
11181 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11184 tree child_fn
, t
, c
;
11185 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
11186 gbind
*tgt_bind
, *bind
;
11187 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
11188 location_t loc
= gimple_location (stmt
);
11189 bool offloaded
, data_region
;
11190 unsigned int map_cnt
= 0;
11192 offloaded
= is_gimple_omp_offloaded (stmt
);
11193 switch (gimple_omp_target_kind (stmt
))
11195 case GF_OMP_TARGET_KIND_REGION
:
11196 case GF_OMP_TARGET_KIND_UPDATE
:
11197 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11198 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11199 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11200 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11201 data_region
= false;
11203 case GF_OMP_TARGET_KIND_DATA
:
11204 case GF_OMP_TARGET_KIND_OACC_DATA
:
11205 data_region
= true;
11208 gcc_unreachable ();
11211 clauses
= gimple_omp_target_clauses (stmt
);
11217 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
11218 tgt_body
= gimple_bind_body (tgt_bind
);
11220 else if (data_region
)
11221 tgt_body
= gimple_omp_body (stmt
);
11222 child_fn
= ctx
->cb
.dst_fn
;
11224 push_gimplify_context ();
11229 && is_gimple_omp_oacc (stmt
))
11230 oacc_process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
11232 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11233 switch (OMP_CLAUSE_CODE (c
))
11239 case OMP_CLAUSE_MAP
:
11240 #ifdef ENABLE_CHECKING
11241 /* First check what we're prepared to handle in the following. */
11242 switch (OMP_CLAUSE_MAP_KIND (c
))
11244 case GOMP_MAP_ALLOC
:
11246 case GOMP_MAP_FROM
:
11247 case GOMP_MAP_TOFROM
:
11248 case GOMP_MAP_POINTER
:
11249 case GOMP_MAP_TO_PSET
:
11251 case GOMP_MAP_FORCE_ALLOC
:
11252 case GOMP_MAP_FORCE_TO
:
11253 case GOMP_MAP_FORCE_FROM
:
11254 case GOMP_MAP_FORCE_TOFROM
:
11255 case GOMP_MAP_FORCE_PRESENT
:
11256 case GOMP_MAP_FORCE_DEALLOC
:
11257 case GOMP_MAP_FORCE_DEVICEPTR
:
11258 gcc_assert (is_gimple_omp_oacc (stmt
));
11261 gcc_unreachable ();
11265 case OMP_CLAUSE_TO
:
11266 case OMP_CLAUSE_FROM
:
11267 var
= OMP_CLAUSE_DECL (c
);
11270 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11271 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11276 if (DECL_SIZE (var
)
11277 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
11279 tree var2
= DECL_VALUE_EXPR (var
);
11280 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
11281 var2
= TREE_OPERAND (var2
, 0);
11282 gcc_assert (DECL_P (var2
));
11286 if (!maybe_lookup_field (var
, ctx
))
11291 x
= build_receiver_ref (var
, true, ctx
);
11292 tree new_var
= lookup_decl (var
, ctx
);
11293 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11294 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11295 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11296 x
= build_simple_mem_ref (x
);
11297 SET_DECL_VALUE_EXPR (new_var
, x
);
11298 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11305 target_nesting_level
++;
11306 lower_omp (&tgt_body
, ctx
);
11307 target_nesting_level
--;
11309 else if (data_region
)
11310 lower_omp (&tgt_body
, ctx
);
11314 /* Declare all the variables created by mapping and the variables
11315 declared in the scope of the target body. */
11316 record_vars_into (ctx
->block_vars
, child_fn
);
11317 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
11322 if (ctx
->record_type
)
11325 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
11326 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11327 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11328 t
= make_tree_vec (3);
11329 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
11330 TREE_VEC_ELT (t
, 1)
11331 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
11332 ".omp_data_sizes");
11333 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
11334 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
11335 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
11338 if (is_gimple_omp_oacc (stmt
))
11340 tkind_type
= short_unsigned_type_node
;
11345 tkind_type
= unsigned_char_type_node
;
11348 TREE_VEC_ELT (t
, 2)
11349 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
11350 ".omp_data_kinds");
11351 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
11352 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
11353 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
11354 gimple_omp_target_set_data_arg (stmt
, t
);
11356 vec
<constructor_elt
, va_gc
> *vsize
;
11357 vec
<constructor_elt
, va_gc
> *vkind
;
11358 vec_alloc (vsize
, map_cnt
);
11359 vec_alloc (vkind
, map_cnt
);
11360 unsigned int map_idx
= 0;
11362 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11363 switch (OMP_CLAUSE_CODE (c
))
11369 case OMP_CLAUSE_MAP
:
11370 case OMP_CLAUSE_TO
:
11371 case OMP_CLAUSE_FROM
:
11373 ovar
= OMP_CLAUSE_DECL (c
);
11374 if (!DECL_P (ovar
))
11376 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11377 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11379 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
11380 == get_base_address (ovar
));
11381 nc
= OMP_CLAUSE_CHAIN (c
);
11382 ovar
= OMP_CLAUSE_DECL (nc
);
11386 tree x
= build_sender_ref (ovar
, ctx
);
11388 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
11389 gimplify_assign (x
, v
, &ilist
);
11395 if (DECL_SIZE (ovar
)
11396 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
11398 tree ovar2
= DECL_VALUE_EXPR (ovar
);
11399 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
11400 ovar2
= TREE_OPERAND (ovar2
, 0);
11401 gcc_assert (DECL_P (ovar2
));
11404 if (!maybe_lookup_field (ovar
, ctx
))
11408 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
11409 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
11410 talign
= DECL_ALIGN_UNIT (ovar
);
11413 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11414 tree x
= build_sender_ref (ovar
, ctx
);
11415 if (maybe_lookup_oacc_reduction (var
, ctx
))
11417 gcc_checking_assert (offloaded
11418 && is_gimple_omp_oacc (stmt
));
11419 gimplify_assign (x
, var
, &ilist
);
11421 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11422 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11423 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11424 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
11426 gcc_assert (offloaded
);
11428 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
11429 mark_addressable (avar
);
11430 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
11431 talign
= DECL_ALIGN_UNIT (avar
);
11432 avar
= build_fold_addr_expr (avar
);
11433 gimplify_assign (x
, avar
, &ilist
);
11435 else if (is_gimple_reg (var
))
11437 gcc_assert (offloaded
);
11438 tree avar
= create_tmp_var (TREE_TYPE (var
));
11439 mark_addressable (avar
);
11440 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
11441 if (GOMP_MAP_COPY_TO_P (map_kind
)
11442 || map_kind
== GOMP_MAP_POINTER
11443 || map_kind
== GOMP_MAP_TO_PSET
11444 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11445 gimplify_assign (avar
, var
, &ilist
);
11446 avar
= build_fold_addr_expr (avar
);
11447 gimplify_assign (x
, avar
, &ilist
);
11448 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
11449 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11450 && !TYPE_READONLY (TREE_TYPE (var
)))
11452 x
= build_sender_ref (ovar
, ctx
);
11453 x
= build_simple_mem_ref (x
);
11454 gimplify_assign (var
, x
, &olist
);
11459 var
= build_fold_addr_expr (var
);
11460 gimplify_assign (x
, var
, &ilist
);
11463 tree s
= OMP_CLAUSE_SIZE (c
);
11464 if (s
== NULL_TREE
)
11465 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11466 s
= fold_convert (size_type_node
, s
);
11467 tree purpose
= size_int (map_idx
++);
11468 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11469 if (TREE_CODE (s
) != INTEGER_CST
)
11470 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11472 unsigned HOST_WIDE_INT tkind
;
11473 switch (OMP_CLAUSE_CODE (c
))
11475 case OMP_CLAUSE_MAP
:
11476 tkind
= OMP_CLAUSE_MAP_KIND (c
);
11478 case OMP_CLAUSE_TO
:
11479 tkind
= GOMP_MAP_TO
;
11481 case OMP_CLAUSE_FROM
:
11482 tkind
= GOMP_MAP_FROM
;
11485 gcc_unreachable ();
11487 gcc_checking_assert (tkind
11488 < (HOST_WIDE_INT_C (1U) << talign_shift
));
11489 talign
= ceil_log2 (talign
);
11490 tkind
|= talign
<< talign_shift
;
11491 gcc_checking_assert (tkind
11492 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11493 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
11494 build_int_cstu (tkind_type
, tkind
));
11499 gcc_assert (map_idx
== map_cnt
);
11501 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
11502 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
11503 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
11504 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
11505 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
11507 gimple_seq initlist
= NULL
;
11508 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
11509 TREE_VEC_ELT (t
, 1)),
11510 &initlist
, true, NULL_TREE
);
11511 gimple_seq_add_seq (&ilist
, initlist
);
11513 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
11515 TREE_THIS_VOLATILE (clobber
) = 1;
11516 gimple_seq_add_stmt (&olist
,
11517 gimple_build_assign (TREE_VEC_ELT (t
, 1),
11521 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
11522 TREE_THIS_VOLATILE (clobber
) = 1;
11523 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11527 /* Once all the expansions are done, sequence all the different
11528 fragments inside gimple_omp_body. */
11533 && ctx
->record_type
)
11535 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11536 /* fixup_child_record_type might have changed receiver_decl's type. */
11537 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11538 gimple_seq_add_stmt (&new_body
,
11539 gimple_build_assign (ctx
->receiver_decl
, t
));
11544 gimple_seq_add_seq (&new_body
, tgt_body
);
11545 new_body
= maybe_catch_exception (new_body
);
11547 else if (data_region
)
11548 new_body
= tgt_body
;
11549 if (offloaded
|| data_region
)
11551 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11552 gimple_omp_set_body (stmt
, new_body
);
11555 bind
= gimple_build_bind (NULL
, NULL
,
11556 tgt_bind
? gimple_bind_block (tgt_bind
)
11558 gsi_replace (gsi_p
, bind
, true);
11559 gimple_bind_add_seq (bind
, irlist
);
11560 gimple_bind_add_seq (bind
, ilist
);
11561 gimple_bind_add_stmt (bind
, stmt
);
11562 gimple_bind_add_seq (bind
, olist
);
11563 gimple_bind_add_seq (bind
, orlist
);
11565 pop_gimplify_context (NULL
);
11568 /* Expand code for an OpenMP teams directive. */
11571 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11573 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
11574 push_gimplify_context ();
11576 tree block
= make_node (BLOCK
);
11577 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
11578 gsi_replace (gsi_p
, bind
, true);
11579 gimple_seq bind_body
= NULL
;
11580 gimple_seq dlist
= NULL
;
11581 gimple_seq olist
= NULL
;
11583 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11584 OMP_CLAUSE_NUM_TEAMS
);
11585 if (num_teams
== NULL_TREE
)
11586 num_teams
= build_int_cst (unsigned_type_node
, 0);
11589 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
11590 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
11591 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
11593 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11594 OMP_CLAUSE_THREAD_LIMIT
);
11595 if (thread_limit
== NULL_TREE
)
11596 thread_limit
= build_int_cst (unsigned_type_node
, 0);
11599 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
11600 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
11601 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
11605 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
11606 &bind_body
, &dlist
, ctx
, NULL
);
11607 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
11608 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
11609 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
11611 location_t loc
= gimple_location (teams_stmt
);
11612 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
11613 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
11614 gimple_set_location (call
, loc
);
11615 gimple_seq_add_stmt (&bind_body
, call
);
11617 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
11618 gimple_omp_set_body (teams_stmt
, NULL
);
11619 gimple_seq_add_seq (&bind_body
, olist
);
11620 gimple_seq_add_seq (&bind_body
, dlist
);
11621 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
11622 gimple_bind_set_body (bind
, bind_body
);
11624 pop_gimplify_context (bind
);
11626 gimple_bind_append_vars (bind
, ctx
->block_vars
);
11627 BLOCK_VARS (block
) = ctx
->block_vars
;
11628 if (BLOCK_VARS (block
))
11629 TREE_USED (block
) = 1;
11633 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11634 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11635 of OMP context, but with task_shared_vars set. */
11638 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
11643 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11644 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
11647 if (task_shared_vars
11649 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
11652 /* If a global variable has been privatized, TREE_CONSTANT on
11653 ADDR_EXPR might be wrong. */
11654 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
11655 recompute_tree_invariant_for_addr_expr (t
);
11657 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
11662 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11664 gimple stmt
= gsi_stmt (*gsi_p
);
11665 struct walk_stmt_info wi
;
11668 if (gimple_has_location (stmt
))
11669 input_location
= gimple_location (stmt
);
11671 if (task_shared_vars
)
11672 memset (&wi
, '\0', sizeof (wi
));
11674 /* If we have issued syntax errors, avoid doing any heavy lifting.
11675 Just replace the OMP directives with a NOP to avoid
11676 confusing RTL expansion. */
11677 if (seen_error () && is_gimple_omp (stmt
))
11679 gsi_replace (gsi_p
, gimple_build_nop (), true);
11683 switch (gimple_code (stmt
))
11687 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
11688 if ((ctx
|| task_shared_vars
)
11689 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
11690 lower_omp_regimplify_p
,
11691 ctx
? NULL
: &wi
, NULL
)
11692 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
11693 lower_omp_regimplify_p
,
11694 ctx
? NULL
: &wi
, NULL
)))
11695 gimple_regimplify_operands (cond_stmt
, gsi_p
);
11699 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
11701 case GIMPLE_EH_FILTER
:
11702 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
11705 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
11706 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
11708 case GIMPLE_TRANSACTION
:
11709 lower_omp (gimple_transaction_body_ptr (
11710 as_a
<gtransaction
*> (stmt
)),
11714 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
11716 case GIMPLE_OMP_PARALLEL
:
11717 case GIMPLE_OMP_TASK
:
11718 ctx
= maybe_lookup_ctx (stmt
);
11720 if (ctx
->cancellable
)
11721 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11722 lower_omp_taskreg (gsi_p
, ctx
);
11724 case GIMPLE_OMP_FOR
:
11725 ctx
= maybe_lookup_ctx (stmt
);
11727 if (ctx
->cancellable
)
11728 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11729 lower_omp_for (gsi_p
, ctx
);
11731 case GIMPLE_OMP_SECTIONS
:
11732 ctx
= maybe_lookup_ctx (stmt
);
11734 if (ctx
->cancellable
)
11735 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11736 lower_omp_sections (gsi_p
, ctx
);
11738 case GIMPLE_OMP_SINGLE
:
11739 ctx
= maybe_lookup_ctx (stmt
);
11741 lower_omp_single (gsi_p
, ctx
);
11743 case GIMPLE_OMP_MASTER
:
11744 ctx
= maybe_lookup_ctx (stmt
);
11746 lower_omp_master (gsi_p
, ctx
);
11748 case GIMPLE_OMP_TASKGROUP
:
11749 ctx
= maybe_lookup_ctx (stmt
);
11751 lower_omp_taskgroup (gsi_p
, ctx
);
11753 case GIMPLE_OMP_ORDERED
:
11754 ctx
= maybe_lookup_ctx (stmt
);
11756 lower_omp_ordered (gsi_p
, ctx
);
11758 case GIMPLE_OMP_CRITICAL
:
11759 ctx
= maybe_lookup_ctx (stmt
);
11761 lower_omp_critical (gsi_p
, ctx
);
11763 case GIMPLE_OMP_ATOMIC_LOAD
:
11764 if ((ctx
|| task_shared_vars
)
11765 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
11766 as_a
<gomp_atomic_load
*> (stmt
)),
11767 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
11768 gimple_regimplify_operands (stmt
, gsi_p
);
11770 case GIMPLE_OMP_TARGET
:
11771 ctx
= maybe_lookup_ctx (stmt
);
11773 lower_omp_target (gsi_p
, ctx
);
11775 case GIMPLE_OMP_TEAMS
:
11776 ctx
= maybe_lookup_ctx (stmt
);
11778 lower_omp_teams (gsi_p
, ctx
);
11782 call_stmt
= as_a
<gcall
*> (stmt
);
11783 fndecl
= gimple_call_fndecl (call_stmt
);
11785 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
11786 switch (DECL_FUNCTION_CODE (fndecl
))
11788 case BUILT_IN_GOMP_BARRIER
:
11792 case BUILT_IN_GOMP_CANCEL
:
11793 case BUILT_IN_GOMP_CANCELLATION_POINT
:
11796 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
11797 cctx
= cctx
->outer
;
11798 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
11799 if (!cctx
->cancellable
)
11801 if (DECL_FUNCTION_CODE (fndecl
)
11802 == BUILT_IN_GOMP_CANCELLATION_POINT
)
11804 stmt
= gimple_build_nop ();
11805 gsi_replace (gsi_p
, stmt
, false);
11809 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
11811 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
11812 gimple_call_set_fndecl (call_stmt
, fndecl
);
11813 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
11816 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
11817 gimple_call_set_lhs (call_stmt
, lhs
);
11818 tree fallthru_label
;
11819 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
11821 g
= gimple_build_label (fallthru_label
);
11822 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11823 g
= gimple_build_cond (NE_EXPR
, lhs
,
11824 fold_convert (TREE_TYPE (lhs
),
11825 boolean_false_node
),
11826 cctx
->cancel_label
, fallthru_label
);
11827 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11834 if ((ctx
|| task_shared_vars
)
11835 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
11838 /* Just remove clobbers, this should happen only if we have
11839 "privatized" local addressable variables in SIMD regions,
11840 the clobber isn't needed in that case and gimplifying address
11841 of the ARRAY_REF into a pointer and creating MEM_REF based
11842 clobber would create worse code than we get with the clobber
11844 if (gimple_clobber_p (stmt
))
11846 gsi_replace (gsi_p
, gimple_build_nop (), true);
11849 gimple_regimplify_operands (stmt
, gsi_p
);
11856 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
11858 location_t saved_location
= input_location
;
11859 gimple_stmt_iterator gsi
;
11860 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11861 lower_omp_1 (&gsi
, ctx
);
11862 /* During gimplification, we haven't folded statments inside offloading
11863 regions (gimplify.c:maybe_fold_stmt); do that now. */
11864 if (target_nesting_level
)
11865 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11867 input_location
= saved_location
;
11870 /* Main entry point. */
11872 static unsigned int
11873 execute_lower_omp (void)
11879 /* This pass always runs, to provide PROP_gimple_lomp.
11880 But often, there is nothing to do. */
11881 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
11882 && flag_openmp_simd
== 0)
11885 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
11886 delete_omp_context
);
11888 body
= gimple_body (current_function_decl
);
11889 scan_omp (&body
, NULL
);
11890 gcc_assert (taskreg_nesting_level
== 0);
11891 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
11892 finish_taskreg_scan (ctx
);
11893 taskreg_contexts
.release ();
11895 if (all_contexts
->root
)
11897 if (task_shared_vars
)
11898 push_gimplify_context ();
11899 lower_omp (&body
, NULL
);
11900 if (task_shared_vars
)
11901 pop_gimplify_context (NULL
);
11906 splay_tree_delete (all_contexts
);
11907 all_contexts
= NULL
;
11909 BITMAP_FREE (task_shared_vars
);
11915 const pass_data pass_data_lower_omp
=
11917 GIMPLE_PASS
, /* type */
11918 "omplower", /* name */
11919 OPTGROUP_NONE
, /* optinfo_flags */
11920 TV_NONE
, /* tv_id */
11921 PROP_gimple_any
, /* properties_required */
11922 PROP_gimple_lomp
, /* properties_provided */
11923 0, /* properties_destroyed */
11924 0, /* todo_flags_start */
11925 0, /* todo_flags_finish */
11928 class pass_lower_omp
: public gimple_opt_pass
11931 pass_lower_omp (gcc::context
*ctxt
)
11932 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
11935 /* opt_pass methods: */
11936 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
11938 }; // class pass_lower_omp
11940 } // anon namespace
11943 make_pass_lower_omp (gcc::context
*ctxt
)
11945 return new pass_lower_omp (ctxt
);
11948 /* The following is a utility to diagnose structured block violations.
11949 It is not part of the "omplower" pass, as that's invoked too late. It
11950 should be invoked by the respective front ends after gimplification. */
11952 static splay_tree all_labels
;
11954 /* Check for mismatched contexts and generate an error if needed. Return
11955 true if an error is detected. */
11958 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
11959 gimple branch_ctx
, gimple label_ctx
)
11961 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
11962 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
11964 if (label_ctx
== branch_ctx
)
11967 const char* kind
= NULL
;
11972 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
11973 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
11975 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
11976 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
11977 kind
= "Cilk Plus";
11981 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
11982 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
11984 gcc_checking_assert (kind
== NULL
);
11990 gcc_checking_assert (flag_openmp
);
11995 Previously we kept track of the label's entire context in diagnose_sb_[12]
11996 so we could traverse it and issue a correct "exit" or "enter" error
11997 message upon a structured block violation.
11999 We built the context by building a list with tree_cons'ing, but there is
12000 no easy counterpart in gimple tuples. It seems like far too much work
12001 for issuing exit/enter error messages. If someone really misses the
12002 distinct error message... patches welcome.
12006 /* Try to avoid confusing the user by producing and error message
12007 with correct "exit" or "enter" verbiage. We prefer "exit"
12008 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12009 if (branch_ctx
== NULL
)
12015 if (TREE_VALUE (label_ctx
) == branch_ctx
)
12020 label_ctx
= TREE_CHAIN (label_ctx
);
12025 error ("invalid exit from %s structured block", kind
);
12027 error ("invalid entry to %s structured block", kind
);
12030 /* If it's obvious we have an invalid entry, be specific about the error. */
12031 if (branch_ctx
== NULL
)
12032 error ("invalid entry to %s structured block", kind
);
12035 /* Otherwise, be vague and lazy, but efficient. */
12036 error ("invalid branch to/from %s structured block", kind
);
12039 gsi_replace (gsi_p
, gimple_build_nop (), false);
12043 /* Pass 1: Create a minimal tree of structured blocks, and record
12044 where each label is found. */
12047 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12048 struct walk_stmt_info
*wi
)
12050 gimple context
= (gimple
) wi
->info
;
12051 gimple inner_context
;
12052 gimple stmt
= gsi_stmt (*gsi_p
);
12054 *handled_ops_p
= true;
12056 switch (gimple_code (stmt
))
12060 case GIMPLE_OMP_PARALLEL
:
12061 case GIMPLE_OMP_TASK
:
12062 case GIMPLE_OMP_SECTIONS
:
12063 case GIMPLE_OMP_SINGLE
:
12064 case GIMPLE_OMP_SECTION
:
12065 case GIMPLE_OMP_MASTER
:
12066 case GIMPLE_OMP_ORDERED
:
12067 case GIMPLE_OMP_CRITICAL
:
12068 case GIMPLE_OMP_TARGET
:
12069 case GIMPLE_OMP_TEAMS
:
12070 case GIMPLE_OMP_TASKGROUP
:
12071 /* The minimal context here is just the current OMP construct. */
12072 inner_context
= stmt
;
12073 wi
->info
= inner_context
;
12074 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12075 wi
->info
= context
;
12078 case GIMPLE_OMP_FOR
:
12079 inner_context
= stmt
;
12080 wi
->info
= inner_context
;
12081 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12083 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
12084 diagnose_sb_1
, NULL
, wi
);
12085 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12086 wi
->info
= context
;
12090 splay_tree_insert (all_labels
,
12091 (splay_tree_key
) gimple_label_label (
12092 as_a
<glabel
*> (stmt
)),
12093 (splay_tree_value
) context
);
12103 /* Pass 2: Check each branch and see if its context differs from that of
12104 the destination label's context. */
12107 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12108 struct walk_stmt_info
*wi
)
12110 gimple context
= (gimple
) wi
->info
;
12112 gimple stmt
= gsi_stmt (*gsi_p
);
12114 *handled_ops_p
= true;
12116 switch (gimple_code (stmt
))
12120 case GIMPLE_OMP_PARALLEL
:
12121 case GIMPLE_OMP_TASK
:
12122 case GIMPLE_OMP_SECTIONS
:
12123 case GIMPLE_OMP_SINGLE
:
12124 case GIMPLE_OMP_SECTION
:
12125 case GIMPLE_OMP_MASTER
:
12126 case GIMPLE_OMP_ORDERED
:
12127 case GIMPLE_OMP_CRITICAL
:
12128 case GIMPLE_OMP_TARGET
:
12129 case GIMPLE_OMP_TEAMS
:
12130 case GIMPLE_OMP_TASKGROUP
:
12132 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12133 wi
->info
= context
;
12136 case GIMPLE_OMP_FOR
:
12138 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12140 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
12141 diagnose_sb_2
, NULL
, wi
);
12142 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12143 wi
->info
= context
;
12148 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
12149 tree lab
= gimple_cond_true_label (cond_stmt
);
12152 n
= splay_tree_lookup (all_labels
,
12153 (splay_tree_key
) lab
);
12154 diagnose_sb_0 (gsi_p
, context
,
12155 n
? (gimple
) n
->value
: NULL
);
12157 lab
= gimple_cond_false_label (cond_stmt
);
12160 n
= splay_tree_lookup (all_labels
,
12161 (splay_tree_key
) lab
);
12162 diagnose_sb_0 (gsi_p
, context
,
12163 n
? (gimple
) n
->value
: NULL
);
12170 tree lab
= gimple_goto_dest (stmt
);
12171 if (TREE_CODE (lab
) != LABEL_DECL
)
12174 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12175 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
12179 case GIMPLE_SWITCH
:
12181 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
12183 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
12185 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
12186 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12187 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
12193 case GIMPLE_RETURN
:
12194 diagnose_sb_0 (gsi_p
, context
, NULL
);
12204 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12207 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
12210 gimple last
= last_stmt (bb
);
12211 enum gimple_code code
= gimple_code (last
);
12212 struct omp_region
*cur_region
= *region
;
12213 bool fallthru
= false;
12217 case GIMPLE_OMP_PARALLEL
:
12218 case GIMPLE_OMP_TASK
:
12219 case GIMPLE_OMP_FOR
:
12220 case GIMPLE_OMP_SINGLE
:
12221 case GIMPLE_OMP_TEAMS
:
12222 case GIMPLE_OMP_MASTER
:
12223 case GIMPLE_OMP_TASKGROUP
:
12224 case GIMPLE_OMP_ORDERED
:
12225 case GIMPLE_OMP_CRITICAL
:
12226 case GIMPLE_OMP_SECTION
:
12227 cur_region
= new_omp_region (bb
, code
, cur_region
);
12231 case GIMPLE_OMP_TARGET
:
12232 cur_region
= new_omp_region (bb
, code
, cur_region
);
12234 switch (gimple_omp_target_kind (last
))
12236 case GF_OMP_TARGET_KIND_REGION
:
12237 case GF_OMP_TARGET_KIND_DATA
:
12238 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12239 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12240 case GF_OMP_TARGET_KIND_OACC_DATA
:
12242 case GF_OMP_TARGET_KIND_UPDATE
:
12243 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12244 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12245 cur_region
= cur_region
->outer
;
12248 gcc_unreachable ();
12252 case GIMPLE_OMP_SECTIONS
:
12253 cur_region
= new_omp_region (bb
, code
, cur_region
);
12257 case GIMPLE_OMP_SECTIONS_SWITCH
:
12261 case GIMPLE_OMP_ATOMIC_LOAD
:
12262 case GIMPLE_OMP_ATOMIC_STORE
:
12266 case GIMPLE_OMP_RETURN
:
12267 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12268 somewhere other than the next block. This will be
12270 cur_region
->exit
= bb
;
12271 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
12272 cur_region
= cur_region
->outer
;
12275 case GIMPLE_OMP_CONTINUE
:
12276 cur_region
->cont
= bb
;
12277 switch (cur_region
->type
)
12279 case GIMPLE_OMP_FOR
:
12280 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12281 succs edges as abnormal to prevent splitting
12283 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
12284 /* Make the loopback edge. */
12285 make_edge (bb
, single_succ (cur_region
->entry
),
12288 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12289 corresponds to the case that the body of the loop
12290 is not executed at all. */
12291 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
12292 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
12296 case GIMPLE_OMP_SECTIONS
:
12297 /* Wire up the edges into and out of the nested sections. */
12299 basic_block switch_bb
= single_succ (cur_region
->entry
);
12301 struct omp_region
*i
;
12302 for (i
= cur_region
->inner
; i
; i
= i
->next
)
12304 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
12305 make_edge (switch_bb
, i
->entry
, 0);
12306 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
12309 /* Make the loopback edge to the block with
12310 GIMPLE_OMP_SECTIONS_SWITCH. */
12311 make_edge (bb
, switch_bb
, 0);
12313 /* Make the edge from the switch to exit. */
12314 make_edge (switch_bb
, bb
->next_bb
, 0);
12320 gcc_unreachable ();
12325 gcc_unreachable ();
12328 if (*region
!= cur_region
)
12330 *region
= cur_region
;
12332 *region_idx
= cur_region
->entry
->index
;
12340 static unsigned int
12341 diagnose_omp_structured_block_errors (void)
12343 struct walk_stmt_info wi
;
12344 gimple_seq body
= gimple_body (current_function_decl
);
12346 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
12348 memset (&wi
, 0, sizeof (wi
));
12349 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
12351 memset (&wi
, 0, sizeof (wi
));
12352 wi
.want_locations
= true;
12353 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
12355 gimple_set_body (current_function_decl
, body
);
12357 splay_tree_delete (all_labels
);
12365 const pass_data pass_data_diagnose_omp_blocks
=
12367 GIMPLE_PASS
, /* type */
12368 "*diagnose_omp_blocks", /* name */
12369 OPTGROUP_NONE
, /* optinfo_flags */
12370 TV_NONE
, /* tv_id */
12371 PROP_gimple_any
, /* properties_required */
12372 0, /* properties_provided */
12373 0, /* properties_destroyed */
12374 0, /* todo_flags_start */
12375 0, /* todo_flags_finish */
12378 class pass_diagnose_omp_blocks
: public gimple_opt_pass
12381 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12382 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
12385 /* opt_pass methods: */
12386 virtual bool gate (function
*)
12388 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
12390 virtual unsigned int execute (function
*)
12392 return diagnose_omp_structured_block_errors ();
12395 }; // class pass_diagnose_omp_blocks
12397 } // anon namespace
12400 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12402 return new pass_diagnose_omp_blocks (ctxt
);
12405 /* SIMD clone supporting code. */
12407 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12408 of arguments to reserve space for. */
12410 static struct cgraph_simd_clone
*
12411 simd_clone_struct_alloc (int nargs
)
12413 struct cgraph_simd_clone
*clone_info
;
12414 size_t len
= (sizeof (struct cgraph_simd_clone
)
12415 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
12416 clone_info
= (struct cgraph_simd_clone
*)
12417 ggc_internal_cleared_alloc (len
);
12421 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12424 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
12425 struct cgraph_simd_clone
*from
)
12427 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
12428 + ((from
->nargs
- from
->inbranch
)
12429 * sizeof (struct cgraph_simd_clone_arg
))));
12432 /* Return vector of parameter types of function FNDECL. This uses
12433 TYPE_ARG_TYPES if available, otherwise falls back to types of
12434 DECL_ARGUMENTS types. */
12437 simd_clone_vector_of_formal_parm_types (tree fndecl
)
12439 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
12440 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
12441 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
12444 FOR_EACH_VEC_ELT (args
, i
, arg
)
12445 args
[i
] = TREE_TYPE (args
[i
]);
12449 /* Given a simd function in NODE, extract the simd specific
12450 information from the OMP clauses passed in CLAUSES, and return
12451 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12452 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12453 otherwise set to FALSE. */
12455 static struct cgraph_simd_clone
*
12456 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
12457 bool *inbranch_specified
)
12459 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12462 *inbranch_specified
= false;
12464 n
= args
.length ();
12465 if (n
> 0 && args
.last () == void_type_node
)
12468 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12469 be cloned have a distinctive artificial label in addition to "omp
12473 && lookup_attribute ("cilk simd function",
12474 DECL_ATTRIBUTES (node
->decl
)));
12476 /* Allocate one more than needed just in case this is an in-branch
12477 clone which will require a mask argument. */
12478 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
12479 clone_info
->nargs
= n
;
12480 clone_info
->cilk_elemental
= cilk_clone
;
12487 clauses
= TREE_VALUE (clauses
);
12488 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
12491 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
12493 switch (OMP_CLAUSE_CODE (t
))
12495 case OMP_CLAUSE_INBRANCH
:
12496 clone_info
->inbranch
= 1;
12497 *inbranch_specified
= true;
12499 case OMP_CLAUSE_NOTINBRANCH
:
12500 clone_info
->inbranch
= 0;
12501 *inbranch_specified
= true;
12503 case OMP_CLAUSE_SIMDLEN
:
12504 clone_info
->simdlen
12505 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
12507 case OMP_CLAUSE_LINEAR
:
12509 tree decl
= OMP_CLAUSE_DECL (t
);
12510 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
12511 int argno
= TREE_INT_CST_LOW (decl
);
12512 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
12514 clone_info
->args
[argno
].arg_type
12515 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
12516 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12517 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
12518 && clone_info
->args
[argno
].linear_step
< n
);
12522 if (POINTER_TYPE_P (args
[argno
]))
12523 step
= fold_convert (ssizetype
, step
);
12524 if (!tree_fits_shwi_p (step
))
12526 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12527 "ignoring large linear step");
12531 else if (integer_zerop (step
))
12533 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12534 "ignoring zero linear step");
12540 clone_info
->args
[argno
].arg_type
12541 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
12542 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12547 case OMP_CLAUSE_UNIFORM
:
12549 tree decl
= OMP_CLAUSE_DECL (t
);
12550 int argno
= tree_to_uhwi (decl
);
12551 clone_info
->args
[argno
].arg_type
12552 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
12555 case OMP_CLAUSE_ALIGNED
:
12557 tree decl
= OMP_CLAUSE_DECL (t
);
12558 int argno
= tree_to_uhwi (decl
);
12559 clone_info
->args
[argno
].alignment
12560 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
12571 /* Given a SIMD clone in NODE, calculate the characteristic data
12572 type and return the coresponding type. The characteristic data
12573 type is computed as described in the Intel Vector ABI. */
12576 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
12577 struct cgraph_simd_clone
*clone_info
)
12579 tree type
= integer_type_node
;
12580 tree fndecl
= node
->decl
;
12582 /* a) For non-void function, the characteristic data type is the
12584 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
12585 type
= TREE_TYPE (TREE_TYPE (fndecl
));
12587 /* b) If the function has any non-uniform, non-linear parameters,
12588 then the characteristic data type is the type of the first
12592 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
12593 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
12594 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
12602 /* c) If the characteristic data type determined by a) or b) above
12603 is struct, union, or class type which is pass-by-value (except
12604 for the type that maps to the built-in complex data type), the
12605 characteristic data type is int. */
12606 if (RECORD_OR_UNION_TYPE_P (type
)
12607 && !aggregate_value_p (type
, NULL
)
12608 && TREE_CODE (type
) != COMPLEX_TYPE
)
12609 return integer_type_node
;
12611 /* d) If none of the above three classes is applicable, the
12612 characteristic data type is int. */
12616 /* e) For Intel Xeon Phi native and offload compilation, if the
12617 resulting characteristic data type is 8-bit or 16-bit integer
12618 data type, the characteristic data type is int. */
12619 /* Well, we don't handle Xeon Phi yet. */
12623 simd_clone_mangle (struct cgraph_node
*node
,
12624 struct cgraph_simd_clone
*clone_info
)
12626 char vecsize_mangle
= clone_info
->vecsize_mangle
;
12627 char mask
= clone_info
->inbranch
? 'M' : 'N';
12628 unsigned int simdlen
= clone_info
->simdlen
;
12632 gcc_assert (vecsize_mangle
&& simdlen
);
12634 pp_string (&pp
, "_ZGV");
12635 pp_character (&pp
, vecsize_mangle
);
12636 pp_character (&pp
, mask
);
12637 pp_decimal_int (&pp
, simdlen
);
12639 for (n
= 0; n
< clone_info
->nargs
; ++n
)
12641 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
12643 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
12644 pp_character (&pp
, 'u');
12645 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12647 gcc_assert (arg
.linear_step
!= 0);
12648 pp_character (&pp
, 'l');
12649 if (arg
.linear_step
> 1)
12650 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12651 else if (arg
.linear_step
< 0)
12653 pp_character (&pp
, 'n');
12654 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
12658 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
12660 pp_character (&pp
, 's');
12661 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12664 pp_character (&pp
, 'v');
12667 pp_character (&pp
, 'a');
12668 pp_decimal_int (&pp
, arg
.alignment
);
12672 pp_underscore (&pp
);
12673 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
12676 pp_string (&pp
, str
);
12677 str
= pp_formatted_text (&pp
);
12679 /* If there already is a SIMD clone with the same mangled name, don't
12680 add another one. This can happen e.g. for
12681 #pragma omp declare simd
12682 #pragma omp declare simd simdlen(8)
12683 int foo (int, int);
12684 if the simdlen is assumed to be 8 for the first one, etc. */
12685 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
12686 clone
= clone
->simdclone
->next_clone
)
12687 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
12691 return get_identifier (str
);
12694 /* Create a simd clone of OLD_NODE and return it. */
12696 static struct cgraph_node
*
12697 simd_clone_create (struct cgraph_node
*old_node
)
12699 struct cgraph_node
*new_node
;
12700 if (old_node
->definition
)
12702 if (!old_node
->has_gimple_body_p ())
12704 old_node
->get_body ();
12705 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
12711 tree old_decl
= old_node
->decl
;
12712 tree new_decl
= copy_node (old_node
->decl
);
12713 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
12714 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
12715 SET_DECL_RTL (new_decl
, NULL
);
12716 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
12717 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
12718 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
12719 symtab
->call_cgraph_insertion_hooks (new_node
);
12721 if (new_node
== NULL
)
12724 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
12726 /* The function cgraph_function_versioning () will force the new
12727 symbol local. Undo this, and inherit external visability from
12729 new_node
->local
.local
= old_node
->local
.local
;
12730 new_node
->externally_visible
= old_node
->externally_visible
;
12735 /* Adjust the return type of the given function to its appropriate
12736 vector counterpart. Returns a simd array to be used throughout the
12737 function as a return value. */
12740 simd_clone_adjust_return_type (struct cgraph_node
*node
)
12742 tree fndecl
= node
->decl
;
12743 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
12744 unsigned int veclen
;
12747 /* Adjust the function return type. */
12748 if (orig_rettype
== void_type_node
)
12750 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
12751 t
= TREE_TYPE (TREE_TYPE (fndecl
));
12752 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
12753 veclen
= node
->simdclone
->vecsize_int
;
12755 veclen
= node
->simdclone
->vecsize_float
;
12756 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
12757 if (veclen
> node
->simdclone
->simdlen
)
12758 veclen
= node
->simdclone
->simdlen
;
12759 if (POINTER_TYPE_P (t
))
12760 t
= pointer_sized_int_node
;
12761 if (veclen
== node
->simdclone
->simdlen
)
12762 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
12765 t
= build_vector_type (t
, veclen
);
12766 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
12768 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
12769 if (!node
->definition
)
12772 t
= DECL_RESULT (fndecl
);
12773 /* Adjust the DECL_RESULT. */
12774 gcc_assert (TREE_TYPE (t
) != void_type_node
);
12775 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
12778 tree atype
= build_array_type_nelts (orig_rettype
,
12779 node
->simdclone
->simdlen
);
12780 if (veclen
!= node
->simdclone
->simdlen
)
12781 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
12783 /* Set up a SIMD array to use as the return value. */
12784 tree retval
= create_tmp_var_raw (atype
, "retval");
12785 gimple_add_tmp_var (retval
);
12789 /* Each vector argument has a corresponding array to be used locally
12790 as part of the eventual loop. Create such temporary array and
12793 PREFIX is the prefix to be used for the temporary.
12795 TYPE is the inner element type.
12797 SIMDLEN is the number of elements. */
12800 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
12802 tree atype
= build_array_type_nelts (type
, simdlen
);
12803 tree avar
= create_tmp_var_raw (atype
, prefix
);
12804 gimple_add_tmp_var (avar
);
12808 /* Modify the function argument types to their corresponding vector
12809 counterparts if appropriate. Also, create one array for each simd
12810 argument to be used locally when using the function arguments as
12813 NODE is the function whose arguments are to be adjusted.
12815 Returns an adjustment vector that will be filled describing how the
12816 argument types will be adjusted. */
12818 static ipa_parm_adjustment_vec
12819 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
12822 ipa_parm_adjustment_vec adjustments
;
12824 if (node
->definition
)
12825 args
= ipa_get_vector_of_formal_parms (node
->decl
);
12827 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12828 adjustments
.create (args
.length ());
12829 unsigned i
, j
, veclen
;
12830 struct ipa_parm_adjustment adj
;
12831 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
12833 memset (&adj
, 0, sizeof (adj
));
12834 tree parm
= args
[i
];
12835 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
12836 adj
.base_index
= i
;
12839 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
12840 node
->simdclone
->args
[i
].orig_type
= parm_type
;
12842 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
12844 /* No adjustment necessary for scalar arguments. */
12845 adj
.op
= IPA_PARM_OP_COPY
;
12849 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
12850 veclen
= node
->simdclone
->vecsize_int
;
12852 veclen
= node
->simdclone
->vecsize_float
;
12853 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
12854 if (veclen
> node
->simdclone
->simdlen
)
12855 veclen
= node
->simdclone
->simdlen
;
12856 adj
.arg_prefix
= "simd";
12857 if (POINTER_TYPE_P (parm_type
))
12858 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12860 adj
.type
= build_vector_type (parm_type
, veclen
);
12861 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
12862 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12864 adjustments
.safe_push (adj
);
12867 memset (&adj
, 0, sizeof (adj
));
12868 adj
.op
= IPA_PARM_OP_NEW
;
12869 adj
.arg_prefix
= "simd";
12870 adj
.base_index
= i
;
12871 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
12875 if (node
->definition
)
12876 node
->simdclone
->args
[i
].simd_array
12877 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
12878 parm_type
, node
->simdclone
->simdlen
);
12880 adjustments
.safe_push (adj
);
12883 if (node
->simdclone
->inbranch
)
12886 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
12889 memset (&adj
, 0, sizeof (adj
));
12890 adj
.op
= IPA_PARM_OP_NEW
;
12891 adj
.arg_prefix
= "mask";
12893 adj
.base_index
= i
;
12894 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
12895 veclen
= node
->simdclone
->vecsize_int
;
12897 veclen
= node
->simdclone
->vecsize_float
;
12898 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
12899 if (veclen
> node
->simdclone
->simdlen
)
12900 veclen
= node
->simdclone
->simdlen
;
12901 if (POINTER_TYPE_P (base_type
))
12902 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12904 adj
.type
= build_vector_type (base_type
, veclen
);
12905 adjustments
.safe_push (adj
);
12907 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12908 adjustments
.safe_push (adj
);
12910 /* We have previously allocated one extra entry for the mask. Use
12912 struct cgraph_simd_clone
*sc
= node
->simdclone
;
12914 if (node
->definition
)
12916 sc
->args
[i
].orig_arg
12917 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
12918 sc
->args
[i
].simd_array
12919 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
12921 sc
->args
[i
].orig_type
= base_type
;
12922 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
12925 if (node
->definition
)
12926 ipa_modify_formal_parameters (node
->decl
, adjustments
);
12929 tree new_arg_types
= NULL_TREE
, new_reversed
;
12930 bool last_parm_void
= false;
12931 if (args
.length () > 0 && args
.last () == void_type_node
)
12932 last_parm_void
= true;
12934 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
12935 j
= adjustments
.length ();
12936 for (i
= 0; i
< j
; i
++)
12938 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
12940 if (adj
->op
== IPA_PARM_OP_COPY
)
12941 ptype
= args
[adj
->base_index
];
12944 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
12946 new_reversed
= nreverse (new_arg_types
);
12947 if (last_parm_void
)
12950 TREE_CHAIN (new_arg_types
) = void_list_node
;
12952 new_reversed
= void_list_node
;
12955 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
12956 TYPE_ARG_TYPES (new_type
) = new_reversed
;
12957 TREE_TYPE (node
->decl
) = new_type
;
12959 adjustments
.release ();
12962 return adjustments
;
12965 /* Initialize and copy the function arguments in NODE to their
12966 corresponding local simd arrays. Returns a fresh gimple_seq with
12967 the instruction sequence generated. */
12970 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
12971 ipa_parm_adjustment_vec adjustments
)
12973 gimple_seq seq
= NULL
;
12974 unsigned i
= 0, j
= 0, k
;
12976 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
12978 arg
= DECL_CHAIN (arg
), i
++, j
++)
12980 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
12983 node
->simdclone
->args
[i
].vector_arg
= arg
;
12985 tree array
= node
->simdclone
->args
[i
].simd_array
;
12986 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
12988 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12989 tree ptr
= build_fold_addr_expr (array
);
12990 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
12991 build_int_cst (ptype
, 0));
12992 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
12993 gimplify_and_add (t
, &seq
);
12997 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
12998 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12999 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
13001 tree ptr
= build_fold_addr_expr (array
);
13005 arg
= DECL_CHAIN (arg
);
13009 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
13010 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
13011 build_int_cst (ptype
, k
* elemsize
));
13012 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
13013 gimplify_and_add (t
, &seq
);
13020 /* Callback info for ipa_simd_modify_stmt_ops below. */
13022 struct modify_stmt_info
{
13023 ipa_parm_adjustment_vec adjustments
;
13025 /* True if the parent statement was modified by
13026 ipa_simd_modify_stmt_ops. */
13030 /* Callback for walk_gimple_op.
13032 Adjust operands from a given statement as specified in the
13033 adjustments vector in the callback data. */
13036 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
13038 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13039 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
13040 tree
*orig_tp
= tp
;
13041 if (TREE_CODE (*tp
) == ADDR_EXPR
)
13042 tp
= &TREE_OPERAND (*tp
, 0);
13043 struct ipa_parm_adjustment
*cand
= NULL
;
13044 if (TREE_CODE (*tp
) == PARM_DECL
)
13045 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
13049 *walk_subtrees
= 0;
13052 tree repl
= NULL_TREE
;
13054 repl
= unshare_expr (cand
->new_decl
);
13059 *walk_subtrees
= 0;
13060 bool modified
= info
->modified
;
13061 info
->modified
= false;
13062 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
13063 if (!info
->modified
)
13065 info
->modified
= modified
;
13068 info
->modified
= modified
;
13077 repl
= build_fold_addr_expr (repl
);
13079 if (is_gimple_debug (info
->stmt
))
13081 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
13082 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
13083 DECL_ARTIFICIAL (vexpr
) = 1;
13084 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
13085 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
13090 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
13091 repl
= gimple_assign_lhs (stmt
);
13093 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
13094 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13097 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
13099 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
13105 info
->modified
= true;
13109 /* Traverse the function body and perform all modifications as
13110 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13111 modified such that the replacement/reduction value will now be an
13112 offset into the corresponding simd_array.
13114 This function will replace all function argument uses with their
13115 corresponding simd array elements, and ajust the return values
13119 ipa_simd_modify_function_body (struct cgraph_node
*node
,
13120 ipa_parm_adjustment_vec adjustments
,
13121 tree retval_array
, tree iter
)
13124 unsigned int i
, j
, l
;
13126 /* Re-use the adjustments array, but this time use it to replace
13127 every function argument use to an offset into the corresponding
13129 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
13131 if (!node
->simdclone
->args
[i
].vector_arg
)
13134 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
13135 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
13136 adjustments
[j
].new_decl
13137 = build4 (ARRAY_REF
,
13139 node
->simdclone
->args
[i
].simd_array
,
13141 NULL_TREE
, NULL_TREE
);
13142 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
13143 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
13144 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
13147 l
= adjustments
.length ();
13148 for (i
= 1; i
< num_ssa_names
; i
++)
13150 tree name
= ssa_name (i
);
13152 && SSA_NAME_VAR (name
)
13153 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
13155 for (j
= 0; j
< l
; j
++)
13156 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
13157 && adjustments
[j
].new_decl
)
13160 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
13163 = copy_var_decl (adjustments
[j
].base
,
13164 DECL_NAME (adjustments
[j
].base
),
13165 TREE_TYPE (adjustments
[j
].base
));
13166 adjustments
[j
].new_ssa_base
= base_var
;
13169 base_var
= adjustments
[j
].new_ssa_base
;
13170 if (SSA_NAME_IS_DEFAULT_DEF (name
))
13172 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13173 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
13174 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
13175 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
13176 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13177 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
13178 gimple stmt
= gimple_build_assign (name
, new_decl
);
13179 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13182 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13187 struct modify_stmt_info info
;
13188 info
.adjustments
= adjustments
;
13190 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
13192 gimple_stmt_iterator gsi
;
13194 gsi
= gsi_start_bb (bb
);
13195 while (!gsi_end_p (gsi
))
13197 gimple stmt
= gsi_stmt (gsi
);
13199 struct walk_stmt_info wi
;
13201 memset (&wi
, 0, sizeof (wi
));
13202 info
.modified
= false;
13204 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
13206 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
13208 tree retval
= gimple_return_retval (return_stmt
);
13211 gsi_remove (&gsi
, true);
13215 /* Replace `return foo' with `retval_array[iter] = foo'. */
13216 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
13217 retval_array
, iter
, NULL
, NULL
);
13218 stmt
= gimple_build_assign (ref
, retval
);
13219 gsi_replace (&gsi
, stmt
, true);
13220 info
.modified
= true;
13225 update_stmt (stmt
);
13226 if (maybe_clean_eh_stmt (stmt
))
13227 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
13234 /* Adjust the argument types in NODE to their appropriate vector
13238 simd_clone_adjust (struct cgraph_node
*node
)
13240 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
13242 targetm
.simd_clone
.adjust (node
);
13244 tree retval
= simd_clone_adjust_return_type (node
);
13245 ipa_parm_adjustment_vec adjustments
13246 = simd_clone_adjust_argument_types (node
);
13248 push_gimplify_context ();
13250 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
13252 /* Adjust all uses of vector arguments accordingly. Adjust all
13253 return values accordingly. */
13254 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
13255 tree iter1
= make_ssa_name (iter
);
13256 tree iter2
= make_ssa_name (iter
);
13257 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
13259 /* Initialize the iteration variable. */
13260 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13261 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
13262 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
13263 /* Insert the SIMD array and iv initialization at function
13265 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
13267 pop_gimplify_context (NULL
);
13269 /* Create a new BB right before the original exit BB, to hold the
13270 iteration increment and the condition/branch. */
13271 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
13272 basic_block incr_bb
= create_empty_bb (orig_exit
);
13273 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
13274 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13275 flag. Set it now to be a FALLTHRU_EDGE. */
13276 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
13277 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
13278 for (unsigned i
= 0;
13279 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
13281 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
13282 redirect_edge_succ (e
, incr_bb
);
13284 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
13285 e
->probability
= REG_BR_PROB_BASE
;
13286 gsi
= gsi_last_bb (incr_bb
);
13287 gimple g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
13288 build_int_cst (unsigned_type_node
, 1));
13289 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13291 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13292 struct loop
*loop
= alloc_loop ();
13293 cfun
->has_force_vectorize_loops
= true;
13294 loop
->safelen
= node
->simdclone
->simdlen
;
13295 loop
->force_vectorize
= true;
13296 loop
->header
= body_bb
;
13298 /* Branch around the body if the mask applies. */
13299 if (node
->simdclone
->inbranch
)
13301 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
13303 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
13304 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
13305 tree aref
= build4 (ARRAY_REF
,
13306 TREE_TYPE (TREE_TYPE (mask_array
)),
13309 g
= gimple_build_assign (mask
, aref
);
13310 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13311 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
13312 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
13314 aref
= build1 (VIEW_CONVERT_EXPR
,
13315 build_nonstandard_integer_type (bitsize
, 0), mask
);
13316 mask
= make_ssa_name (TREE_TYPE (aref
));
13317 g
= gimple_build_assign (mask
, aref
);
13318 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13321 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
13323 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13324 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
13325 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
13328 /* Generate the condition. */
13329 g
= gimple_build_cond (LT_EXPR
,
13331 build_int_cst (unsigned_type_node
,
13332 node
->simdclone
->simdlen
),
13334 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13335 e
= split_block (incr_bb
, gsi_stmt (gsi
));
13336 basic_block latch_bb
= e
->dest
;
13337 basic_block new_exit_bb
;
13338 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
13339 loop
->latch
= latch_bb
;
13341 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
13343 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
13344 /* The successor of incr_bb is already pointing to latch_bb; just
13346 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13347 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
13349 gphi
*phi
= create_phi_node (iter1
, body_bb
);
13350 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
13351 edge latch_edge
= single_succ_edge (latch_bb
);
13352 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
13354 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13356 /* Generate the new return. */
13357 gsi
= gsi_last_bb (new_exit_bb
);
13359 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
13360 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
13361 retval
= TREE_OPERAND (retval
, 0);
13364 retval
= build1 (VIEW_CONVERT_EXPR
,
13365 TREE_TYPE (TREE_TYPE (node
->decl
)),
13367 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
13368 false, GSI_CONTINUE_LINKING
);
13370 g
= gimple_build_return (retval
);
13371 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13373 /* Handle aligned clauses by replacing default defs of the aligned
13374 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13375 lhs. Handle linear by adding PHIs. */
13376 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
13377 if (node
->simdclone
->args
[i
].alignment
13378 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13379 && (node
->simdclone
->args
[i
].alignment
13380 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
13381 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13384 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
13385 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13386 tree def
= ssa_default_def (cfun
, orig_arg
);
13387 if (def
&& !has_zero_uses (def
))
13389 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
13390 gimple_seq seq
= NULL
;
13391 bool need_cvt
= false;
13393 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
13395 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
13398 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
13399 gimple_call_set_lhs (g
, t
);
13400 gimple_seq_add_stmt_without_update (&seq
, g
);
13403 t
= make_ssa_name (orig_arg
);
13404 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
13405 gimple_seq_add_stmt_without_update (&seq
, g
);
13407 gsi_insert_seq_on_edge_immediate
13408 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
13410 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13411 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
13413 node
->create_edge (cgraph_node::get_create (fn
),
13414 call
, entry_bb
->count
, freq
);
13416 imm_use_iterator iter
;
13417 use_operand_p use_p
;
13419 tree repl
= gimple_get_lhs (g
);
13420 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13421 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
13424 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13425 SET_USE (use_p
, repl
);
13428 else if (node
->simdclone
->args
[i
].arg_type
13429 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
13431 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13432 tree def
= ssa_default_def (cfun
, orig_arg
);
13433 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13434 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
13435 if (def
&& !has_zero_uses (def
))
13437 iter1
= make_ssa_name (orig_arg
);
13438 iter2
= make_ssa_name (orig_arg
);
13439 phi
= create_phi_node (iter1
, body_bb
);
13440 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
13441 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13442 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13443 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
13444 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13445 ? TREE_TYPE (orig_arg
) : sizetype
;
13447 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
13448 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
13449 gsi
= gsi_last_bb (incr_bb
);
13450 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13452 imm_use_iterator iter
;
13453 use_operand_p use_p
;
13455 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13456 if (use_stmt
== phi
)
13459 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13460 SET_USE (use_p
, iter1
);
13464 calculate_dominance_info (CDI_DOMINATORS
);
13465 add_loop (loop
, loop
->header
->loop_father
);
13466 update_ssa (TODO_update_ssa
);
13471 /* If the function in NODE is tagged as an elemental SIMD function,
13472 create the appropriate SIMD clones. */
13475 expand_simd_clones (struct cgraph_node
*node
)
13477 tree attr
= lookup_attribute ("omp declare simd",
13478 DECL_ATTRIBUTES (node
->decl
));
13479 if (attr
== NULL_TREE
13480 || node
->global
.inlined_to
13481 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
13485 #pragma omp declare simd
13487 in C, there we don't know the argument types at all. */
13488 if (!node
->definition
13489 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
13494 /* Start with parsing the "omp declare simd" attribute(s). */
13495 bool inbranch_clause_specified
;
13496 struct cgraph_simd_clone
*clone_info
13497 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
13498 &inbranch_clause_specified
);
13499 if (clone_info
== NULL
)
13502 int orig_simdlen
= clone_info
->simdlen
;
13503 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
13504 /* The target can return 0 (no simd clones should be created),
13505 1 (just one ISA of simd clones should be created) or higher
13506 count of ISA variants. In that case, clone_info is initialized
13507 for the first ISA variant. */
13509 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
13514 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13515 also create one inbranch and one !inbranch clone of it. */
13516 for (int i
= 0; i
< count
* 2; i
++)
13518 struct cgraph_simd_clone
*clone
= clone_info
;
13519 if (inbranch_clause_specified
&& (i
& 1) != 0)
13524 clone
= simd_clone_struct_alloc (clone_info
->nargs
13526 simd_clone_struct_copy (clone
, clone_info
);
13527 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13528 and simd_clone_adjust_argument_types did to the first
13530 clone
->nargs
-= clone_info
->inbranch
;
13531 clone
->simdlen
= orig_simdlen
;
13532 /* And call the target hook again to get the right ISA. */
13533 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
13537 clone
->inbranch
= 1;
13540 /* simd_clone_mangle might fail if such a clone has been created
13542 tree id
= simd_clone_mangle (node
, clone
);
13543 if (id
== NULL_TREE
)
13546 /* Only when we are sure we want to create the clone actually
13547 clone the function (or definitions) or create another
13548 extern FUNCTION_DECL (for prototypes without definitions). */
13549 struct cgraph_node
*n
= simd_clone_create (node
);
13553 n
->simdclone
= clone
;
13554 clone
->origin
= node
;
13555 clone
->next_clone
= NULL
;
13556 if (node
->simd_clones
== NULL
)
13558 clone
->prev_clone
= n
;
13559 node
->simd_clones
= n
;
13563 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
13564 clone
->prev_clone
->simdclone
->next_clone
= n
;
13565 node
->simd_clones
->simdclone
->prev_clone
= n
;
13567 symtab
->change_decl_assembler_name (n
->decl
, id
);
13568 /* And finally adjust the return type, parameters and for
13569 definitions also function body. */
13570 if (node
->definition
)
13571 simd_clone_adjust (n
);
13574 simd_clone_adjust_return_type (n
);
13575 simd_clone_adjust_argument_types (n
);
13579 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
13582 /* Entry point for IPA simd clone creation pass. */
13584 static unsigned int
13585 ipa_omp_simd_clone (void)
13587 struct cgraph_node
*node
;
13588 FOR_EACH_FUNCTION (node
)
13589 expand_simd_clones (node
);
13595 const pass_data pass_data_omp_simd_clone
=
13597 SIMPLE_IPA_PASS
, /* type */
13598 "simdclone", /* name */
13599 OPTGROUP_NONE
, /* optinfo_flags */
13600 TV_NONE
, /* tv_id */
13601 ( PROP_ssa
| PROP_cfg
), /* properties_required */
13602 0, /* properties_provided */
13603 0, /* properties_destroyed */
13604 0, /* todo_flags_start */
13605 0, /* todo_flags_finish */
13608 class pass_omp_simd_clone
: public simple_ipa_opt_pass
13611 pass_omp_simd_clone(gcc::context
*ctxt
)
13612 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
13615 /* opt_pass methods: */
13616 virtual bool gate (function
*);
13617 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
13621 pass_omp_simd_clone::gate (function
*)
13623 return ((flag_openmp
|| flag_openmp_simd
13625 || (in_lto_p
&& !flag_wpa
))
13626 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
13629 } // anon namespace
13631 simple_ipa_opt_pass
*
13632 make_pass_omp_simd_clone (gcc::context
*ctxt
)
13634 return new pass_omp_simd_clone (ctxt
);
13637 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
13638 adds their addresses and sizes to constructor-vector V_CTOR. */
13640 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
13641 vec
<constructor_elt
, va_gc
> *v_ctor
)
13643 unsigned len
= vec_safe_length (v_decls
);
13644 for (unsigned i
= 0; i
< len
; i
++)
13646 tree it
= (*v_decls
)[i
];
13647 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
13649 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
13651 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
13652 fold_convert (const_ptr_type_node
,
13653 DECL_SIZE_UNIT (it
)));
13657 /* Create new symbols containing (address, size) pairs for global variables,
13658 marked with "omp declare target" attribute, as well as addresses for the
13659 functions, which are outlined offloading regions. */
13661 omp_finish_file (void)
13663 unsigned num_funcs
= vec_safe_length (offload_funcs
);
13664 unsigned num_vars
= vec_safe_length (offload_vars
);
13666 if (num_funcs
== 0 && num_vars
== 0)
13669 if (targetm_common
.have_named_sections
)
13671 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
13672 vec_alloc (v_f
, num_funcs
);
13673 vec_alloc (v_v
, num_vars
* 2);
13675 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
13676 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
13678 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13680 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13682 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13683 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13684 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
13685 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
13686 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
13687 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
13688 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13689 get_identifier (".offload_func_table"),
13691 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13692 get_identifier (".offload_var_table"),
13694 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
13695 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
13696 otherwise a joint table in a binary will contain padding between
13697 tables from multiple object files. */
13698 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
13699 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
13700 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
13701 DECL_INITIAL (funcs_decl
) = ctor_f
;
13702 DECL_INITIAL (vars_decl
) = ctor_v
;
13703 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
13704 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
13706 varpool_node::finalize_decl (vars_decl
);
13707 varpool_node::finalize_decl (funcs_decl
);
13711 for (unsigned i
= 0; i
< num_funcs
; i
++)
13713 tree it
= (*offload_funcs
)[i
];
13714 targetm
.record_offload_symbol (it
);
13716 for (unsigned i
= 0; i
< num_vars
; i
++)
13718 tree it
= (*offload_vars
)[i
];
13719 targetm
.record_offload_symbol (it
);
13724 #include "gt-omp-low.h"