1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
37 #include "hard-reg-set.h"
40 #include "dominance.h"
43 #include "basic-block.h"
44 #include "tree-ssa-alias.h"
45 #include "internal-fn.h"
46 #include "gimple-fold.h"
47 #include "gimple-expr.h"
51 #include "gimple-iterator.h"
52 #include "gimplify-me.h"
53 #include "gimple-walk.h"
54 #include "tree-iterator.h"
55 #include "tree-inline.h"
56 #include "langhooks.h"
57 #include "diagnostic-core.h"
58 #include "gimple-ssa.h"
60 #include "plugin-api.h"
64 #include "tree-phinodes.h"
65 #include "ssa-iterators.h"
66 #include "tree-ssanames.h"
67 #include "tree-into-ssa.h"
73 #include "tree-pass.h"
75 #include "splay-tree.h"
79 #include "common/common-target.h"
81 #include "gimple-low.h"
82 #include "tree-cfgcleanup.h"
83 #include "pretty-print.h"
84 #include "alloc-pool.h"
86 #include "tree-nested.h"
89 #include "lto-section-names.h"
92 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
93 phases. The first phase scans the function looking for OMP statements
94 and then for variables that must be replaced to satisfy data sharing
95 clauses. The second phase expands code for the constructs, as well as
96 re-gimplifying things when variables have been replaced with complex
99 Final code generation is done by pass_expand_omp. The flowgraph is
100 scanned for parallel regions which are then moved to a new
101 function, to be invoked by the thread library. */
103 /* Parallel region information. Every parallel and workshare
104 directive is enclosed between two markers, the OMP_* directive
105 and a corresponding OMP_RETURN statement. */
109 /* The enclosing region. */
110 struct omp_region
*outer
;
112 /* First child region. */
113 struct omp_region
*inner
;
115 /* Next peer region. */
116 struct omp_region
*next
;
118 /* Block containing the omp directive as its last stmt. */
121 /* Block containing the OMP_RETURN as its last stmt. */
124 /* Block containing the OMP_CONTINUE as its last stmt. */
127 /* If this is a combined parallel+workshare region, this is a list
128 of additional arguments needed by the combined parallel+workshare
130 vec
<tree
, va_gc
> *ws_args
;
132 /* The code for the omp directive of this region. */
133 enum gimple_code type
;
135 /* Schedule kind, only used for OMP_FOR type regions. */
136 enum omp_clause_schedule_kind sched_kind
;
138 /* True if this is a combined parallel+workshare region. */
139 bool is_combined_parallel
;
142 /* Context structure. Used to store information about each parallel
143 directive in the code. */
145 typedef struct omp_context
147 /* This field must be at the beginning, as we do "inheritance": Some
148 callback functions for tree-inline.c (e.g., omp_copy_decl)
149 receive a copy_body_data pointer that is up-casted to an
150 omp_context pointer. */
153 /* The tree of contexts corresponding to the encountered constructs. */
154 struct omp_context
*outer
;
157 /* Map variables to fields in a structure that allows communication
158 between sending and receiving threads. */
159 splay_tree field_map
;
164 /* These are used just by task contexts, if task firstprivate fn is
165 needed. srecord_type is used to communicate from the thread
166 that encountered the task construct to task firstprivate fn,
167 record_type is allocated by GOMP_task, initialized by task firstprivate
168 fn and passed to the task body fn. */
169 splay_tree sfield_map
;
172 /* A chain of variables to add to the top-level block surrounding the
173 construct. In the case of a parallel, this is in the child function. */
176 /* A map of reduction pointer variables. For accelerators, each
177 reduction variable is replaced with an array. Each thread, in turn,
178 is assigned to a slot on that array. */
179 splay_tree reduction_map
;
181 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
182 barriers should jump to during omplower pass. */
185 /* What to do with variables with implicitly determined sharing
187 enum omp_clause_default_kind default_kind
;
189 /* Nesting depth of this context. Used to beautify error messages re
190 invalid gotos. The outermost ctx is depth 1, with depth 0 being
191 reserved for the main body of the function. */
194 /* True if this parallel directive is nested within another. */
197 /* True if this construct can be cancelled. */
201 /* A structure holding the elements of:
202 for (V = N1; V cond N2; V += STEP) [...] */
204 struct omp_for_data_loop
206 tree v
, n1
, n2
, step
;
207 enum tree_code cond_code
;
210 /* A structure describing the main elements of a parallel loop. */
214 struct omp_for_data_loop loop
;
219 bool have_nowait
, have_ordered
;
220 enum omp_clause_schedule_kind sched_kind
;
221 struct omp_for_data_loop
*loops
;
225 static splay_tree all_contexts
;
226 static int taskreg_nesting_level
;
227 static int target_nesting_level
;
228 static struct omp_region
*root_omp_region
;
229 static bitmap task_shared_vars
;
230 static vec
<omp_context
*> taskreg_contexts
;
232 static void scan_omp (gimple_seq
*, omp_context
*);
233 static tree
scan_omp_1_op (tree
*, int *, void *);
235 #define WALK_SUBSTMTS \
239 case GIMPLE_EH_FILTER: \
240 case GIMPLE_TRANSACTION: \
241 /* The sub-statements for these should be walked. */ \
242 *handled_ops_p = false; \
245 /* Helper function to get the reduction array name */
247 omp_get_id (tree node
)
249 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
250 int len
= strlen ("omp$") + strlen (id
);
251 char *temp_name
= (char *)alloca (len
+1);
252 snprintf (temp_name
, len
+1, "gfc$%s", id
);
253 return IDENTIFIER_POINTER(get_identifier (temp_name
));
256 /* Determine the number of threads OpenACC threads used to determine the
257 size of the array of partial reductions. Currently, this is num_gangs
258 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
259 because it is independed of the device used. */
262 oacc_max_threads (omp_context
*ctx
)
264 tree nthreads
, vector_length
, gangs
, clauses
;
266 gangs
= fold_convert (sizetype
, integer_one_node
);
267 vector_length
= gangs
;
269 /* The reduction clause may be nested inside a loop directive.
270 Scan for the innermost vector_length clause. */
271 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
273 if (gimple_code (oc
->stmt
) != GIMPLE_OACC_PARALLEL
)
276 clauses
= gimple_oacc_parallel_clauses (oc
->stmt
);
278 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
280 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
282 OMP_CLAUSE_VECTOR_LENGTH_EXPR
285 vector_length
= fold_convert (sizetype
, integer_one_node
);
287 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
289 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
290 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
292 gangs
= fold_convert (sizetype
, integer_one_node
);
297 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
302 /* Holds a decl for __OPENMP_TARGET__. */
303 static GTY(()) tree offload_symbol_decl
;
305 /* Holds offload tables with decls. */
306 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
308 /* Get the __OPENMP_TARGET__ symbol. */
310 get_offload_symbol_decl (void)
312 if (!offload_symbol_decl
)
314 tree decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
315 get_identifier ("__OPENMP_TARGET__"),
317 TREE_ADDRESSABLE (decl
) = 1;
318 TREE_PUBLIC (decl
) = 1;
319 DECL_EXTERNAL (decl
) = 1;
320 DECL_WEAK (decl
) = 1;
321 DECL_ATTRIBUTES (decl
)
322 = tree_cons (get_identifier ("weak"),
323 NULL_TREE
, DECL_ATTRIBUTES (decl
));
324 offload_symbol_decl
= decl
;
326 return offload_symbol_decl
;
329 /* Convenience function for calling scan_omp_1_op on tree operands. */
332 scan_omp_op (tree
*tp
, omp_context
*ctx
)
334 struct walk_stmt_info wi
;
336 memset (&wi
, 0, sizeof (wi
));
338 wi
.want_locations
= true;
340 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
343 static void lower_omp (gimple_seq
*, omp_context
*);
344 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
345 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
347 /* Find an OpenMP clause of type KIND within CLAUSES. */
350 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
352 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
353 if (OMP_CLAUSE_CODE (clauses
) == kind
)
359 /* Return true if CTX is for an omp parallel. */
362 is_parallel_ctx (omp_context
*ctx
)
364 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
368 /* Return true if CTX is for an omp task. */
371 is_task_ctx (omp_context
*ctx
)
373 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
377 /* Return true if CTX is for an omp parallel or omp task. */
380 is_taskreg_ctx (omp_context
*ctx
)
382 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
383 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
387 /* Return true if REGION is a combined parallel+workshare region. */
390 is_combined_parallel (struct omp_region
*region
)
392 return region
->is_combined_parallel
;
396 /* Extract the header elements of parallel loop FOR_STMT and store
400 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
401 struct omp_for_data_loop
*loops
)
403 tree t
, var
, *collapse_iter
, *collapse_count
;
404 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
405 struct omp_for_data_loop
*loop
;
407 struct omp_for_data_loop dummy_loop
;
408 location_t loc
= gimple_location (for_stmt
);
409 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
410 bool distribute
= gimple_omp_for_kind (for_stmt
)
411 == GF_OMP_FOR_KIND_DISTRIBUTE
;
413 fd
->for_stmt
= for_stmt
;
415 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
416 if (fd
->collapse
> 1)
419 fd
->loops
= &fd
->loop
;
421 fd
->have_nowait
= (gimple_omp_for_kind (for_stmt
) != GF_OMP_FOR_KIND_FOR
);
422 fd
->have_ordered
= false;
423 fd
->sched_kind
= /* TODO: OACC_LOOP */ OMP_CLAUSE_SCHEDULE_STATIC
;
424 fd
->chunk_size
= NULL_TREE
;
425 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
426 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
427 collapse_iter
= NULL
;
428 collapse_count
= NULL
;
430 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
431 switch (OMP_CLAUSE_CODE (t
))
433 case OMP_CLAUSE_NOWAIT
:
434 fd
->have_nowait
= true;
436 case OMP_CLAUSE_ORDERED
:
437 fd
->have_ordered
= true;
439 case OMP_CLAUSE_SCHEDULE
:
440 gcc_assert (!distribute
);
441 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
442 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
444 case OMP_CLAUSE_DIST_SCHEDULE
:
445 gcc_assert (distribute
);
446 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
448 case OMP_CLAUSE_COLLAPSE
:
449 if (fd
->collapse
> 1)
451 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
452 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
459 /* FIXME: for now map schedule(auto) to schedule(static).
460 There should be analysis to determine whether all iterations
461 are approximately the same amount of work (then schedule(static)
462 is best) or if it varies (then schedule(dynamic,N) is better). */
463 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
465 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
466 gcc_assert (fd
->chunk_size
== NULL
);
468 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
469 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
470 gcc_assert (fd
->chunk_size
== NULL
);
471 else if (fd
->chunk_size
== NULL
)
473 /* We only need to compute a default chunk size for ordered
474 static loops and dynamic loops. */
475 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
477 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
478 ? integer_zero_node
: integer_one_node
;
481 for (i
= 0; i
< fd
->collapse
; i
++)
483 if (fd
->collapse
== 1)
485 else if (loops
!= NULL
)
490 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
491 gcc_assert (SSA_VAR_P (loop
->v
));
492 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
493 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
494 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
495 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
497 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
498 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
499 switch (loop
->cond_code
)
505 gcc_assert (gimple_omp_for_kind (for_stmt
)
506 == GF_OMP_FOR_KIND_CILKSIMD
507 || (gimple_omp_for_kind (for_stmt
)
508 == GF_OMP_FOR_KIND_CILKFOR
));
511 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
512 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
514 loop
->n2
= fold_build2_loc (loc
,
515 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
516 build_int_cst (TREE_TYPE (loop
->n2
), 1));
517 loop
->cond_code
= LT_EXPR
;
520 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
521 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
523 loop
->n2
= fold_build2_loc (loc
,
524 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
525 build_int_cst (TREE_TYPE (loop
->n2
), 1));
526 loop
->cond_code
= GT_EXPR
;
532 t
= gimple_omp_for_incr (for_stmt
, i
);
533 gcc_assert (TREE_OPERAND (t
, 0) == var
);
534 switch (TREE_CODE (t
))
537 loop
->step
= TREE_OPERAND (t
, 1);
539 case POINTER_PLUS_EXPR
:
540 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
543 loop
->step
= TREE_OPERAND (t
, 1);
544 loop
->step
= fold_build1_loc (loc
,
545 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
553 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
554 && !fd
->have_ordered
))
556 if (fd
->collapse
== 1)
557 iter_type
= TREE_TYPE (loop
->v
);
559 || TYPE_PRECISION (iter_type
)
560 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
562 = build_nonstandard_integer_type
563 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
565 else if (iter_type
!= long_long_unsigned_type_node
)
567 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
568 iter_type
= long_long_unsigned_type_node
;
569 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
570 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
571 >= TYPE_PRECISION (iter_type
))
575 if (loop
->cond_code
== LT_EXPR
)
576 n
= fold_build2_loc (loc
,
577 PLUS_EXPR
, TREE_TYPE (loop
->v
),
578 loop
->n2
, loop
->step
);
581 if (TREE_CODE (n
) != INTEGER_CST
582 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
583 iter_type
= long_long_unsigned_type_node
;
585 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
586 > TYPE_PRECISION (iter_type
))
590 if (loop
->cond_code
== LT_EXPR
)
593 n2
= fold_build2_loc (loc
,
594 PLUS_EXPR
, TREE_TYPE (loop
->v
),
595 loop
->n2
, loop
->step
);
599 n1
= fold_build2_loc (loc
,
600 MINUS_EXPR
, TREE_TYPE (loop
->v
),
601 loop
->n2
, loop
->step
);
604 if (TREE_CODE (n1
) != INTEGER_CST
605 || TREE_CODE (n2
) != INTEGER_CST
606 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
607 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
608 iter_type
= long_long_unsigned_type_node
;
612 if (collapse_count
&& *collapse_count
== NULL
)
614 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
615 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
616 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
617 if (t
&& integer_zerop (t
))
618 count
= build_zero_cst (long_long_unsigned_type_node
);
619 else if ((i
== 0 || count
!= NULL_TREE
)
620 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
621 && TREE_CONSTANT (loop
->n1
)
622 && TREE_CONSTANT (loop
->n2
)
623 && TREE_CODE (loop
->step
) == INTEGER_CST
)
625 tree itype
= TREE_TYPE (loop
->v
);
627 if (POINTER_TYPE_P (itype
))
628 itype
= signed_type_for (itype
);
629 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
630 t
= fold_build2_loc (loc
,
632 fold_convert_loc (loc
, itype
, loop
->step
), t
);
633 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
634 fold_convert_loc (loc
, itype
, loop
->n2
));
635 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
636 fold_convert_loc (loc
, itype
, loop
->n1
));
637 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
638 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
639 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
640 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
641 fold_convert_loc (loc
, itype
,
644 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
645 fold_convert_loc (loc
, itype
, loop
->step
));
646 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
647 if (count
!= NULL_TREE
)
648 count
= fold_build2_loc (loc
,
649 MULT_EXPR
, long_long_unsigned_type_node
,
653 if (TREE_CODE (count
) != INTEGER_CST
)
656 else if (count
&& !integer_zerop (count
))
663 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
664 || fd
->have_ordered
))
666 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
667 iter_type
= long_long_unsigned_type_node
;
669 iter_type
= long_integer_type_node
;
671 else if (collapse_iter
&& *collapse_iter
!= NULL
)
672 iter_type
= TREE_TYPE (*collapse_iter
);
673 fd
->iter_type
= iter_type
;
674 if (collapse_iter
&& *collapse_iter
== NULL
)
675 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
676 if (collapse_count
&& *collapse_count
== NULL
)
679 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
681 *collapse_count
= create_tmp_var (iter_type
, ".count");
684 if (fd
->collapse
> 1)
686 fd
->loop
.v
= *collapse_iter
;
687 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
688 fd
->loop
.n2
= *collapse_count
;
689 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
690 fd
->loop
.cond_code
= LT_EXPR
;
694 /* For OpenACC loops, force a chunk size of one, as this avoids the default
695 scheduling where several subsequent iterations are being executed by the
697 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
699 gcc_assert (fd
->chunk_size
== NULL_TREE
);
700 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
705 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
706 is the immediate dominator of PAR_ENTRY_BB, return true if there
707 are no data dependencies that would prevent expanding the parallel
708 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
710 When expanding a combined parallel+workshare region, the call to
711 the child function may need additional arguments in the case of
712 GIMPLE_OMP_FOR regions. In some cases, these arguments are
713 computed out of variables passed in from the parent to the child
714 via 'struct .omp_data_s'. For instance:
716 #pragma omp parallel for schedule (guided, i * 4)
721 # BLOCK 2 (PAR_ENTRY_BB)
723 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
725 # BLOCK 3 (WS_ENTRY_BB)
726 .omp_data_i = &.omp_data_o;
727 D.1667 = .omp_data_i->i;
729 #pragma omp for schedule (guided, D.1598)
731 When we outline the parallel region, the call to the child function
732 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
733 that value is computed *after* the call site. So, in principle we
734 cannot do the transformation.
736 To see whether the code in WS_ENTRY_BB blocks the combined
737 parallel+workshare call, we collect all the variables used in the
738 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
739 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
742 FIXME. If we had the SSA form built at this point, we could merely
743 hoist the code in block 3 into block 2 and be done with it. But at
744 this point we don't have dataflow information and though we could
745 hack something up here, it is really not worth the aggravation. */
748 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
750 struct omp_for_data fd
;
751 gimple ws_stmt
= last_stmt (ws_entry_bb
);
753 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
756 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
758 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
760 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
762 if (fd
.iter_type
!= long_integer_type_node
)
765 /* FIXME. We give up too easily here. If any of these arguments
766 are not constants, they will likely involve variables that have
767 been mapped into fields of .omp_data_s for sharing with the child
768 function. With appropriate data flow, it would be possible to
770 if (!is_gimple_min_invariant (fd
.loop
.n1
)
771 || !is_gimple_min_invariant (fd
.loop
.n2
)
772 || !is_gimple_min_invariant (fd
.loop
.step
)
773 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
780 /* Collect additional arguments needed to emit a combined
781 parallel+workshare call. WS_STMT is the workshare directive being
784 static vec
<tree
, va_gc
> *
785 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
788 location_t loc
= gimple_location (ws_stmt
);
789 vec
<tree
, va_gc
> *ws_args
;
791 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
793 struct omp_for_data fd
;
796 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
800 if (gimple_omp_for_combined_into_p (ws_stmt
))
803 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
804 OMP_CLAUSE__LOOPTEMP_
);
806 n1
= OMP_CLAUSE_DECL (innerc
);
807 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
808 OMP_CLAUSE__LOOPTEMP_
);
810 n2
= OMP_CLAUSE_DECL (innerc
);
813 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
815 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
816 ws_args
->quick_push (t
);
818 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
819 ws_args
->quick_push (t
);
821 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
822 ws_args
->quick_push (t
);
826 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
827 ws_args
->quick_push (t
);
832 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
834 /* Number of sections is equal to the number of edges from the
835 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
836 the exit of the sections region. */
837 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
838 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
839 vec_alloc (ws_args
, 1);
840 ws_args
->quick_push (t
);
848 /* Discover whether REGION is a combined parallel+workshare region. */
851 determine_parallel_type (struct omp_region
*region
)
853 basic_block par_entry_bb
, par_exit_bb
;
854 basic_block ws_entry_bb
, ws_exit_bb
;
856 if (region
== NULL
|| region
->inner
== NULL
857 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
858 || region
->inner
->cont
== NULL
)
861 /* We only support parallel+for and parallel+sections. */
862 if (region
->type
!= GIMPLE_OMP_PARALLEL
863 || (region
->inner
->type
!= GIMPLE_OMP_FOR
864 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
867 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
868 WS_EXIT_BB -> PAR_EXIT_BB. */
869 par_entry_bb
= region
->entry
;
870 par_exit_bb
= region
->exit
;
871 ws_entry_bb
= region
->inner
->entry
;
872 ws_exit_bb
= region
->inner
->exit
;
874 if (single_succ (par_entry_bb
) == ws_entry_bb
875 && single_succ (ws_exit_bb
) == par_exit_bb
876 && workshare_safe_to_combine_p (ws_entry_bb
)
877 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
878 || (last_and_only_stmt (ws_entry_bb
)
879 && last_and_only_stmt (par_exit_bb
))))
881 gimple par_stmt
= last_stmt (par_entry_bb
);
882 gimple ws_stmt
= last_stmt (ws_entry_bb
);
884 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
886 /* If this is a combined parallel loop, we need to determine
887 whether or not to use the combined library calls. There
888 are two cases where we do not apply the transformation:
889 static loops and any kind of ordered loop. In the first
890 case, we already open code the loop so there is no need
891 to do anything else. In the latter case, the combined
892 parallel loop call would still need extra synchronization
893 to implement ordered semantics, so there would not be any
894 gain in using the combined call. */
895 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
896 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
898 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
899 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
901 region
->is_combined_parallel
= false;
902 region
->inner
->is_combined_parallel
= false;
907 region
->is_combined_parallel
= true;
908 region
->inner
->is_combined_parallel
= true;
909 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
914 /* Return true if EXPR is variable sized. */
917 is_variable_sized (const_tree expr
)
919 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
922 /* Return true if DECL is a reference type. */
925 is_reference (tree decl
)
927 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
930 /* Return the type of a decl. If the decl is reference type,
931 return its base type. */
933 get_base_type (tree decl
)
935 tree type
= TREE_TYPE (decl
);
936 if (is_reference (decl
))
937 type
= TREE_TYPE (type
);
941 /* Lookup variables. The "maybe" form
942 allows for the variable form to not have been entered, otherwise we
943 assert that the variable must have been entered. */
946 lookup_decl (tree var
, omp_context
*ctx
)
948 tree
*n
= ctx
->cb
.decl_map
->get (var
);
953 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
955 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
956 return n
? *n
: NULL_TREE
;
960 lookup_field (tree var
, omp_context
*ctx
)
963 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
964 return (tree
) n
->value
;
968 lookup_sfield (tree var
, omp_context
*ctx
)
971 n
= splay_tree_lookup (ctx
->sfield_map
972 ? ctx
->sfield_map
: ctx
->field_map
,
973 (splay_tree_key
) var
);
974 return (tree
) n
->value
;
978 maybe_lookup_field (tree var
, omp_context
*ctx
)
981 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
982 return n
? (tree
) n
->value
: NULL_TREE
;
986 lookup_reduction (const char *id
, omp_context
*ctx
)
988 gcc_assert (is_gimple_omp_oacc_specifically (ctx
->stmt
));
991 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
992 return (tree
) n
->value
;
996 maybe_lookup_reduction (tree var
, omp_context
*ctx
)
998 splay_tree_node n
= NULL
;
999 if (ctx
->reduction_map
)
1000 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
1001 return n
? (tree
) n
->value
: NULL_TREE
;
1004 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1005 the parallel context if DECL is to be shared. */
1008 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1010 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1013 /* We can only use copy-in/copy-out semantics for shared variables
1014 when we know the value is not accessible from an outer scope. */
1017 gcc_assert (!is_gimple_omp_oacc_specifically (shared_ctx
->stmt
));
1019 /* ??? Trivially accessible from anywhere. But why would we even
1020 be passing an address in this case? Should we simply assert
1021 this to be false, or should we have a cleanup pass that removes
1022 these from the list of mappings? */
1023 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1026 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1027 without analyzing the expression whether or not its location
1028 is accessible to anyone else. In the case of nested parallel
1029 regions it certainly may be. */
1030 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1033 /* Do not use copy-in/copy-out for variables that have their
1035 if (TREE_ADDRESSABLE (decl
))
1038 /* lower_send_shared_vars only uses copy-in, but not copy-out
1040 if (TREE_READONLY (decl
)
1041 || ((TREE_CODE (decl
) == RESULT_DECL
1042 || TREE_CODE (decl
) == PARM_DECL
)
1043 && DECL_BY_REFERENCE (decl
)))
1046 /* Disallow copy-in/out in nested parallel if
1047 decl is shared in outer parallel, otherwise
1048 each thread could store the shared variable
1049 in its own copy-in location, making the
1050 variable no longer really shared. */
1051 if (shared_ctx
->is_nested
)
1055 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1056 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1063 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1064 c
; c
= OMP_CLAUSE_CHAIN (c
))
1065 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1066 && OMP_CLAUSE_DECL (c
) == decl
)
1070 goto maybe_mark_addressable_and_ret
;
1074 /* For tasks avoid using copy-in/out. As tasks can be
1075 deferred or executed in different thread, when GOMP_task
1076 returns, the task hasn't necessarily terminated. */
1077 if (is_task_ctx (shared_ctx
))
1080 maybe_mark_addressable_and_ret
:
1081 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1082 if (is_gimple_reg (outer
))
1084 /* Taking address of OUTER in lower_send_shared_vars
1085 might need regimplification of everything that uses the
1087 if (!task_shared_vars
)
1088 task_shared_vars
= BITMAP_ALLOC (NULL
);
1089 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1090 TREE_ADDRESSABLE (outer
) = 1;
1099 /* Construct a new automatic decl similar to VAR. */
1102 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1104 tree copy
= copy_var_decl (var
, name
, type
);
1106 DECL_CONTEXT (copy
) = current_function_decl
;
1107 DECL_CHAIN (copy
) = ctx
->block_vars
;
1108 ctx
->block_vars
= copy
;
1114 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1116 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1119 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1122 omp_build_component_ref (tree obj
, tree field
)
1124 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1125 if (TREE_THIS_VOLATILE (field
))
1126 TREE_THIS_VOLATILE (ret
) |= 1;
1127 if (TREE_READONLY (field
))
1128 TREE_READONLY (ret
) |= 1;
1132 /* Build tree nodes to access the field for VAR on the receiver side. */
1135 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1137 tree x
, field
= lookup_field (var
, ctx
);
1139 /* If the receiver record type was remapped in the child function,
1140 remap the field into the new record type. */
1141 x
= maybe_lookup_field (field
, ctx
);
1145 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1146 x
= omp_build_component_ref (x
, field
);
1148 x
= build_simple_mem_ref (x
);
1153 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1154 of a parallel, this is a component reference; for workshare constructs
1155 this is some variable. */
1158 build_outer_var_ref (tree var
, omp_context
*ctx
)
1162 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1164 else if (is_variable_sized (var
))
1166 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1167 x
= build_outer_var_ref (x
, ctx
);
1168 x
= build_simple_mem_ref (x
);
1170 else if (is_taskreg_ctx (ctx
))
1172 bool by_ref
= use_pointer_for_field (var
, NULL
);
1173 x
= build_receiver_ref (var
, by_ref
, ctx
);
1175 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1176 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1178 /* #pragma omp simd isn't a worksharing construct, and can reference even
1179 private vars in its linear etc. clauses. */
1181 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1182 x
= lookup_decl (var
, ctx
->outer
);
1183 else if (ctx
->outer
)
1184 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1188 else if (ctx
->outer
)
1189 x
= lookup_decl (var
, ctx
->outer
);
1190 else if (is_reference (var
))
1191 /* This can happen with orphaned constructs. If var is reference, it is
1192 possible it is shared and as such valid. */
1197 if (is_reference (var
))
1198 x
= build_simple_mem_ref (x
);
1203 /* Build tree nodes to access the field for VAR on the sender side. */
1206 build_sender_ref (tree var
, omp_context
*ctx
)
1208 tree field
= lookup_sfield (var
, ctx
);
1209 return omp_build_component_ref (ctx
->sender_decl
, field
);
1212 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1215 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1217 tree field
, type
, sfield
= NULL_TREE
;
1219 gcc_assert ((mask
& 1) == 0
1220 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1221 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1222 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1223 gcc_assert ((mask
& 3) == 3
1224 || !is_gimple_omp_oacc_specifically (ctx
->stmt
));
1226 type
= TREE_TYPE (var
);
1229 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1230 type
= build_pointer_type (build_pointer_type (type
));
1233 type
= build_pointer_type (type
);
1234 else if ((mask
& 3) == 1 && is_reference (var
))
1235 type
= TREE_TYPE (type
);
1237 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1238 FIELD_DECL
, DECL_NAME (var
), type
);
1240 /* Remember what variable this field was created for. This does have a
1241 side effect of making dwarf2out ignore this member, so for helpful
1242 debugging we clear it later in delete_omp_context. */
1243 DECL_ABSTRACT_ORIGIN (field
) = var
;
1244 if (type
== TREE_TYPE (var
))
1246 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1247 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1248 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1251 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1253 if ((mask
& 3) == 3)
1255 insert_field_into_struct (ctx
->record_type
, field
);
1256 if (ctx
->srecord_type
)
1258 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1259 FIELD_DECL
, DECL_NAME (var
), type
);
1260 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1261 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1262 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1263 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1264 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1269 if (ctx
->srecord_type
== NULL_TREE
)
1273 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1274 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1275 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1277 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1278 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1279 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1280 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1281 splay_tree_insert (ctx
->sfield_map
,
1282 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1283 (splay_tree_value
) sfield
);
1287 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1288 : ctx
->srecord_type
, field
);
1292 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1293 (splay_tree_value
) field
);
1294 if ((mask
& 2) && ctx
->sfield_map
)
1295 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1296 (splay_tree_value
) sfield
);
1300 install_var_local (tree var
, omp_context
*ctx
)
1302 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1303 insert_decl_map (&ctx
->cb
, var
, new_var
);
1307 /* Adjust the replacement for DECL in CTX for the new context. This means
1308 copying the DECL_VALUE_EXPR, and fixing up the type. */
1311 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1313 tree new_decl
, size
;
1315 new_decl
= lookup_decl (decl
, ctx
);
1317 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1319 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1320 && DECL_HAS_VALUE_EXPR_P (decl
))
1322 tree ve
= DECL_VALUE_EXPR (decl
);
1323 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1324 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1325 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1328 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1330 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1331 if (size
== error_mark_node
)
1332 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1333 DECL_SIZE (new_decl
) = size
;
1335 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1336 if (size
== error_mark_node
)
1337 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1338 DECL_SIZE_UNIT (new_decl
) = size
;
1342 /* The callback for remap_decl. Search all containing contexts for a
1343 mapping of the variable; this avoids having to duplicate the splay
1344 tree ahead of time. We know a mapping doesn't already exist in the
1345 given context. Create new mappings to implement default semantics. */
1348 omp_copy_decl (tree var
, copy_body_data
*cb
)
1350 omp_context
*ctx
= (omp_context
*) cb
;
1353 if (TREE_CODE (var
) == LABEL_DECL
)
1355 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1356 DECL_CONTEXT (new_var
) = current_function_decl
;
1357 insert_decl_map (&ctx
->cb
, var
, new_var
);
1361 while (!is_taskreg_ctx (ctx
))
1366 new_var
= maybe_lookup_decl (var
, ctx
);
1371 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1374 return error_mark_node
;
1378 /* Debugging dumps for parallel regions. */
1379 void dump_omp_region (FILE *, struct omp_region
*, int);
1380 void debug_omp_region (struct omp_region
*);
1381 void debug_all_omp_regions (void);
1383 /* Dump the parallel region tree rooted at REGION. */
1386 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1388 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1389 gimple_code_name
[region
->type
]);
1392 dump_omp_region (file
, region
->inner
, indent
+ 4);
1396 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1397 region
->cont
->index
);
1401 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1402 region
->exit
->index
);
1404 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1407 dump_omp_region (file
, region
->next
, indent
);
1411 debug_omp_region (struct omp_region
*region
)
1413 dump_omp_region (stderr
, region
, 0);
1417 debug_all_omp_regions (void)
1419 dump_omp_region (stderr
, root_omp_region
, 0);
1423 /* Create a new parallel region starting at STMT inside region PARENT. */
1425 static struct omp_region
*
1426 new_omp_region (basic_block bb
, enum gimple_code type
,
1427 struct omp_region
*parent
)
1429 struct omp_region
*region
= XCNEW (struct omp_region
);
1431 region
->outer
= parent
;
1433 region
->type
= type
;
1437 /* This is a nested region. Add it to the list of inner
1438 regions in PARENT. */
1439 region
->next
= parent
->inner
;
1440 parent
->inner
= region
;
1444 /* This is a toplevel region. Add it to the list of toplevel
1445 regions in ROOT_OMP_REGION. */
1446 region
->next
= root_omp_region
;
1447 root_omp_region
= region
;
1453 /* Release the memory associated with the region tree rooted at REGION. */
1456 free_omp_region_1 (struct omp_region
*region
)
1458 struct omp_region
*i
, *n
;
1460 for (i
= region
->inner
; i
; i
= n
)
1463 free_omp_region_1 (i
);
1469 /* Release the memory for the entire omp region tree. */
1472 free_omp_regions (void)
1474 struct omp_region
*r
, *n
;
1475 for (r
= root_omp_region
; r
; r
= n
)
1478 free_omp_region_1 (r
);
1480 root_omp_region
= NULL
;
1484 /* Create a new context, with OUTER_CTX being the surrounding context. */
1486 static omp_context
*
1487 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1489 omp_context
*ctx
= XCNEW (omp_context
);
1491 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1492 (splay_tree_value
) ctx
);
1497 ctx
->outer
= outer_ctx
;
1498 ctx
->cb
= outer_ctx
->cb
;
1499 ctx
->cb
.block
= NULL
;
1500 ctx
->depth
= outer_ctx
->depth
+ 1;
1501 /* FIXME: handle reductions recursively. */
1502 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1506 ctx
->cb
.src_fn
= current_function_decl
;
1507 ctx
->cb
.dst_fn
= current_function_decl
;
1508 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1509 gcc_checking_assert (ctx
->cb
.src_node
);
1510 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1511 ctx
->cb
.src_cfun
= cfun
;
1512 ctx
->cb
.copy_decl
= omp_copy_decl
;
1513 ctx
->cb
.eh_lp_nr
= 0;
1514 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1516 //TODO ctx->reduction_map = TODO;
1519 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1524 static gimple_seq
maybe_catch_exception (gimple_seq
);
1526 /* Finalize task copyfn. */
1529 finalize_task_copyfn (gimple task_stmt
)
1531 struct function
*child_cfun
;
1533 gimple_seq seq
= NULL
, new_seq
;
1536 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1537 if (child_fn
== NULL_TREE
)
1540 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1541 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1543 push_cfun (child_cfun
);
1544 bind
= gimplify_body (child_fn
, false);
1545 gimple_seq_add_stmt (&seq
, bind
);
1546 new_seq
= maybe_catch_exception (seq
);
1549 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1551 gimple_seq_add_stmt (&seq
, bind
);
1553 gimple_set_body (child_fn
, seq
);
1556 /* Inform the callgraph about the new function. */
1557 cgraph_node::add_new_function (child_fn
, false);
1560 /* Destroy a omp_context data structures. Called through the splay tree
1561 value delete callback. */
1564 delete_omp_context (splay_tree_value value
)
1566 omp_context
*ctx
= (omp_context
*) value
;
1568 delete ctx
->cb
.decl_map
;
1571 splay_tree_delete (ctx
->field_map
);
1572 if (ctx
->sfield_map
)
1573 splay_tree_delete (ctx
->sfield_map
);
1574 if (ctx
->reduction_map
1575 /* Shared over several omp_contexts. */
1576 && (ctx
->outer
== NULL
1577 || ctx
->reduction_map
!= ctx
->outer
->reduction_map
))
1578 splay_tree_delete (ctx
->reduction_map
);
1580 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1581 it produces corrupt debug information. */
1582 if (ctx
->record_type
)
1585 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1586 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1588 if (ctx
->srecord_type
)
1591 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1592 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1595 if (is_task_ctx (ctx
))
1596 finalize_task_copyfn (ctx
->stmt
);
1601 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1605 fixup_child_record_type (omp_context
*ctx
)
1607 tree f
, type
= ctx
->record_type
;
1609 /* ??? It isn't sufficient to just call remap_type here, because
1610 variably_modified_type_p doesn't work the way we expect for
1611 record types. Testing each field for whether it needs remapping
1612 and creating a new record by hand works, however. */
1613 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1614 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1618 tree name
, new_fields
= NULL
;
1620 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1621 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1622 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1623 TYPE_DECL
, name
, type
);
1624 TYPE_NAME (type
) = name
;
1626 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1628 tree new_f
= copy_node (f
);
1629 DECL_CONTEXT (new_f
) = type
;
1630 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1631 DECL_CHAIN (new_f
) = new_fields
;
1632 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1633 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1635 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1639 /* Arrange to be able to look up the receiver field
1640 given the sender field. */
1641 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1642 (splay_tree_value
) new_f
);
1644 TYPE_FIELDS (type
) = nreverse (new_fields
);
1648 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1651 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1652 specified by CLAUSES. */
1655 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1658 bool scan_array_reductions
= false;
1660 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1664 switch (OMP_CLAUSE_CODE (c
))
1666 case OMP_CLAUSE_PRIVATE
:
1667 decl
= OMP_CLAUSE_DECL (c
);
1668 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1670 else if (!is_variable_sized (decl
))
1671 install_var_local (decl
, ctx
);
1674 case OMP_CLAUSE_SHARED
:
1675 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1676 decl
= OMP_CLAUSE_DECL (c
);
1677 /* Ignore shared directives in teams construct. */
1678 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1680 /* Global variables don't need to be copied,
1681 the receiver side will use them directly. */
1682 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1683 if (is_global_var (odecl
))
1685 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1688 gcc_assert (is_taskreg_ctx (ctx
));
1689 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1690 || !is_variable_sized (decl
));
1691 /* Global variables don't need to be copied,
1692 the receiver side will use them directly. */
1693 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1695 by_ref
= use_pointer_for_field (decl
, ctx
);
1696 if (! TREE_READONLY (decl
)
1697 || TREE_ADDRESSABLE (decl
)
1699 || is_reference (decl
))
1701 install_var_field (decl
, by_ref
, 3, ctx
);
1702 install_var_local (decl
, ctx
);
1705 /* We don't need to copy const scalar vars back. */
1706 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1709 case OMP_CLAUSE_LASTPRIVATE
:
1710 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1711 /* Let the corresponding firstprivate clause create
1713 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1717 case OMP_CLAUSE_FIRSTPRIVATE
:
1718 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
1720 sorry ("clause not supported yet");
1724 case OMP_CLAUSE_LINEAR
:
1725 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1727 case OMP_CLAUSE_REDUCTION
:
1728 decl
= OMP_CLAUSE_DECL (c
);
1730 if (is_variable_sized (decl
))
1732 if (is_task_ctx (ctx
))
1733 install_var_field (decl
, false, 1, ctx
);
1736 else if (is_taskreg_ctx (ctx
))
1739 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1740 by_ref
= use_pointer_for_field (decl
, NULL
);
1742 if (is_task_ctx (ctx
)
1743 && (global
|| by_ref
|| is_reference (decl
)))
1745 install_var_field (decl
, false, 1, ctx
);
1747 install_var_field (decl
, by_ref
, 2, ctx
);
1750 install_var_field (decl
, by_ref
, 3, ctx
);
1752 install_var_local (decl
, ctx
);
1754 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
1756 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
1758 /* Create a decl for the reduction array. */
1759 tree var
= OMP_CLAUSE_DECL (c
);
1760 tree type
= get_base_type (var
);
1761 tree ptype
= build_pointer_type (type
);
1762 tree array
= create_tmp_var (ptype
, omp_get_id (var
));
1763 omp_context
*c
= (ctx
->field_map
? ctx
: ctx
->outer
);
1764 install_var_field (array
, true, 3, c
);
1765 install_var_local (array
, c
);
1767 /* Insert it into the current context. */
1768 splay_tree_insert (ctx
->reduction_map
,
1769 (splay_tree_key
) omp_get_id(var
),
1770 (splay_tree_value
) array
);
1771 splay_tree_insert (ctx
->reduction_map
,
1772 (splay_tree_key
) array
,
1773 (splay_tree_value
) array
);
1778 case OMP_CLAUSE__LOOPTEMP_
:
1779 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1780 gcc_assert (is_parallel_ctx (ctx
));
1781 decl
= OMP_CLAUSE_DECL (c
);
1782 install_var_field (decl
, false, 3, ctx
);
1783 install_var_local (decl
, ctx
);
1786 case OMP_CLAUSE_COPYPRIVATE
:
1787 case OMP_CLAUSE_COPYIN
:
1788 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1789 decl
= OMP_CLAUSE_DECL (c
);
1790 by_ref
= use_pointer_for_field (decl
, NULL
);
1791 install_var_field (decl
, by_ref
, 3, ctx
);
1794 case OMP_CLAUSE_DEFAULT
:
1795 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1796 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1799 case OMP_CLAUSE_FINAL
:
1800 case OMP_CLAUSE_NUM_THREADS
:
1801 case OMP_CLAUSE_NUM_TEAMS
:
1802 case OMP_CLAUSE_THREAD_LIMIT
:
1803 case OMP_CLAUSE_DEVICE
:
1804 case OMP_CLAUSE_SCHEDULE
:
1805 case OMP_CLAUSE_DIST_SCHEDULE
:
1806 case OMP_CLAUSE_DEPEND
:
1807 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1808 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1811 case OMP_CLAUSE_NUM_GANGS
:
1812 case OMP_CLAUSE_NUM_WORKERS
:
1813 case OMP_CLAUSE_VECTOR_LENGTH
:
1815 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1819 case OMP_CLAUSE_FROM
:
1820 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1822 case OMP_CLAUSE_MAP
:
1823 switch (OMP_CLAUSE_CODE (c
))
1826 case OMP_CLAUSE_FROM
:
1827 /* The to and from clauses are only ever seen with OpenMP target
1828 update constructs. */
1829 gcc_assert (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
1830 && (gimple_omp_target_kind (ctx
->stmt
)
1831 == GF_OMP_TARGET_KIND_UPDATE
));
1833 case OMP_CLAUSE_MAP
:
1834 /* The map clause is never seen with OpenMP target update
1836 gcc_assert (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
1837 || (gimple_omp_target_kind (ctx
->stmt
)
1838 != GF_OMP_TARGET_KIND_UPDATE
));
1844 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1845 decl
= OMP_CLAUSE_DECL (c
);
1846 /* Global variables with "omp declare target" attribute
1847 don't need to be copied, the receiver side will use them
1849 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1851 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1852 && lookup_attribute ("omp declare target",
1853 DECL_ATTRIBUTES (decl
)))
1855 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1858 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1859 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1861 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1862 target regions that are not offloaded; there is nothing to map for
1864 if (!is_gimple_omp_offloaded (ctx
->stmt
)
1865 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1869 /* In target regions that are not offloaded, libgomp won't pay
1870 attention to OMP_CLAUSE_MAP_FORCE_DEVICEPTR -- but I think we need
1871 to handle it here anyway, in order to create a visible copy of the
1873 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1874 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
1876 if (!is_gimple_omp_offloaded (ctx
->stmt
))
1882 if (DECL_SIZE (decl
)
1883 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1885 tree decl2
= DECL_VALUE_EXPR (decl
);
1886 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1887 decl2
= TREE_OPERAND (decl2
, 0);
1888 gcc_assert (DECL_P (decl2
));
1889 install_var_field (decl2
, true, 3, ctx
);
1890 install_var_local (decl2
, ctx
);
1891 install_var_local (decl
, ctx
);
1895 gcc_assert (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
1896 || (OMP_CLAUSE_MAP_KIND (c
)
1897 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
1898 || TREE_CODE (TREE_TYPE (decl
)) != ARRAY_TYPE
);
1899 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1900 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1901 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1902 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1903 install_var_field (decl
, true, 7, ctx
);
1906 if (!is_gimple_omp_oacc_specifically (ctx
->stmt
))
1907 install_var_field (decl
, true, 3, ctx
);
1910 /* decl goes heres. */
1911 omp_context
*c
= (ctx
->field_map
? ctx
: ctx
->outer
);
1912 install_var_field (decl
, true, 3, c
);
1915 if (is_gimple_omp_offloaded (ctx
->stmt
))
1916 install_var_local (decl
, ctx
);
1921 tree base
= get_base_address (decl
);
1922 tree nc
= OMP_CLAUSE_CHAIN (c
);
1925 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1926 && OMP_CLAUSE_DECL (nc
) == base
1927 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1928 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1930 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1931 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1937 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1938 decl
= OMP_CLAUSE_DECL (c
);
1940 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1941 (splay_tree_key
) decl
));
1943 = build_decl (OMP_CLAUSE_LOCATION (c
),
1944 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1945 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1946 insert_field_into_struct (ctx
->record_type
, field
);
1947 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1948 (splay_tree_value
) field
);
1953 case OMP_CLAUSE_NOWAIT
:
1954 case OMP_CLAUSE_ORDERED
:
1955 case OMP_CLAUSE_UNTIED
:
1956 case OMP_CLAUSE_MERGEABLE
:
1957 case OMP_CLAUSE_PROC_BIND
:
1958 case OMP_CLAUSE_SAFELEN
:
1959 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
1961 sorry ("clause not supported yet");
1965 case OMP_CLAUSE_COLLAPSE
:
1966 case OMP_CLAUSE_ASYNC
:
1967 case OMP_CLAUSE_WAIT
:
1970 case OMP_CLAUSE_ALIGNED
:
1971 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1972 decl
= OMP_CLAUSE_DECL (c
);
1973 if (is_global_var (decl
)
1974 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1975 install_var_local (decl
, ctx
);
1978 case OMP_CLAUSE_DEVICE_RESIDENT
:
1979 case OMP_CLAUSE_USE_DEVICE
:
1980 case OMP_CLAUSE_GANG
:
1981 case OMP_CLAUSE__CACHE_
:
1982 case OMP_CLAUSE_INDEPENDENT
:
1983 case OMP_CLAUSE_WORKER
:
1984 case OMP_CLAUSE_VECTOR
:
1985 sorry ("Clause not supported yet");
1993 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1995 switch (OMP_CLAUSE_CODE (c
))
1997 case OMP_CLAUSE_LASTPRIVATE
:
1998 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1999 /* Let the corresponding firstprivate clause create
2001 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2002 scan_array_reductions
= true;
2003 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2007 case OMP_CLAUSE_FIRSTPRIVATE
:
2008 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
2010 sorry ("clause not supported yet");
2014 case OMP_CLAUSE_LINEAR
:
2015 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
2017 case OMP_CLAUSE_REDUCTION
:
2018 case OMP_CLAUSE_PRIVATE
:
2019 decl
= OMP_CLAUSE_DECL (c
);
2020 if (is_variable_sized (decl
))
2021 install_var_local (decl
, ctx
);
2022 fixup_remapped_decl (decl
, ctx
,
2023 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2024 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2025 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2026 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2027 scan_array_reductions
= true;
2028 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2029 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2030 scan_array_reductions
= true;
2033 case OMP_CLAUSE_SHARED
:
2034 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
2035 /* Ignore shared directives in teams construct. */
2036 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2038 decl
= OMP_CLAUSE_DECL (c
);
2039 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2040 fixup_remapped_decl (decl
, ctx
, false);
2043 case OMP_CLAUSE_MAP
:
2044 /* The map clause is never seen with OpenMP target update
2046 gcc_assert (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2047 || (gimple_omp_target_kind (ctx
->stmt
)
2048 != GF_OMP_TARGET_KIND_UPDATE
));
2049 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2051 decl
= OMP_CLAUSE_DECL (c
);
2053 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2054 && lookup_attribute ("omp declare target",
2055 DECL_ATTRIBUTES (decl
)))
2057 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
2062 gcc_assert ((OMP_CLAUSE_MAP_KIND (c
)
2063 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
2064 || TREE_CODE (TREE_TYPE (decl
)) != ARRAY_TYPE
);
2065 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
2066 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2067 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2069 tree new_decl
= lookup_decl (decl
, ctx
);
2070 TREE_TYPE (new_decl
)
2071 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2073 else if (DECL_SIZE (decl
)
2074 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2076 gcc_assert (OMP_CLAUSE_MAP_KIND (c
)
2077 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR
);
2079 tree decl2
= DECL_VALUE_EXPR (decl
);
2080 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2081 decl2
= TREE_OPERAND (decl2
, 0);
2082 gcc_assert (DECL_P (decl2
));
2083 fixup_remapped_decl (decl2
, ctx
, false);
2084 fixup_remapped_decl (decl
, ctx
, true);
2087 fixup_remapped_decl (decl
, ctx
, false);
2091 case OMP_CLAUSE_COPYPRIVATE
:
2092 case OMP_CLAUSE_COPYIN
:
2093 case OMP_CLAUSE_DEFAULT
:
2094 case OMP_CLAUSE_NUM_THREADS
:
2095 case OMP_CLAUSE_NUM_TEAMS
:
2096 case OMP_CLAUSE_THREAD_LIMIT
:
2097 case OMP_CLAUSE_DEVICE
:
2098 case OMP_CLAUSE_SCHEDULE
:
2099 case OMP_CLAUSE_DIST_SCHEDULE
:
2100 case OMP_CLAUSE_NOWAIT
:
2101 case OMP_CLAUSE_ORDERED
:
2102 case OMP_CLAUSE_UNTIED
:
2103 case OMP_CLAUSE_FINAL
:
2104 case OMP_CLAUSE_MERGEABLE
:
2105 case OMP_CLAUSE_PROC_BIND
:
2106 case OMP_CLAUSE_SAFELEN
:
2107 case OMP_CLAUSE_ALIGNED
:
2108 case OMP_CLAUSE_DEPEND
:
2109 case OMP_CLAUSE__LOOPTEMP_
:
2111 case OMP_CLAUSE_FROM
:
2112 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2113 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
2115 case OMP_CLAUSE_ASYNC
:
2116 case OMP_CLAUSE_COLLAPSE
:
2118 case OMP_CLAUSE_NUM_GANGS
:
2119 case OMP_CLAUSE_NUM_WORKERS
:
2120 case OMP_CLAUSE_VECTOR_LENGTH
:
2121 case OMP_CLAUSE_WAIT
:
2124 case OMP_CLAUSE_DEVICE_RESIDENT
:
2125 case OMP_CLAUSE_USE_DEVICE
:
2126 case OMP_CLAUSE_GANG
:
2127 case OMP_CLAUSE__CACHE_
:
2128 case OMP_CLAUSE_INDEPENDENT
:
2129 case OMP_CLAUSE_WORKER
:
2130 case OMP_CLAUSE_VECTOR
:
2131 sorry ("Clause not supported yet");
2139 if (scan_array_reductions
)
2141 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
2142 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2143 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2144 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2146 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2147 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2149 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2150 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2151 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2152 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2153 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2154 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2158 /* Create a new name for omp child function. Returns an identifier. If
2159 IS_CILK_FOR is true then the suffix for the child function is
2163 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2166 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2167 return clone_function_name (current_function_decl
,
2168 task_copy
? "_omp_cpyfn" : "_omp_fn");
2171 /* Returns the type of the induction variable for the child function for
2172 _Cilk_for and the types for _high and _low variables based on TYPE. */
2175 cilk_for_check_loop_diff_type (tree type
)
2177 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2179 if (TYPE_UNSIGNED (type
))
2180 return uint32_type_node
;
2182 return integer_type_node
;
2186 if (TYPE_UNSIGNED (type
))
2187 return uint64_type_node
;
2189 return long_long_integer_type_node
;
2193 /* Build a decl for the omp child function. It'll not contain a body
2194 yet, just the bare decl. */
2197 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2199 tree decl
, type
, name
, t
;
2202 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2203 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2204 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2205 tree cilk_var_type
= NULL_TREE
;
2207 name
= create_omp_child_function_name (task_copy
,
2208 cilk_for_count
!= NULL_TREE
);
2210 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2211 ptr_type_node
, NULL_TREE
);
2212 else if (cilk_for_count
)
2214 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2215 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2216 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2217 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2220 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2222 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2224 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
)
2227 ctx
->cb
.dst_fn
= decl
;
2229 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2231 TREE_STATIC (decl
) = 1;
2232 TREE_USED (decl
) = 1;
2233 DECL_ARTIFICIAL (decl
) = 1;
2234 DECL_IGNORED_P (decl
) = 0;
2235 TREE_PUBLIC (decl
) = 0;
2236 DECL_UNINLINABLE (decl
) = 1;
2237 DECL_EXTERNAL (decl
) = 0;
2238 DECL_CONTEXT (decl
) = NULL_TREE
;
2239 DECL_INITIAL (decl
) = make_node (BLOCK
);
2240 bool target_p
= false;
2241 if (lookup_attribute ("omp declare target",
2242 DECL_ATTRIBUTES (current_function_decl
)))
2247 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2248 if (is_gimple_omp_offloaded (octx
->stmt
))
2255 DECL_ATTRIBUTES (decl
)
2256 = tree_cons (get_identifier ("omp declare target"),
2257 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2259 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2260 RESULT_DECL
, NULL_TREE
, void_type_node
);
2261 DECL_ARTIFICIAL (t
) = 1;
2262 DECL_IGNORED_P (t
) = 1;
2263 DECL_CONTEXT (t
) = decl
;
2264 DECL_RESULT (decl
) = t
;
2266 /* _Cilk_for's child function requires two extra parameters called
2267 __low and __high that are set the by Cilk runtime when it calls this
2271 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2272 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2273 DECL_ARTIFICIAL (t
) = 1;
2274 DECL_NAMELESS (t
) = 1;
2275 DECL_ARG_TYPE (t
) = ptr_type_node
;
2276 DECL_CONTEXT (t
) = current_function_decl
;
2278 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2279 DECL_ARGUMENTS (decl
) = t
;
2281 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2282 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2283 DECL_ARTIFICIAL (t
) = 1;
2284 DECL_NAMELESS (t
) = 1;
2285 DECL_ARG_TYPE (t
) = ptr_type_node
;
2286 DECL_CONTEXT (t
) = current_function_decl
;
2288 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2289 DECL_ARGUMENTS (decl
) = t
;
2292 tree data_name
= get_identifier (".omp_data_i");
2293 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2295 DECL_ARTIFICIAL (t
) = 1;
2296 DECL_NAMELESS (t
) = 1;
2297 DECL_ARG_TYPE (t
) = ptr_type_node
;
2298 DECL_CONTEXT (t
) = current_function_decl
;
2301 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2302 DECL_ARGUMENTS (decl
) = t
;
2304 ctx
->receiver_decl
= t
;
2307 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2308 PARM_DECL
, get_identifier (".omp_data_o"),
2310 DECL_ARTIFICIAL (t
) = 1;
2311 DECL_NAMELESS (t
) = 1;
2312 DECL_ARG_TYPE (t
) = ptr_type_node
;
2313 DECL_CONTEXT (t
) = current_function_decl
;
2315 TREE_ADDRESSABLE (t
) = 1;
2316 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2317 DECL_ARGUMENTS (decl
) = t
;
2320 /* Allocate memory for the function structure. The call to
2321 allocate_struct_function clobbers CFUN, so we need to restore
2323 push_struct_function (decl
);
2324 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2328 /* Callback for walk_gimple_seq. Check if combined parallel
2329 contains gimple_omp_for_combined_into_p OMP_FOR. */
2332 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2333 bool *handled_ops_p
,
2334 struct walk_stmt_info
*wi
)
2336 gimple stmt
= gsi_stmt (*gsi_p
);
2338 *handled_ops_p
= true;
2339 switch (gimple_code (stmt
))
2343 case GIMPLE_OMP_FOR
:
2344 if (gimple_omp_for_combined_into_p (stmt
)
2345 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2348 return integer_zero_node
;
2357 /* Scan an OpenMP parallel directive. */
2360 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2364 gimple stmt
= gsi_stmt (*gsi
);
2366 /* Ignore parallel directives with empty bodies, unless there
2367 are copyin clauses. */
2369 && empty_body_p (gimple_omp_body (stmt
))
2370 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2371 OMP_CLAUSE_COPYIN
) == NULL
)
2373 gsi_replace (gsi
, gimple_build_nop (), false);
2377 if (gimple_omp_parallel_combined_p (stmt
))
2380 struct walk_stmt_info wi
;
2382 memset (&wi
, 0, sizeof (wi
));
2384 walk_gimple_seq (gimple_omp_body (stmt
),
2385 find_combined_for
, NULL
, &wi
);
2386 for_stmt
= (gimple
) wi
.info
;
2389 struct omp_for_data fd
;
2390 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2391 /* We need two temporaries with fd.loop.v type (istart/iend)
2392 and then (fd.collapse - 1) temporaries with the same
2393 type for count2 ... countN-1 vars if not constant. */
2394 size_t count
= 2, i
;
2395 tree type
= fd
.iter_type
;
2397 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2398 count
+= fd
.collapse
- 1;
2399 for (i
= 0; i
< count
; i
++)
2401 tree temp
= create_tmp_var (type
, NULL
);
2402 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2403 OMP_CLAUSE__LOOPTEMP_
);
2404 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2405 OMP_CLAUSE_DECL (c
) = temp
;
2406 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2407 gimple_omp_parallel_set_clauses (stmt
, c
);
2412 ctx
= new_omp_context (stmt
, outer_ctx
);
2413 taskreg_contexts
.safe_push (ctx
);
2414 if (taskreg_nesting_level
> 1)
2415 ctx
->is_nested
= true;
2416 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2417 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2418 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2419 name
= create_tmp_var_name (".omp_data_s");
2420 name
= build_decl (gimple_location (stmt
),
2421 TYPE_DECL
, name
, ctx
->record_type
);
2422 DECL_ARTIFICIAL (name
) = 1;
2423 DECL_NAMELESS (name
) = 1;
2424 TYPE_NAME (ctx
->record_type
) = name
;
2425 create_omp_child_function (ctx
, false);
2426 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2428 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2429 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2431 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2432 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2435 /* Scan an OpenMP task directive. */
2438 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2442 gimple stmt
= gsi_stmt (*gsi
);
2444 /* Ignore task directives with empty bodies. */
2446 && empty_body_p (gimple_omp_body (stmt
)))
2448 gsi_replace (gsi
, gimple_build_nop (), false);
2452 ctx
= new_omp_context (stmt
, outer_ctx
);
2453 taskreg_contexts
.safe_push (ctx
);
2454 if (taskreg_nesting_level
> 1)
2455 ctx
->is_nested
= true;
2456 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2457 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2458 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2459 name
= create_tmp_var_name (".omp_data_s");
2460 name
= build_decl (gimple_location (stmt
),
2461 TYPE_DECL
, name
, ctx
->record_type
);
2462 DECL_ARTIFICIAL (name
) = 1;
2463 DECL_NAMELESS (name
) = 1;
2464 TYPE_NAME (ctx
->record_type
) = name
;
2465 create_omp_child_function (ctx
, false);
2466 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2468 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2470 if (ctx
->srecord_type
)
2472 name
= create_tmp_var_name (".omp_data_a");
2473 name
= build_decl (gimple_location (stmt
),
2474 TYPE_DECL
, name
, ctx
->srecord_type
);
2475 DECL_ARTIFICIAL (name
) = 1;
2476 DECL_NAMELESS (name
) = 1;
2477 TYPE_NAME (ctx
->srecord_type
) = name
;
2478 create_omp_child_function (ctx
, true);
2481 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2483 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2485 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2486 t
= build_int_cst (long_integer_type_node
, 0);
2487 gimple_omp_task_set_arg_size (stmt
, t
);
2488 t
= build_int_cst (long_integer_type_node
, 1);
2489 gimple_omp_task_set_arg_align (stmt
, t
);
2494 /* If any decls have been made addressable during scan_omp,
2495 adjust their fields if needed, and layout record types
2496 of parallel/task constructs. */
2499 finish_taskreg_scan (omp_context
*ctx
)
2501 if (ctx
->record_type
== NULL_TREE
)
2504 /* If any task_shared_vars were needed, verify all
2505 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2506 statements if use_pointer_for_field hasn't changed
2507 because of that. If it did, update field types now. */
2508 if (task_shared_vars
)
2512 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2513 c
; c
= OMP_CLAUSE_CHAIN (c
))
2514 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2516 tree decl
= OMP_CLAUSE_DECL (c
);
2518 /* Global variables don't need to be copied,
2519 the receiver side will use them directly. */
2520 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2522 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2523 || !use_pointer_for_field (decl
, ctx
))
2525 tree field
= lookup_field (decl
, ctx
);
2526 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2527 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2529 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2530 TREE_THIS_VOLATILE (field
) = 0;
2531 DECL_USER_ALIGN (field
) = 0;
2532 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2533 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2534 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2535 if (ctx
->srecord_type
)
2537 tree sfield
= lookup_sfield (decl
, ctx
);
2538 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2539 TREE_THIS_VOLATILE (sfield
) = 0;
2540 DECL_USER_ALIGN (sfield
) = 0;
2541 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2542 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2543 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2548 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2550 layout_type (ctx
->record_type
);
2551 fixup_child_record_type (ctx
);
2555 location_t loc
= gimple_location (ctx
->stmt
);
2556 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2557 /* Move VLA fields to the end. */
2558 p
= &TYPE_FIELDS (ctx
->record_type
);
2560 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2561 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2564 *p
= TREE_CHAIN (*p
);
2565 TREE_CHAIN (*q
) = NULL_TREE
;
2566 q
= &TREE_CHAIN (*q
);
2569 p
= &DECL_CHAIN (*p
);
2571 layout_type (ctx
->record_type
);
2572 fixup_child_record_type (ctx
);
2573 if (ctx
->srecord_type
)
2574 layout_type (ctx
->srecord_type
);
2575 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2576 TYPE_SIZE_UNIT (ctx
->record_type
));
2577 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2578 t
= build_int_cst (long_integer_type_node
,
2579 TYPE_ALIGN_UNIT (ctx
->record_type
));
2580 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2585 /* Scan a GIMPLE_OMP_FOR. */
2588 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2593 ctx
= new_omp_context (stmt
, outer_ctx
);
2595 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2597 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2598 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2600 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2601 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2602 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2603 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2605 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2608 /* Scan an OpenMP sections directive. */
2611 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2615 ctx
= new_omp_context (stmt
, outer_ctx
);
2616 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2617 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2620 /* Scan an OpenMP single directive. */
2623 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2628 ctx
= new_omp_context (stmt
, outer_ctx
);
2629 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2630 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2631 name
= create_tmp_var_name (".omp_copy_s");
2632 name
= build_decl (gimple_location (stmt
),
2633 TYPE_DECL
, name
, ctx
->record_type
);
2634 TYPE_NAME (ctx
->record_type
) = name
;
2636 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2637 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2639 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2640 ctx
->record_type
= NULL
;
2642 layout_type (ctx
->record_type
);
2645 /* Scan a GIMPLE_OMP_TARGET. */
2648 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2653 void (*gimple_omp_set_child_fn
) (gimple
, tree
);
2654 tree (*gimple_omp_clauses
) (const_gimple
);
2656 offloaded
= is_gimple_omp_offloaded (stmt
);
2657 switch (gimple_code (stmt
))
2659 case GIMPLE_OACC_KERNELS
:
2660 gimple_omp_set_child_fn
= gimple_oacc_kernels_set_child_fn
;
2661 gimple_omp_clauses
= gimple_oacc_kernels_clauses
;
2663 case GIMPLE_OACC_PARALLEL
:
2664 gimple_omp_set_child_fn
= gimple_oacc_parallel_set_child_fn
;
2665 gimple_omp_clauses
= gimple_oacc_parallel_clauses
;
2667 case GIMPLE_OMP_TARGET
:
2668 gimple_omp_set_child_fn
= gimple_omp_target_set_child_fn
;
2669 gimple_omp_clauses
= gimple_omp_target_clauses
;
2675 if (is_gimple_omp_oacc_specifically (stmt
))
2677 gcc_assert (taskreg_nesting_level
== 0);
2678 gcc_assert (target_nesting_level
== 0);
2681 ctx
= new_omp_context (stmt
, outer_ctx
);
2682 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2683 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2684 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2685 name
= create_tmp_var_name (".omp_data_t");
2686 name
= build_decl (gimple_location (stmt
),
2687 TYPE_DECL
, name
, ctx
->record_type
);
2688 DECL_ARTIFICIAL (name
) = 1;
2689 DECL_NAMELESS (name
) = 1;
2690 TYPE_NAME (ctx
->record_type
) = name
;
2693 if (is_gimple_omp_oacc_specifically (stmt
))
2694 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
2697 create_omp_child_function (ctx
, false);
2698 gimple_omp_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2701 scan_sharing_clauses (gimple_omp_clauses (stmt
), ctx
);
2702 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2704 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2705 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2708 TYPE_FIELDS (ctx
->record_type
)
2709 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2710 #ifdef ENABLE_CHECKING
2712 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2713 for (field
= TYPE_FIELDS (ctx
->record_type
);
2715 field
= DECL_CHAIN (field
))
2716 gcc_assert (DECL_ALIGN (field
) == align
);
2718 layout_type (ctx
->record_type
);
2720 fixup_child_record_type (ctx
);
2724 /* Scan an OpenMP teams directive. */
2727 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2729 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2730 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2731 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2734 /* Check OpenMP nesting restrictions. */
2736 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2738 /* TODO: While the OpenACC specification does allow for certain kinds of
2739 nesting, we don't support many of these yet. */
2740 if (is_gimple_omp (stmt
)
2741 && is_gimple_omp_oacc_specifically (stmt
))
2743 /* Regular handling of OpenACC loop constructs. */
2744 if (gimple_code (stmt
) == GIMPLE_OMP_FOR
2745 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
2747 /* No nesting of OpenACC STMT inside any OpenACC or OpenMP CTX different
2748 from an OpenACC data construct. */
2749 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
2750 if (is_gimple_omp (ctx_
->stmt
)
2751 && !(gimple_code (ctx_
->stmt
) == GIMPLE_OMP_TARGET
2752 && (gimple_omp_target_kind (ctx_
->stmt
)
2753 == GF_OMP_TARGET_KIND_OACC_DATA
)))
2755 error_at (gimple_location (stmt
),
2756 "may not be nested");
2762 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP
2763 builtin) inside any OpenACC CTX. */
2764 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
2765 if (is_gimple_omp (ctx_
->stmt
)
2766 && is_gimple_omp_oacc_specifically (ctx_
->stmt
))
2768 error_at (gimple_location (stmt
),
2769 "may not be nested");
2777 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2778 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2780 error_at (gimple_location (stmt
),
2781 "OpenMP constructs may not be nested inside simd region");
2784 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2786 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2787 || (gimple_omp_for_kind (stmt
)
2788 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2789 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2791 error_at (gimple_location (stmt
),
2792 "only distribute or parallel constructs are allowed to "
2793 "be closely nested inside teams construct");
2798 switch (gimple_code (stmt
))
2800 case GIMPLE_OMP_FOR
:
2801 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2803 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2805 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2807 error_at (gimple_location (stmt
),
2808 "distribute construct must be closely nested inside "
2816 if (is_gimple_call (stmt
)
2817 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2818 == BUILT_IN_GOMP_CANCEL
2819 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2820 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2822 const char *bad
= NULL
;
2823 const char *kind
= NULL
;
2826 error_at (gimple_location (stmt
), "orphaned %qs construct",
2827 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2828 == BUILT_IN_GOMP_CANCEL
2829 ? "#pragma omp cancel"
2830 : "#pragma omp cancellation point");
2833 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2834 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2838 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2839 bad
= "#pragma omp parallel";
2840 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2841 == BUILT_IN_GOMP_CANCEL
2842 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2843 ctx
->cancellable
= true;
2847 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2848 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2849 bad
= "#pragma omp for";
2850 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2851 == BUILT_IN_GOMP_CANCEL
2852 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2854 ctx
->cancellable
= true;
2855 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2857 warning_at (gimple_location (stmt
), 0,
2858 "%<#pragma omp cancel for%> inside "
2859 "%<nowait%> for construct");
2860 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2861 OMP_CLAUSE_ORDERED
))
2862 warning_at (gimple_location (stmt
), 0,
2863 "%<#pragma omp cancel for%> inside "
2864 "%<ordered%> for construct");
2869 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2870 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2871 bad
= "#pragma omp sections";
2872 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2873 == BUILT_IN_GOMP_CANCEL
2874 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2876 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2878 ctx
->cancellable
= true;
2879 if (find_omp_clause (gimple_omp_sections_clauses
2882 warning_at (gimple_location (stmt
), 0,
2883 "%<#pragma omp cancel sections%> inside "
2884 "%<nowait%> sections construct");
2888 gcc_assert (ctx
->outer
2889 && gimple_code (ctx
->outer
->stmt
)
2890 == GIMPLE_OMP_SECTIONS
);
2891 ctx
->outer
->cancellable
= true;
2892 if (find_omp_clause (gimple_omp_sections_clauses
2895 warning_at (gimple_location (stmt
), 0,
2896 "%<#pragma omp cancel sections%> inside "
2897 "%<nowait%> sections construct");
2903 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2904 bad
= "#pragma omp task";
2906 ctx
->cancellable
= true;
2910 error_at (gimple_location (stmt
), "invalid arguments");
2915 error_at (gimple_location (stmt
),
2916 "%<%s %s%> construct not closely nested inside of %qs",
2917 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2918 == BUILT_IN_GOMP_CANCEL
2919 ? "#pragma omp cancel"
2920 : "#pragma omp cancellation point", kind
, bad
);
2925 case GIMPLE_OMP_SECTIONS
:
2926 case GIMPLE_OMP_SINGLE
:
2927 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2928 switch (gimple_code (ctx
->stmt
))
2930 case GIMPLE_OMP_FOR
:
2931 case GIMPLE_OMP_SECTIONS
:
2932 case GIMPLE_OMP_SINGLE
:
2933 case GIMPLE_OMP_ORDERED
:
2934 case GIMPLE_OMP_MASTER
:
2935 case GIMPLE_OMP_TASK
:
2936 case GIMPLE_OMP_CRITICAL
:
2937 if (is_gimple_call (stmt
))
2939 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2940 != BUILT_IN_GOMP_BARRIER
)
2942 error_at (gimple_location (stmt
),
2943 "barrier region may not be closely nested inside "
2944 "of work-sharing, critical, ordered, master or "
2945 "explicit task region");
2948 error_at (gimple_location (stmt
),
2949 "work-sharing region may not be closely nested inside "
2950 "of work-sharing, critical, ordered, master or explicit "
2953 case GIMPLE_OACC_KERNELS
:
2954 case GIMPLE_OACC_PARALLEL
:
2955 case GIMPLE_OMP_PARALLEL
:
2961 case GIMPLE_OMP_MASTER
:
2962 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2963 switch (gimple_code (ctx
->stmt
))
2965 case GIMPLE_OMP_FOR
:
2966 case GIMPLE_OMP_SECTIONS
:
2967 case GIMPLE_OMP_SINGLE
:
2968 case GIMPLE_OMP_TASK
:
2969 error_at (gimple_location (stmt
),
2970 "master region may not be closely nested inside "
2971 "of work-sharing or explicit task region");
2973 case GIMPLE_OMP_PARALLEL
:
2979 case GIMPLE_OMP_ORDERED
:
2980 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2981 switch (gimple_code (ctx
->stmt
))
2983 case GIMPLE_OMP_CRITICAL
:
2984 case GIMPLE_OMP_TASK
:
2985 error_at (gimple_location (stmt
),
2986 "ordered region may not be closely nested inside "
2987 "of critical or explicit task region");
2989 case GIMPLE_OMP_FOR
:
2990 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2991 OMP_CLAUSE_ORDERED
) == NULL
)
2993 error_at (gimple_location (stmt
),
2994 "ordered region must be closely nested inside "
2995 "a loop region with an ordered clause");
2999 case GIMPLE_OMP_PARALLEL
:
3000 error_at (gimple_location (stmt
),
3001 "ordered region must be closely nested inside "
3002 "a loop region with an ordered clause");
3008 case GIMPLE_OMP_CRITICAL
:
3009 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3010 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
3011 && (gimple_omp_critical_name (stmt
)
3012 == gimple_omp_critical_name (ctx
->stmt
)))
3014 error_at (gimple_location (stmt
),
3015 "critical region may not be nested inside a critical "
3016 "region with the same name");
3020 case GIMPLE_OMP_TEAMS
:
3022 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3023 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3025 error_at (gimple_location (stmt
),
3026 "teams construct not closely nested inside of target "
3031 case GIMPLE_OMP_TARGET
:
3032 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3033 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
3034 && gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_REGION
)
3037 switch (gimple_omp_target_kind (stmt
))
3039 case GF_OMP_TARGET_KIND_REGION
: name
= "target"; break;
3040 case GF_OMP_TARGET_KIND_DATA
: name
= "target data"; break;
3041 case GF_OMP_TARGET_KIND_UPDATE
: name
= "target update"; break;
3042 default: gcc_unreachable ();
3044 warning_at (gimple_location (stmt
), 0,
3045 "%s construct inside of target region", name
);
3055 /* Helper function scan_omp.
3057 Callback for walk_tree or operators in walk_gimple_stmt used to
3058 scan for OpenMP directives in TP. */
3061 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3063 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3064 omp_context
*ctx
= (omp_context
*) wi
->info
;
3067 switch (TREE_CODE (t
))
3074 *tp
= remap_decl (t
, &ctx
->cb
);
3078 if (ctx
&& TYPE_P (t
))
3079 *tp
= remap_type (t
, &ctx
->cb
);
3080 else if (!DECL_P (t
))
3085 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3086 if (tem
!= TREE_TYPE (t
))
3088 if (TREE_CODE (t
) == INTEGER_CST
)
3089 *tp
= wide_int_to_tree (tem
, t
);
3091 TREE_TYPE (t
) = tem
;
3101 /* Return true if FNDECL is a setjmp or a longjmp. */
3104 setjmp_or_longjmp_p (const_tree fndecl
)
3106 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3107 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3108 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3111 tree declname
= DECL_NAME (fndecl
);
3114 const char *name
= IDENTIFIER_POINTER (declname
);
3115 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3119 /* Helper function for scan_omp.
3121 Callback for walk_gimple_stmt used to scan for OpenMP directives in
3122 the current statement in GSI. */
3125 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3126 struct walk_stmt_info
*wi
)
3128 gimple stmt
= gsi_stmt (*gsi
);
3129 omp_context
*ctx
= (omp_context
*) wi
->info
;
3131 if (gimple_has_location (stmt
))
3132 input_location
= gimple_location (stmt
);
3134 /* Check the OpenMP nesting restrictions. */
3135 bool remove
= false;
3136 if (is_gimple_omp (stmt
))
3137 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3138 else if (is_gimple_call (stmt
))
3140 tree fndecl
= gimple_call_fndecl (stmt
);
3143 if (setjmp_or_longjmp_p (fndecl
)
3145 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3146 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3149 error_at (gimple_location (stmt
),
3150 "setjmp/longjmp inside simd construct");
3152 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3153 switch (DECL_FUNCTION_CODE (fndecl
))
3155 case BUILT_IN_GOMP_BARRIER
:
3156 case BUILT_IN_GOMP_CANCEL
:
3157 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3158 case BUILT_IN_GOMP_TASKYIELD
:
3159 case BUILT_IN_GOMP_TASKWAIT
:
3160 case BUILT_IN_GOMP_TASKGROUP_START
:
3161 case BUILT_IN_GOMP_TASKGROUP_END
:
3162 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3171 stmt
= gimple_build_nop ();
3172 gsi_replace (gsi
, stmt
, false);
3175 *handled_ops_p
= true;
3177 switch (gimple_code (stmt
))
3179 case GIMPLE_OMP_PARALLEL
:
3180 taskreg_nesting_level
++;
3181 scan_omp_parallel (gsi
, ctx
);
3182 taskreg_nesting_level
--;
3185 case GIMPLE_OMP_TASK
:
3186 taskreg_nesting_level
++;
3187 scan_omp_task (gsi
, ctx
);
3188 taskreg_nesting_level
--;
3191 case GIMPLE_OMP_FOR
:
3192 scan_omp_for (stmt
, ctx
);
3195 case GIMPLE_OMP_SECTIONS
:
3196 scan_omp_sections (stmt
, ctx
);
3199 case GIMPLE_OMP_SINGLE
:
3200 scan_omp_single (stmt
, ctx
);
3203 case GIMPLE_OMP_SECTION
:
3204 case GIMPLE_OMP_MASTER
:
3205 case GIMPLE_OMP_TASKGROUP
:
3206 case GIMPLE_OMP_ORDERED
:
3207 case GIMPLE_OMP_CRITICAL
:
3208 ctx
= new_omp_context (stmt
, ctx
);
3209 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3212 case GIMPLE_OACC_KERNELS
:
3213 case GIMPLE_OACC_PARALLEL
:
3214 case GIMPLE_OMP_TARGET
:
3215 scan_omp_target (stmt
, ctx
);
3218 case GIMPLE_OMP_TEAMS
:
3219 scan_omp_teams (stmt
, ctx
);
3226 *handled_ops_p
= false;
3228 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
3229 insert_decl_map (&ctx
->cb
, var
, var
);
3233 *handled_ops_p
= false;
3241 /* Scan all the statements starting at the current statement. CTX
3242 contains context information about the OpenMP directives and
3243 clauses found during the scan. */
3246 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3248 location_t saved_location
;
3249 struct walk_stmt_info wi
;
3251 memset (&wi
, 0, sizeof (wi
));
3253 wi
.want_locations
= true;
3255 saved_location
= input_location
;
3256 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3257 input_location
= saved_location
;
3260 /* Re-gimplification and code generation routines. */
3262 /* Build a call to GOMP_barrier. */
3265 build_omp_barrier (tree lhs
)
3267 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3268 : BUILT_IN_GOMP_BARRIER
);
3269 gimple g
= gimple_build_call (fndecl
, 0);
3271 gimple_call_set_lhs (g
, lhs
);
3275 /* If a context was created for STMT when it was scanned, return it. */
3277 static omp_context
*
3278 maybe_lookup_ctx (gimple stmt
)
3281 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3282 return n
? (omp_context
*) n
->value
: NULL
;
3286 /* Find the mapping for DECL in CTX or the immediately enclosing
3287 context that has a mapping for DECL.
3289 If CTX is a nested parallel directive, we may have to use the decl
3290 mappings created in CTX's parent context. Suppose that we have the
3291 following parallel nesting (variable UIDs showed for clarity):
3294 #omp parallel shared(iD.1562) -> outer parallel
3295 iD.1562 = iD.1562 + 1;
3297 #omp parallel shared (iD.1562) -> inner parallel
3298 iD.1562 = iD.1562 - 1;
3300 Each parallel structure will create a distinct .omp_data_s structure
3301 for copying iD.1562 in/out of the directive:
3303 outer parallel .omp_data_s.1.i -> iD.1562
3304 inner parallel .omp_data_s.2.i -> iD.1562
3306 A shared variable mapping will produce a copy-out operation before
3307 the parallel directive and a copy-in operation after it. So, in
3308 this case we would have:
3311 .omp_data_o.1.i = iD.1562;
3312 #omp parallel shared(iD.1562) -> outer parallel
3313 .omp_data_i.1 = &.omp_data_o.1
3314 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3316 .omp_data_o.2.i = iD.1562; -> **
3317 #omp parallel shared(iD.1562) -> inner parallel
3318 .omp_data_i.2 = &.omp_data_o.2
3319 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3322 ** This is a problem. The symbol iD.1562 cannot be referenced
3323 inside the body of the outer parallel region. But since we are
3324 emitting this copy operation while expanding the inner parallel
3325 directive, we need to access the CTX structure of the outer
3326 parallel directive to get the correct mapping:
3328 .omp_data_o.2.i = .omp_data_i.1->i
3330 Since there may be other workshare or parallel directives enclosing
3331 the parallel directive, it may be necessary to walk up the context
3332 parent chain. This is not a problem in general because nested
3333 parallelism happens only rarely. */
3336 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3341 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3342 t
= maybe_lookup_decl (decl
, up
);
3344 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3346 return t
? t
: decl
;
3350 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3351 in outer contexts. */
3354 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3359 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3360 t
= maybe_lookup_decl (decl
, up
);
3362 return t
? t
: decl
;
3366 /* Construct the initialization value for reduction CLAUSE. */
3369 omp_reduction_init (tree clause
, tree type
)
3371 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3372 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3379 case TRUTH_ORIF_EXPR
:
3380 case TRUTH_XOR_EXPR
:
3382 return build_zero_cst (type
);
3385 case TRUTH_AND_EXPR
:
3386 case TRUTH_ANDIF_EXPR
:
3388 return fold_convert_loc (loc
, type
, integer_one_node
);
3391 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3394 if (SCALAR_FLOAT_TYPE_P (type
))
3396 REAL_VALUE_TYPE max
, min
;
3397 if (HONOR_INFINITIES (TYPE_MODE (type
)))
3400 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3403 real_maxval (&min
, 1, TYPE_MODE (type
));
3404 return build_real (type
, min
);
3408 gcc_assert (INTEGRAL_TYPE_P (type
));
3409 return TYPE_MIN_VALUE (type
);
3413 if (SCALAR_FLOAT_TYPE_P (type
))
3415 REAL_VALUE_TYPE max
;
3416 if (HONOR_INFINITIES (TYPE_MODE (type
)))
3419 real_maxval (&max
, 0, TYPE_MODE (type
));
3420 return build_real (type
, max
);
3424 gcc_assert (INTEGRAL_TYPE_P (type
));
3425 return TYPE_MAX_VALUE (type
);
3433 /* Return alignment to be assumed for var in CLAUSE, which should be
3434 OMP_CLAUSE_ALIGNED. */
3437 omp_clause_aligned_alignment (tree clause
)
3439 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3440 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3442 /* Otherwise return implementation defined alignment. */
3443 unsigned int al
= 1;
3444 machine_mode mode
, vmode
;
3445 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3447 vs
= 1 << floor_log2 (vs
);
3448 static enum mode_class classes
[]
3449 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3450 for (int i
= 0; i
< 4; i
+= 2)
3451 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3453 mode
= GET_MODE_WIDER_MODE (mode
))
3455 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3456 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3459 && GET_MODE_SIZE (vmode
) < vs
3460 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3461 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3463 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3464 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3466 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3467 / GET_MODE_SIZE (mode
));
3468 if (TYPE_MODE (type
) != vmode
)
3470 if (TYPE_ALIGN_UNIT (type
) > al
)
3471 al
= TYPE_ALIGN_UNIT (type
);
3473 return build_int_cst (integer_type_node
, al
);
3476 /* Return maximum possible vectorization factor for the target. */
3483 || !flag_tree_loop_optimize
3484 || (!flag_tree_loop_vectorize
3485 && (global_options_set
.x_flag_tree_loop_vectorize
3486 || global_options_set
.x_flag_tree_vectorize
)))
3489 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3492 vs
= 1 << floor_log2 (vs
);
3495 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3496 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3497 return GET_MODE_NUNITS (vqimode
);
3501 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3505 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3506 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3508 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
3512 max_vf
= omp_max_vf ();
3515 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3516 OMP_CLAUSE_SAFELEN
);
3517 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3519 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3521 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3525 idx
= create_tmp_var (unsigned_type_node
, NULL
);
3526 lane
= create_tmp_var (unsigned_type_node
, NULL
);
3532 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3533 tree avar
= create_tmp_var_raw (atype
, NULL
);
3534 if (TREE_ADDRESSABLE (new_var
))
3535 TREE_ADDRESSABLE (avar
) = 1;
3536 DECL_ATTRIBUTES (avar
)
3537 = tree_cons (get_identifier ("omp simd array"), NULL
,
3538 DECL_ATTRIBUTES (avar
));
3539 gimple_add_tmp_var (avar
);
3540 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3541 NULL_TREE
, NULL_TREE
);
3542 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3543 NULL_TREE
, NULL_TREE
);
3544 if (DECL_P (new_var
))
3546 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3547 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3552 /* Helper function of lower_rec_input_clauses. For a reference
3553 in simd reduction, add an underlying variable it will reference. */
3556 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3558 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3559 if (TREE_CONSTANT (z
))
3561 const char *name
= NULL
;
3562 if (DECL_NAME (new_vard
))
3563 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3565 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3566 gimple_add_tmp_var (z
);
3567 TREE_ADDRESSABLE (z
) = 1;
3568 z
= build_fold_addr_expr_loc (loc
, z
);
3569 gimplify_assign (new_vard
, z
, ilist
);
3573 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3574 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3575 private variables. Initialization statements go in ILIST, while calls
3576 to destructors go in DLIST. */
3579 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3580 omp_context
*ctx
, struct omp_for_data
*fd
)
3582 tree c
, dtor
, copyin_seq
, x
, ptr
;
3583 bool copyin_by_ref
= false;
3584 bool lastprivate_firstprivate
= false;
3585 bool reduction_omp_orig_ref
= false;
3587 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3588 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3590 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3591 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3592 gimple_seq llist
[2] = { NULL
, NULL
};
3596 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3597 with data sharing clauses referencing variable sized vars. That
3598 is unnecessarily hard to support and very unlikely to result in
3599 vectorized code anyway. */
3601 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3602 switch (OMP_CLAUSE_CODE (c
))
3604 case OMP_CLAUSE_LINEAR
:
3605 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3608 case OMP_CLAUSE_REDUCTION
:
3609 case OMP_CLAUSE_PRIVATE
:
3610 case OMP_CLAUSE_FIRSTPRIVATE
:
3611 case OMP_CLAUSE_LASTPRIVATE
:
3612 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3619 /* Do all the fixed sized types in the first pass, and the variable sized
3620 types in the second pass. This makes sure that the scalar arguments to
3621 the variable sized types are processed before we use them in the
3622 variable sized operations. */
3623 for (pass
= 0; pass
< 2; ++pass
)
3625 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3627 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3630 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3634 case OMP_CLAUSE_PRIVATE
:
3635 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3638 case OMP_CLAUSE_SHARED
:
3639 /* Ignore shared directives in teams construct. */
3640 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3642 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3644 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3647 case OMP_CLAUSE_FIRSTPRIVATE
:
3648 case OMP_CLAUSE_COPYIN
:
3649 case OMP_CLAUSE_LINEAR
:
3651 case OMP_CLAUSE_REDUCTION
:
3652 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3653 reduction_omp_orig_ref
= true;
3655 case OMP_CLAUSE__LOOPTEMP_
:
3656 /* Handle _looptemp_ clauses only on parallel. */
3660 case OMP_CLAUSE_LASTPRIVATE
:
3661 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3663 lastprivate_firstprivate
= true;
3667 /* Even without corresponding firstprivate, if
3668 decl is Fortran allocatable, it needs outer var
3671 && lang_hooks
.decls
.omp_private_outer_ref
3672 (OMP_CLAUSE_DECL (c
)))
3673 lastprivate_firstprivate
= true;
3675 case OMP_CLAUSE_ALIGNED
:
3678 var
= OMP_CLAUSE_DECL (c
);
3679 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3680 && !is_global_var (var
))
3682 new_var
= maybe_lookup_decl (var
, ctx
);
3683 if (new_var
== NULL_TREE
)
3684 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3685 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3686 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3687 omp_clause_aligned_alignment (c
));
3688 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3689 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3690 gimplify_and_add (x
, ilist
);
3692 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3693 && is_global_var (var
))
3695 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3696 new_var
= lookup_decl (var
, ctx
);
3697 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3698 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3699 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3700 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3701 omp_clause_aligned_alignment (c
));
3702 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3703 x
= create_tmp_var (ptype
, NULL
);
3704 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3705 gimplify_and_add (t
, ilist
);
3706 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3707 SET_DECL_VALUE_EXPR (new_var
, t
);
3708 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3715 new_var
= var
= OMP_CLAUSE_DECL (c
);
3716 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3717 new_var
= lookup_decl (var
, ctx
);
3719 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3724 else if (is_variable_sized (var
))
3726 /* For variable sized types, we need to allocate the
3727 actual storage here. Call alloca and store the
3728 result in the pointer decl that we created elsewhere. */
3732 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3737 ptr
= DECL_VALUE_EXPR (new_var
);
3738 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3739 ptr
= TREE_OPERAND (ptr
, 0);
3740 gcc_assert (DECL_P (ptr
));
3741 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3743 /* void *tmp = __builtin_alloca */
3744 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3745 stmt
= gimple_build_call (atmp
, 1, x
);
3746 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3747 gimple_add_tmp_var (tmp
);
3748 gimple_call_set_lhs (stmt
, tmp
);
3750 gimple_seq_add_stmt (ilist
, stmt
);
3752 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3753 gimplify_assign (ptr
, x
, ilist
);
3756 else if (is_reference (var
))
3758 /* For references that are being privatized for Fortran,
3759 allocate new backing storage for the new pointer
3760 variable. This allows us to avoid changing all the
3761 code that expects a pointer to something that expects
3762 a direct variable. */
3766 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3767 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3769 x
= build_receiver_ref (var
, false, ctx
);
3770 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3772 else if (TREE_CONSTANT (x
))
3774 /* For reduction in SIMD loop, defer adding the
3775 initialization of the reference, because if we decide
3776 to use SIMD array for it, the initilization could cause
3778 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3782 const char *name
= NULL
;
3783 if (DECL_NAME (var
))
3784 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3786 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3788 gimple_add_tmp_var (x
);
3789 TREE_ADDRESSABLE (x
) = 1;
3790 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3795 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3796 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3801 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3802 gimplify_assign (new_var
, x
, ilist
);
3805 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3807 else if (c_kind
== OMP_CLAUSE_REDUCTION
3808 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3816 switch (OMP_CLAUSE_CODE (c
))
3818 case OMP_CLAUSE_SHARED
:
3819 /* Ignore shared directives in teams construct. */
3820 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3822 /* Shared global vars are just accessed directly. */
3823 if (is_global_var (new_var
))
3825 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3826 needs to be delayed until after fixup_child_record_type so
3827 that we get the correct type during the dereference. */
3828 by_ref
= use_pointer_for_field (var
, ctx
);
3829 x
= build_receiver_ref (var
, by_ref
, ctx
);
3830 SET_DECL_VALUE_EXPR (new_var
, x
);
3831 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3833 /* ??? If VAR is not passed by reference, and the variable
3834 hasn't been initialized yet, then we'll get a warning for
3835 the store into the omp_data_s structure. Ideally, we'd be
3836 able to notice this and not store anything at all, but
3837 we're generating code too early. Suppress the warning. */
3839 TREE_NO_WARNING (var
) = 1;
3842 case OMP_CLAUSE_LASTPRIVATE
:
3843 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3847 case OMP_CLAUSE_PRIVATE
:
3848 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3849 x
= build_outer_var_ref (var
, ctx
);
3850 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3852 if (is_task_ctx (ctx
))
3853 x
= build_receiver_ref (var
, false, ctx
);
3855 x
= build_outer_var_ref (var
, ctx
);
3861 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3864 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3865 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3866 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3867 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3868 idx
, lane
, ivar
, lvar
))
3871 x
= lang_hooks
.decls
.omp_clause_default_ctor
3872 (c
, unshare_expr (ivar
), x
);
3874 gimplify_and_add (x
, &llist
[0]);
3877 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3880 gimple_seq tseq
= NULL
;
3883 gimplify_stmt (&dtor
, &tseq
);
3884 gimple_seq_add_seq (&llist
[1], tseq
);
3891 gimplify_and_add (nx
, ilist
);
3895 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3898 gimple_seq tseq
= NULL
;
3901 gimplify_stmt (&dtor
, &tseq
);
3902 gimple_seq_add_seq (dlist
, tseq
);
3906 case OMP_CLAUSE_LINEAR
:
3907 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3908 goto do_firstprivate
;
3909 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3912 x
= build_outer_var_ref (var
, ctx
);
3915 case OMP_CLAUSE_FIRSTPRIVATE
:
3916 if (is_task_ctx (ctx
))
3918 if (is_reference (var
) || is_variable_sized (var
))
3920 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3922 || use_pointer_for_field (var
, NULL
))
3924 x
= build_receiver_ref (var
, false, ctx
);
3925 SET_DECL_VALUE_EXPR (new_var
, x
);
3926 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3931 x
= build_outer_var_ref (var
, ctx
);
3934 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3935 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3937 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3938 tree stept
= TREE_TYPE (t
);
3939 tree ct
= find_omp_clause (clauses
,
3940 OMP_CLAUSE__LOOPTEMP_
);
3942 tree l
= OMP_CLAUSE_DECL (ct
);
3943 tree n1
= fd
->loop
.n1
;
3944 tree step
= fd
->loop
.step
;
3945 tree itype
= TREE_TYPE (l
);
3946 if (POINTER_TYPE_P (itype
))
3947 itype
= signed_type_for (itype
);
3948 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3949 if (TYPE_UNSIGNED (itype
)
3950 && fd
->loop
.cond_code
== GT_EXPR
)
3951 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3952 fold_build1 (NEGATE_EXPR
, itype
, l
),
3953 fold_build1 (NEGATE_EXPR
,
3956 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3957 t
= fold_build2 (MULT_EXPR
, stept
,
3958 fold_convert (stept
, l
), t
);
3960 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3962 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3964 gimplify_and_add (x
, ilist
);
3968 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3969 x
= fold_build2 (POINTER_PLUS_EXPR
,
3970 TREE_TYPE (x
), x
, t
);
3972 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3975 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3976 || TREE_ADDRESSABLE (new_var
))
3977 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3978 idx
, lane
, ivar
, lvar
))
3980 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3982 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3983 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3984 gimplify_and_add (x
, ilist
);
3985 gimple_stmt_iterator gsi
3986 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3988 = gimple_build_assign (unshare_expr (lvar
), iv
);
3989 gsi_insert_before_without_update (&gsi
, g
,
3991 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3992 enum tree_code code
= PLUS_EXPR
;
3993 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3994 code
= POINTER_PLUS_EXPR
;
3995 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3996 gsi_insert_before_without_update (&gsi
, g
,
4000 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4001 (c
, unshare_expr (ivar
), x
);
4002 gimplify_and_add (x
, &llist
[0]);
4003 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4006 gimple_seq tseq
= NULL
;
4009 gimplify_stmt (&dtor
, &tseq
);
4010 gimple_seq_add_seq (&llist
[1], tseq
);
4015 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
4016 gimplify_and_add (x
, ilist
);
4019 case OMP_CLAUSE__LOOPTEMP_
:
4020 gcc_assert (is_parallel_ctx (ctx
));
4021 x
= build_outer_var_ref (var
, ctx
);
4022 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4023 gimplify_and_add (x
, ilist
);
4026 case OMP_CLAUSE_COPYIN
:
4027 by_ref
= use_pointer_for_field (var
, NULL
);
4028 x
= build_receiver_ref (var
, by_ref
, ctx
);
4029 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4030 append_to_statement_list (x
, ©in_seq
);
4031 copyin_by_ref
|= by_ref
;
4034 case OMP_CLAUSE_REDUCTION
:
4035 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4037 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4039 x
= build_outer_var_ref (var
, ctx
);
4041 if (is_reference (var
)
4042 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4044 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4045 SET_DECL_VALUE_EXPR (placeholder
, x
);
4046 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4047 tree new_vard
= new_var
;
4048 if (is_reference (var
))
4050 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4051 new_vard
= TREE_OPERAND (new_var
, 0);
4052 gcc_assert (DECL_P (new_vard
));
4055 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4056 idx
, lane
, ivar
, lvar
))
4058 if (new_vard
== new_var
)
4060 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4061 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4065 SET_DECL_VALUE_EXPR (new_vard
,
4066 build_fold_addr_expr (ivar
));
4067 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4069 x
= lang_hooks
.decls
.omp_clause_default_ctor
4070 (c
, unshare_expr (ivar
),
4071 build_outer_var_ref (var
, ctx
));
4073 gimplify_and_add (x
, &llist
[0]);
4074 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4076 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4077 lower_omp (&tseq
, ctx
);
4078 gimple_seq_add_seq (&llist
[0], tseq
);
4080 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4081 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4082 lower_omp (&tseq
, ctx
);
4083 gimple_seq_add_seq (&llist
[1], tseq
);
4084 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4085 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4086 if (new_vard
== new_var
)
4087 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4089 SET_DECL_VALUE_EXPR (new_vard
,
4090 build_fold_addr_expr (lvar
));
4091 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4096 gimplify_stmt (&dtor
, &tseq
);
4097 gimple_seq_add_seq (&llist
[1], tseq
);
4101 /* If this is a reference to constant size reduction var
4102 with placeholder, we haven't emitted the initializer
4103 for it because it is undesirable if SIMD arrays are used.
4104 But if they aren't used, we need to emit the deferred
4105 initialization now. */
4106 else if (is_reference (var
) && is_simd
)
4107 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4108 x
= lang_hooks
.decls
.omp_clause_default_ctor
4109 (c
, unshare_expr (new_var
),
4110 build_outer_var_ref (var
, ctx
));
4112 gimplify_and_add (x
, ilist
);
4113 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4115 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4116 lower_omp (&tseq
, ctx
);
4117 gimple_seq_add_seq (ilist
, tseq
);
4119 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4122 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4123 lower_omp (&tseq
, ctx
);
4124 gimple_seq_add_seq (dlist
, tseq
);
4125 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4127 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4132 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4133 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4134 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4136 /* reduction(-:var) sums up the partial results, so it
4137 acts identically to reduction(+:var). */
4138 if (code
== MINUS_EXPR
)
4141 tree new_vard
= new_var
;
4142 if (is_simd
&& is_reference (var
))
4144 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4145 new_vard
= TREE_OPERAND (new_var
, 0);
4146 gcc_assert (DECL_P (new_vard
));
4149 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4150 idx
, lane
, ivar
, lvar
))
4152 tree ref
= build_outer_var_ref (var
, ctx
);
4154 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
4156 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
4157 ref
= build_outer_var_ref (var
, ctx
);
4158 gimplify_assign (ref
, x
, &llist
[1]);
4160 if (new_vard
!= new_var
)
4162 SET_DECL_VALUE_EXPR (new_vard
,
4163 build_fold_addr_expr (lvar
));
4164 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4169 if (is_reference (var
) && is_simd
)
4170 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4171 gimplify_assign (new_var
, x
, ilist
);
4174 tree ref
= build_outer_var_ref (var
, ctx
);
4176 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4177 ref
= build_outer_var_ref (var
, ctx
);
4178 gimplify_assign (ref
, x
, dlist
);
4192 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
4193 /* Don't want uninit warnings on simduid, it is always uninitialized,
4194 but we use it not for the value, but for the DECL_UID only. */
4195 TREE_NO_WARNING (uid
) = 1;
4197 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
4198 gimple_call_set_lhs (g
, lane
);
4199 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4200 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
4201 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
4202 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
4203 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4204 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4205 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
4206 build_int_cst (unsigned_type_node
, 0),
4208 gimple_seq_add_stmt (ilist
, g
);
4209 for (int i
= 0; i
< 2; i
++)
4212 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
4213 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
4214 gimple_call_set_lhs (g
, vf
);
4215 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
4216 gimple_seq_add_stmt (seq
, g
);
4217 tree t
= build_int_cst (unsigned_type_node
, 0);
4218 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
4219 gimple_seq_add_stmt (seq
, g
);
4220 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4221 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
4222 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4223 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
4224 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
4225 gimple_seq_add_seq (seq
, llist
[i
]);
4226 t
= build_int_cst (unsigned_type_node
, 1);
4227 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
4228 gimple_seq_add_stmt (seq
, g
);
4229 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
4230 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
4231 gimple_seq_add_stmt (seq
, g
);
4232 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
4236 /* The copyin sequence is not to be executed by the main thread, since
4237 that would result in self-copies. Perhaps not visible to scalars,
4238 but it certainly is to C++ operator=. */
4241 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
4243 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
4244 build_int_cst (TREE_TYPE (x
), 0));
4245 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
4246 gimplify_and_add (x
, ilist
);
4249 /* If any copyin variable is passed by reference, we must ensure the
4250 master thread doesn't modify it before it is copied over in all
4251 threads. Similarly for variables in both firstprivate and
4252 lastprivate clauses we need to ensure the lastprivate copying
4253 happens after firstprivate copying in all threads. And similarly
4254 for UDRs if initializer expression refers to omp_orig. */
4255 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
4257 /* Don't add any barrier for #pragma omp simd or
4258 #pragma omp distribute. */
4259 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
4260 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
4261 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
4264 /* If max_vf is non-zero, then we can use only a vectorization factor
4265 up to the max_vf we chose. So stick it into the safelen clause. */
4268 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4269 OMP_CLAUSE_SAFELEN
);
4271 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
4272 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4275 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
4276 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
4278 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4279 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4285 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4286 both parallel and workshare constructs. PREDICATE may be NULL if it's
4290 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
4293 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
4294 bool par_clauses
= false;
4295 tree simduid
= NULL
, lastlane
= NULL
;
4297 /* Early exit if there are no lastprivate or linear clauses. */
4298 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
4299 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
4300 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
4301 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
4303 if (clauses
== NULL
)
4305 /* If this was a workshare clause, see if it had been combined
4306 with its parallel. In that case, look for the clauses on the
4307 parallel statement itself. */
4308 if (is_parallel_ctx (ctx
))
4312 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4315 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4316 OMP_CLAUSE_LASTPRIVATE
);
4317 if (clauses
== NULL
)
4325 tree label_true
, arm1
, arm2
;
4327 label
= create_artificial_label (UNKNOWN_LOCATION
);
4328 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
4329 arm1
= TREE_OPERAND (predicate
, 0);
4330 arm2
= TREE_OPERAND (predicate
, 1);
4331 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4332 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4333 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
4335 gimple_seq_add_stmt (stmt_list
, stmt
);
4336 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
4339 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4340 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4342 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
4344 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
4347 for (c
= clauses
; c
;)
4350 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4352 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4353 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4354 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
4356 var
= OMP_CLAUSE_DECL (c
);
4357 new_var
= lookup_decl (var
, ctx
);
4359 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
4361 tree val
= DECL_VALUE_EXPR (new_var
);
4362 if (TREE_CODE (val
) == ARRAY_REF
4363 && VAR_P (TREE_OPERAND (val
, 0))
4364 && lookup_attribute ("omp simd array",
4365 DECL_ATTRIBUTES (TREE_OPERAND (val
,
4368 if (lastlane
== NULL
)
4370 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
4372 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4374 TREE_OPERAND (val
, 1));
4375 gimple_call_set_lhs (g
, lastlane
);
4376 gimple_seq_add_stmt (stmt_list
, g
);
4378 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4379 TREE_OPERAND (val
, 0), lastlane
,
4380 NULL_TREE
, NULL_TREE
);
4384 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4385 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4387 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4388 gimple_seq_add_seq (stmt_list
,
4389 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4390 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4392 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4393 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4395 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4396 gimple_seq_add_seq (stmt_list
,
4397 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4398 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4401 x
= build_outer_var_ref (var
, ctx
);
4402 if (is_reference (var
))
4403 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4404 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4405 gimplify_and_add (x
, stmt_list
);
4407 c
= OMP_CLAUSE_CHAIN (c
);
4408 if (c
== NULL
&& !par_clauses
)
4410 /* If this was a workshare clause, see if it had been combined
4411 with its parallel. In that case, continue looking for the
4412 clauses also on the parallel statement itself. */
4413 if (is_parallel_ctx (ctx
))
4417 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4420 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4421 OMP_CLAUSE_LASTPRIVATE
);
4427 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4431 lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
, tree tid
,
4432 tree var
, tree new_var
)
4434 /* The atomic add at the end of the sum creates unnecessary
4435 write contention on accelerators. To work around this,
4436 create an array to store the partial reductions. Later, in
4437 lower_omp_for (for openacc), the values of array will be
4440 tree t
= NULL_TREE
, array
, x
;
4441 tree type
= get_base_type (var
);
4444 /* Now insert the partial reductions into the array. */
4446 /* Find the reduction array. */
4448 tree ptype
= build_pointer_type (type
);
4450 t
= lookup_reduction (omp_get_id (var
), ctx
);
4451 t
= build_receiver_ref (t
, false, ctx
->outer
);
4453 array
= create_tmp_var (ptype
, NULL
);
4454 gimplify_assign (array
, t
, stmt_seqp
);
4456 tree ptr
= create_tmp_var (TREE_TYPE (array
), NULL
);
4458 /* Find the reduction array. */
4460 /* testing a unary conversion. */
4461 tree offset
= create_tmp_var (sizetype
, NULL
);
4462 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
4464 t
= create_tmp_var (sizetype
, NULL
);
4465 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
4467 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, offset
, offset
, t
);
4468 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4470 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4471 of adding sizeof(var) to the array? */
4472 ptr
= create_tmp_var (ptype
, NULL
);
4473 stmt
= gimple_build_assign_with_ops (POINTER_PLUS_EXPR
, unshare_expr(ptr
),
4475 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4477 /* Move the local sum to gfc$sum[i]. */
4478 x
= unshare_expr (build_simple_mem_ref (ptr
));
4479 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
4482 /* Generate code to implement the REDUCTION clauses. */
4485 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4487 gimple_seq sub_seq
= NULL
;
4489 tree x
, c
, tid
= NULL_TREE
;
4492 /* SIMD reductions are handled in lower_rec_input_clauses. */
4493 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4494 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4497 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4498 update in that case, otherwise use a lock. */
4499 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4500 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4502 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4504 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4514 /* Initialize thread info for OpenACC. */
4515 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
4517 /* Get the current thread id. */
4518 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
4519 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)), NULL
);
4520 gimple stmt
= gimple_build_call (call
, 0);
4521 gimple_call_set_lhs (stmt
, tid
);
4522 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4525 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4527 tree var
, ref
, new_var
;
4528 enum tree_code code
;
4529 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4531 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4534 var
= OMP_CLAUSE_DECL (c
);
4535 new_var
= lookup_decl (var
, ctx
);
4536 if (is_reference (var
))
4537 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4538 ref
= build_outer_var_ref (var
, ctx
);
4539 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4541 /* reduction(-:var) sums up the partial results, so it acts
4542 identically to reduction(+:var). */
4543 if (code
== MINUS_EXPR
)
4548 if (!is_gimple_omp_oacc_specifically (ctx
->stmt
))
4550 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4552 addr
= save_expr (addr
);
4553 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4554 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4555 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4556 gimplify_and_add (x
, stmt_seqp
);
4561 lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
4566 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4568 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4570 if (is_reference (var
)
4571 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4573 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4574 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4575 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4576 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4577 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4578 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4579 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4583 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
4585 lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
4589 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4590 ref
= build_outer_var_ref (var
, ctx
);
4591 gimplify_assign (ref
, x
, &sub_seq
);
4596 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
4599 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4601 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4603 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4605 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4607 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4611 /* Generate code to implement the COPYPRIVATE clauses. */
4614 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4617 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
4621 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4623 tree var
, new_var
, ref
, x
;
4625 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4627 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4630 var
= OMP_CLAUSE_DECL (c
);
4631 by_ref
= use_pointer_for_field (var
, NULL
);
4633 ref
= build_sender_ref (var
, ctx
);
4634 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4637 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4638 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4640 gimplify_assign (ref
, x
, slist
);
4642 ref
= build_receiver_ref (var
, false, ctx
);
4645 ref
= fold_convert_loc (clause_loc
,
4646 build_pointer_type (TREE_TYPE (new_var
)),
4648 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4650 if (is_reference (var
))
4652 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4653 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4654 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4656 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4657 gimplify_and_add (x
, rlist
);
4662 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4663 and REDUCTION from the sender (aka parent) side. */
4666 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4669 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
4673 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4675 tree val
, ref
, x
, var
;
4676 bool by_ref
, do_in
= false, do_out
= false;
4677 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4679 switch (OMP_CLAUSE_CODE (c
))
4681 case OMP_CLAUSE_PRIVATE
:
4682 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4685 case OMP_CLAUSE_FIRSTPRIVATE
:
4686 case OMP_CLAUSE_COPYIN
:
4687 case OMP_CLAUSE_LASTPRIVATE
:
4688 case OMP_CLAUSE_REDUCTION
:
4689 case OMP_CLAUSE__LOOPTEMP_
:
4695 val
= OMP_CLAUSE_DECL (c
);
4696 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4698 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4699 && is_global_var (var
))
4701 if (is_variable_sized (val
))
4703 by_ref
= use_pointer_for_field (val
, NULL
);
4705 switch (OMP_CLAUSE_CODE (c
))
4707 case OMP_CLAUSE_PRIVATE
:
4708 case OMP_CLAUSE_FIRSTPRIVATE
:
4709 case OMP_CLAUSE_COPYIN
:
4710 case OMP_CLAUSE__LOOPTEMP_
:
4714 case OMP_CLAUSE_LASTPRIVATE
:
4715 if (by_ref
|| is_reference (val
))
4717 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4724 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4729 case OMP_CLAUSE_REDUCTION
:
4731 do_out
= !(by_ref
|| is_reference (val
));
4740 ref
= build_sender_ref (val
, ctx
);
4741 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4742 gimplify_assign (ref
, x
, ilist
);
4743 if (is_task_ctx (ctx
))
4744 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4749 ref
= build_sender_ref (val
, ctx
);
4750 gimplify_assign (var
, ref
, olist
);
4755 /* Generate code to implement SHARED from the sender (aka parent)
4756 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4757 list things that got automatically shared. */
4760 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4762 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
4764 tree var
, ovar
, nvar
, f
, x
, record_type
;
4766 if (ctx
->record_type
== NULL
)
4769 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4770 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4772 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4773 nvar
= maybe_lookup_decl (ovar
, ctx
);
4774 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4777 /* If CTX is a nested parallel directive. Find the immediately
4778 enclosing parallel or workshare construct that contains a
4779 mapping for OVAR. */
4780 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4782 if (use_pointer_for_field (ovar
, ctx
))
4784 x
= build_sender_ref (ovar
, ctx
);
4785 var
= build_fold_addr_expr (var
);
4786 gimplify_assign (x
, var
, ilist
);
4790 x
= build_sender_ref (ovar
, ctx
);
4791 gimplify_assign (x
, var
, ilist
);
4793 if (!TREE_READONLY (var
)
4794 /* We don't need to receive a new reference to a result
4795 or parm decl. In fact we may not store to it as we will
4796 invalidate any pending RSO and generate wrong gimple
4798 && !((TREE_CODE (var
) == RESULT_DECL
4799 || TREE_CODE (var
) == PARM_DECL
)
4800 && DECL_BY_REFERENCE (var
)))
4802 x
= build_sender_ref (ovar
, ctx
);
4803 gimplify_assign (var
, x
, olist
);
4810 /* A convenience function to build an empty GIMPLE_COND with just the
4814 gimple_build_cond_empty (tree cond
)
4816 enum tree_code pred_code
;
4819 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4820 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4824 /* Build the function calls to GOMP_parallel_start etc to actually
4825 generate the parallel operation. REGION is the parallel region
4826 being expanded. BB is the block where to insert the code. WS_ARGS
4827 will be set if this is a call to a combined parallel+workshare
4828 construct, it contains the list of additional arguments needed by
4829 the workshare construct. */
4832 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4833 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4835 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4836 gimple_stmt_iterator gsi
;
4838 enum built_in_function start_ix
;
4840 location_t clause_loc
;
4841 vec
<tree
, va_gc
> *args
;
4843 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4845 /* Determine what flavor of GOMP_parallel we will be
4847 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4848 if (is_combined_parallel (region
))
4850 switch (region
->inner
->type
)
4852 case GIMPLE_OMP_FOR
:
4853 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4854 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4855 + (region
->inner
->sched_kind
4856 == OMP_CLAUSE_SCHEDULE_RUNTIME
4857 ? 3 : region
->inner
->sched_kind
));
4858 start_ix
= (enum built_in_function
)start_ix2
;
4860 case GIMPLE_OMP_SECTIONS
:
4861 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4868 /* By default, the value of NUM_THREADS is zero (selected at run time)
4869 and there is no conditional. */
4871 val
= build_int_cst (unsigned_type_node
, 0);
4872 flags
= build_int_cst (unsigned_type_node
, 0);
4874 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4876 cond
= OMP_CLAUSE_IF_EXPR (c
);
4878 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4881 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4882 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4885 clause_loc
= gimple_location (entry_stmt
);
4887 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4889 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4891 /* Ensure 'val' is of the correct type. */
4892 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4894 /* If we found the clause 'if (cond)', build either
4895 (cond != 0) or (cond ? val : 1u). */
4898 cond
= gimple_boolify (cond
);
4900 if (integer_zerop (val
))
4901 val
= fold_build2_loc (clause_loc
,
4902 EQ_EXPR
, unsigned_type_node
, cond
,
4903 build_int_cst (TREE_TYPE (cond
), 0));
4906 basic_block cond_bb
, then_bb
, else_bb
;
4907 edge e
, e_then
, e_else
;
4908 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4910 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4911 if (gimple_in_ssa_p (cfun
))
4913 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4914 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4915 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4924 e
= split_block (bb
, NULL
);
4929 then_bb
= create_empty_bb (cond_bb
);
4930 else_bb
= create_empty_bb (then_bb
);
4931 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4932 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4934 stmt
= gimple_build_cond_empty (cond
);
4935 gsi
= gsi_start_bb (cond_bb
);
4936 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4938 gsi
= gsi_start_bb (then_bb
);
4939 stmt
= gimple_build_assign (tmp_then
, val
);
4940 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4942 gsi
= gsi_start_bb (else_bb
);
4943 stmt
= gimple_build_assign
4944 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4945 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4947 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4948 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4949 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4950 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4951 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4952 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4954 if (gimple_in_ssa_p (cfun
))
4956 gimple phi
= create_phi_node (tmp_join
, bb
);
4957 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4958 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4964 gsi
= gsi_start_bb (bb
);
4965 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4966 false, GSI_CONTINUE_LINKING
);
4969 gsi
= gsi_last_bb (bb
);
4970 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4972 t1
= null_pointer_node
;
4974 t1
= build_fold_addr_expr (t
);
4975 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4977 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4978 args
->quick_push (t2
);
4979 args
->quick_push (t1
);
4980 args
->quick_push (val
);
4982 args
->splice (*ws_args
);
4983 args
->quick_push (flags
);
4985 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4986 builtin_decl_explicit (start_ix
), args
);
4988 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4989 false, GSI_CONTINUE_LINKING
);
4992 /* Insert a function call whose name is FUNC_NAME with the information from
4993 ENTRY_STMT into the basic_block BB. */
4996 expand_cilk_for_call (basic_block bb
, gimple entry_stmt
,
4997 vec
<tree
, va_gc
> *ws_args
)
5000 gimple_stmt_iterator gsi
;
5001 vec
<tree
, va_gc
> *args
;
5003 gcc_assert (vec_safe_length (ws_args
) == 2);
5004 tree func_name
= (*ws_args
)[0];
5005 tree grain
= (*ws_args
)[1];
5007 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
5008 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
5009 gcc_assert (count
!= NULL_TREE
);
5010 count
= OMP_CLAUSE_OPERAND (count
, 0);
5012 gsi
= gsi_last_bb (bb
);
5013 t
= gimple_omp_parallel_data_arg (entry_stmt
);
5015 t1
= null_pointer_node
;
5017 t1
= build_fold_addr_expr (t
);
5018 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5020 vec_alloc (args
, 4);
5021 args
->quick_push (t2
);
5022 args
->quick_push (t1
);
5023 args
->quick_push (count
);
5024 args
->quick_push (grain
);
5025 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
5027 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
5028 GSI_CONTINUE_LINKING
);
5031 /* Build the function call to GOMP_task to actually
5032 generate the task operation. BB is the block where to insert the code. */
5035 expand_task_call (basic_block bb
, gimple entry_stmt
)
5037 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
5038 gimple_stmt_iterator gsi
;
5039 location_t loc
= gimple_location (entry_stmt
);
5041 clauses
= gimple_omp_task_clauses (entry_stmt
);
5043 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
5045 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
5047 cond
= boolean_true_node
;
5049 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
5050 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
5051 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
5052 flags
= build_int_cst (unsigned_type_node
,
5053 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
5055 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
5058 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
5059 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
5060 build_int_cst (unsigned_type_node
, 2),
5061 build_int_cst (unsigned_type_node
, 0));
5062 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
5065 depend
= OMP_CLAUSE_DECL (depend
);
5067 depend
= build_int_cst (ptr_type_node
, 0);
5069 gsi
= gsi_last_bb (bb
);
5070 t
= gimple_omp_task_data_arg (entry_stmt
);
5072 t2
= null_pointer_node
;
5074 t2
= build_fold_addr_expr_loc (loc
, t
);
5075 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
5076 t
= gimple_omp_task_copy_fn (entry_stmt
);
5078 t3
= null_pointer_node
;
5080 t3
= build_fold_addr_expr_loc (loc
, t
);
5082 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
5084 gimple_omp_task_arg_size (entry_stmt
),
5085 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
5088 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5089 false, GSI_CONTINUE_LINKING
);
5093 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5094 catch handler and return it. This prevents programs from violating the
5095 structured block semantics with throws. */
5098 maybe_catch_exception (gimple_seq body
)
5103 if (!flag_exceptions
)
5106 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
5107 decl
= lang_hooks
.eh_protect_cleanup_actions ();
5109 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
5111 g
= gimple_build_eh_must_not_throw (decl
);
5112 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
5115 return gimple_seq_alloc_with_stmt (g
);
5118 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5121 vec2chain (vec
<tree
, va_gc
> *v
)
5123 tree chain
= NULL_TREE
, t
;
5126 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
5128 DECL_CHAIN (t
) = chain
;
5136 /* Remove barriers in REGION->EXIT's block. Note that this is only
5137 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5138 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5139 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5143 remove_exit_barrier (struct omp_region
*region
)
5145 gimple_stmt_iterator gsi
;
5146 basic_block exit_bb
;
5150 int any_addressable_vars
= -1;
5152 exit_bb
= region
->exit
;
5154 /* If the parallel region doesn't return, we don't have REGION->EXIT
5159 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5160 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5161 statements that can appear in between are extremely limited -- no
5162 memory operations at all. Here, we allow nothing at all, so the
5163 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5164 gsi
= gsi_last_bb (exit_bb
);
5165 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5167 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
5170 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
5172 gsi
= gsi_last_bb (e
->src
);
5173 if (gsi_end_p (gsi
))
5175 stmt
= gsi_stmt (gsi
);
5176 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
5177 && !gimple_omp_return_nowait_p (stmt
))
5179 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5180 in many cases. If there could be tasks queued, the barrier
5181 might be needed to let the tasks run before some local
5182 variable of the parallel that the task uses as shared
5183 runs out of scope. The task can be spawned either
5184 from within current function (this would be easy to check)
5185 or from some function it calls and gets passed an address
5186 of such a variable. */
5187 if (any_addressable_vars
< 0)
5189 gimple parallel_stmt
= last_stmt (region
->entry
);
5190 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
5191 tree local_decls
, block
, decl
;
5194 any_addressable_vars
= 0;
5195 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
5196 if (TREE_ADDRESSABLE (decl
))
5198 any_addressable_vars
= 1;
5201 for (block
= gimple_block (stmt
);
5202 !any_addressable_vars
5204 && TREE_CODE (block
) == BLOCK
;
5205 block
= BLOCK_SUPERCONTEXT (block
))
5207 for (local_decls
= BLOCK_VARS (block
);
5209 local_decls
= DECL_CHAIN (local_decls
))
5210 if (TREE_ADDRESSABLE (local_decls
))
5212 any_addressable_vars
= 1;
5215 if (block
== gimple_block (parallel_stmt
))
5219 if (!any_addressable_vars
)
5220 gimple_omp_return_set_nowait (stmt
);
5226 remove_exit_barriers (struct omp_region
*region
)
5228 if (region
->type
== GIMPLE_OMP_PARALLEL
)
5229 remove_exit_barrier (region
);
5233 region
= region
->inner
;
5234 remove_exit_barriers (region
);
5235 while (region
->next
)
5237 region
= region
->next
;
5238 remove_exit_barriers (region
);
5243 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5244 calls. These can't be declared as const functions, but
5245 within one parallel body they are constant, so they can be
5246 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5247 which are declared const. Similarly for task body, except
5248 that in untied task omp_get_thread_num () can change at any task
5249 scheduling point. */
5252 optimize_omp_library_calls (gimple entry_stmt
)
5255 gimple_stmt_iterator gsi
;
5256 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5257 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
5258 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5259 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
5260 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
5261 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
5262 OMP_CLAUSE_UNTIED
) != NULL
);
5264 FOR_EACH_BB_FN (bb
, cfun
)
5265 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5267 gimple call
= gsi_stmt (gsi
);
5270 if (is_gimple_call (call
)
5271 && (decl
= gimple_call_fndecl (call
))
5272 && DECL_EXTERNAL (decl
)
5273 && TREE_PUBLIC (decl
)
5274 && DECL_INITIAL (decl
) == NULL
)
5278 if (DECL_NAME (decl
) == thr_num_id
)
5280 /* In #pragma omp task untied omp_get_thread_num () can change
5281 during the execution of the task region. */
5284 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5286 else if (DECL_NAME (decl
) == num_thr_id
)
5287 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5291 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
5292 || gimple_call_num_args (call
) != 0)
5295 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
5298 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
5299 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
5300 TREE_TYPE (TREE_TYPE (built_in
))))
5303 gimple_call_set_fndecl (call
, built_in
);
5308 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5312 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
5316 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5317 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
5320 if (TREE_CODE (t
) == ADDR_EXPR
)
5321 recompute_tree_invariant_for_addr_expr (t
);
5323 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
5327 /* Prepend TO = FROM assignment before *GSI_P. */
5330 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
5332 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
5333 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
5334 true, GSI_SAME_STMT
);
5335 gimple stmt
= gimple_build_assign (to
, from
);
5336 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
5337 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
5338 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
5340 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
5341 gimple_regimplify_operands (stmt
, &gsi
);
5345 /* Expand the OpenMP parallel or task directive starting at REGION. */
5348 expand_omp_taskreg (struct omp_region
*region
)
5350 basic_block entry_bb
, exit_bb
, new_bb
;
5351 struct function
*child_cfun
;
5352 tree child_fn
, block
, t
;
5353 gimple_stmt_iterator gsi
;
5354 gimple entry_stmt
, stmt
;
5356 vec
<tree
, va_gc
> *ws_args
;
5358 entry_stmt
= last_stmt (region
->entry
);
5359 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
5360 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
5362 entry_bb
= region
->entry
;
5363 exit_bb
= region
->exit
;
5367 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
5368 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
5369 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
5372 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5373 and the inner statement contains the name of the built-in function
5375 ws_args
= region
->inner
->ws_args
;
5376 else if (is_combined_parallel (region
))
5377 ws_args
= region
->ws_args
;
5381 if (child_cfun
->cfg
)
5383 /* Due to inlining, it may happen that we have already outlined
5384 the region, in which case all we need to do is make the
5385 sub-graph unreachable and emit the parallel call. */
5386 edge entry_succ_e
, exit_succ_e
;
5388 entry_succ_e
= single_succ_edge (entry_bb
);
5390 gsi
= gsi_last_bb (entry_bb
);
5391 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
5392 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
5393 gsi_remove (&gsi
, true);
5398 exit_succ_e
= single_succ_edge (exit_bb
);
5399 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
5401 remove_edge_and_dominated_blocks (entry_succ_e
);
5405 unsigned srcidx
, dstidx
, num
;
5407 /* If the parallel region needs data sent from the parent
5408 function, then the very first statement (except possible
5409 tree profile counter updates) of the parallel body
5410 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5411 &.OMP_DATA_O is passed as an argument to the child function,
5412 we need to replace it with the argument as seen by the child
5415 In most cases, this will end up being the identity assignment
5416 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5417 a function call that has been inlined, the original PARM_DECL
5418 .OMP_DATA_I may have been converted into a different local
5419 variable. In which case, we need to keep the assignment. */
5420 if (gimple_omp_taskreg_data_arg (entry_stmt
))
5422 basic_block entry_succ_bb
= single_succ (entry_bb
);
5424 gimple parcopy_stmt
= NULL
;
5426 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
5430 gcc_assert (!gsi_end_p (gsi
));
5431 stmt
= gsi_stmt (gsi
);
5432 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
5435 if (gimple_num_ops (stmt
) == 2)
5437 tree arg
= gimple_assign_rhs1 (stmt
);
5439 /* We're ignore the subcode because we're
5440 effectively doing a STRIP_NOPS. */
5442 if (TREE_CODE (arg
) == ADDR_EXPR
5443 && TREE_OPERAND (arg
, 0)
5444 == gimple_omp_taskreg_data_arg (entry_stmt
))
5446 parcopy_stmt
= stmt
;
5452 gcc_assert (parcopy_stmt
!= NULL
);
5453 arg
= DECL_ARGUMENTS (child_fn
);
5455 if (!gimple_in_ssa_p (cfun
))
5457 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5458 gsi_remove (&gsi
, true);
5461 /* ?? Is setting the subcode really necessary ?? */
5462 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5463 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5468 /* If we are in ssa form, we must load the value from the default
5469 definition of the argument. That should not be defined now,
5470 since the argument is not used uninitialized. */
5471 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5472 narg
= make_ssa_name (arg
, gimple_build_nop ());
5473 set_ssa_default_def (cfun
, arg
, narg
);
5474 /* ?? Is setting the subcode really necessary ?? */
5475 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5476 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5477 update_stmt (parcopy_stmt
);
5481 /* Declare local variables needed in CHILD_CFUN. */
5482 block
= DECL_INITIAL (child_fn
);
5483 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5484 /* The gimplifier could record temporaries in parallel/task block
5485 rather than in containing function's local_decls chain,
5486 which would mean cgraph missed finalizing them. Do it now. */
5487 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5488 if (TREE_CODE (t
) == VAR_DECL
5490 && !DECL_EXTERNAL (t
))
5491 varpool_node::finalize_decl (t
);
5492 DECL_SAVED_TREE (child_fn
) = NULL
;
5493 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5494 gimple_set_body (child_fn
, NULL
);
5495 TREE_USED (block
) = 1;
5497 /* Reset DECL_CONTEXT on function arguments. */
5498 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5499 DECL_CONTEXT (t
) = child_fn
;
5501 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5502 so that it can be moved to the child function. */
5503 gsi
= gsi_last_bb (entry_bb
);
5504 stmt
= gsi_stmt (gsi
);
5505 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5506 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5507 gsi_remove (&gsi
, true);
5508 e
= split_block (entry_bb
, stmt
);
5510 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5512 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5515 gsi
= gsi_last_bb (exit_bb
);
5516 gcc_assert (!gsi_end_p (gsi
)
5517 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5518 stmt
= gimple_build_return (NULL
);
5519 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5520 gsi_remove (&gsi
, true);
5523 /* Move the parallel region into CHILD_CFUN. */
5525 if (gimple_in_ssa_p (cfun
))
5527 init_tree_ssa (child_cfun
);
5528 init_ssa_operands (child_cfun
);
5529 child_cfun
->gimple_df
->in_ssa_p
= true;
5533 block
= gimple_block (entry_stmt
);
5535 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5537 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5538 /* When the OMP expansion process cannot guarantee an up-to-date
5539 loop tree arrange for the child function to fixup loops. */
5540 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5541 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5543 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5544 num
= vec_safe_length (child_cfun
->local_decls
);
5545 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5547 t
= (*child_cfun
->local_decls
)[srcidx
];
5548 if (DECL_CONTEXT (t
) == cfun
->decl
)
5550 if (srcidx
!= dstidx
)
5551 (*child_cfun
->local_decls
)[dstidx
] = t
;
5555 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5557 /* Inform the callgraph about the new function. */
5558 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5559 cgraph_node::add_new_function (child_fn
, true);
5561 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5562 fixed in a following pass. */
5563 push_cfun (child_cfun
);
5565 optimize_omp_library_calls (entry_stmt
);
5566 cgraph_edge::rebuild_edges ();
5568 /* Some EH regions might become dead, see PR34608. If
5569 pass_cleanup_cfg isn't the first pass to happen with the
5570 new child, these dead EH edges might cause problems.
5571 Clean them up now. */
5572 if (flag_exceptions
)
5575 bool changed
= false;
5577 FOR_EACH_BB_FN (bb
, cfun
)
5578 changed
|= gimple_purge_dead_eh_edges (bb
);
5580 cleanup_tree_cfg ();
5582 if (gimple_in_ssa_p (cfun
))
5583 update_ssa (TODO_update_ssa
);
5587 /* Emit a library call to launch the children threads. */
5589 expand_cilk_for_call (new_bb
, entry_stmt
, ws_args
);
5590 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5591 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
5593 expand_task_call (new_bb
, entry_stmt
);
5594 if (gimple_in_ssa_p (cfun
))
5595 update_ssa (TODO_update_ssa_only_virtuals
);
5599 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5600 of the combined collapse > 1 loop constructs, generate code like:
5601 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5606 count3 = (adj + N32 - N31) / STEP3;
5607 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5612 count2 = (adj + N22 - N21) / STEP2;
5613 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5618 count1 = (adj + N12 - N11) / STEP1;
5619 count = count1 * count2 * count3;
5620 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5622 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5623 of the combined loop constructs, just initialize COUNTS array
5624 from the _looptemp_ clauses. */
5626 /* NOTE: It *could* be better to moosh all of the BBs together,
5627 creating one larger BB with all the computation and the unexpected
5628 jump at the end. I.e.
5630 bool zero3, zero2, zero1, zero;
5633 count3 = (N32 - N31) /[cl] STEP3;
5635 count2 = (N22 - N21) /[cl] STEP2;
5637 count1 = (N12 - N11) /[cl] STEP1;
5638 zero = zero3 || zero2 || zero1;
5639 count = count1 * count2 * count3;
5640 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5642 After all, we expect the zero=false, and thus we expect to have to
5643 evaluate all of the comparison expressions, so short-circuiting
5644 oughtn't be a win. Since the condition isn't protecting a
5645 denominator, we're not concerned about divide-by-zero, so we can
5646 fully evaluate count even if a numerator turned out to be wrong.
5648 It seems like putting this all together would create much better
5649 scheduling opportunities, and less pressure on the chip's branch
5653 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5654 basic_block
&entry_bb
, tree
*counts
,
5655 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5656 basic_block
&l2_dom_bb
)
5658 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5663 /* Collapsed loops need work for expansion into SSA form. */
5664 gcc_assert (!gimple_in_ssa_p (cfun
));
5666 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5667 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5669 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5670 isn't supposed to be handled, as the inner loop doesn't
5672 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5673 OMP_CLAUSE__LOOPTEMP_
);
5674 gcc_assert (innerc
);
5675 for (i
= 0; i
< fd
->collapse
; i
++)
5677 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5678 OMP_CLAUSE__LOOPTEMP_
);
5679 gcc_assert (innerc
);
5681 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5683 counts
[0] = NULL_TREE
;
5688 for (i
= 0; i
< fd
->collapse
; i
++)
5690 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5692 if (SSA_VAR_P (fd
->loop
.n2
)
5693 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5694 fold_convert (itype
, fd
->loops
[i
].n1
),
5695 fold_convert (itype
, fd
->loops
[i
].n2
)))
5696 == NULL_TREE
|| !integer_onep (t
)))
5699 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5700 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5701 true, GSI_SAME_STMT
);
5702 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5703 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5704 true, GSI_SAME_STMT
);
5705 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5706 NULL_TREE
, NULL_TREE
);
5707 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5708 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5709 expand_omp_regimplify_p
, NULL
, NULL
)
5710 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5711 expand_omp_regimplify_p
, NULL
, NULL
))
5713 *gsi
= gsi_for_stmt (stmt
);
5714 gimple_regimplify_operands (stmt
, gsi
);
5716 e
= split_block (entry_bb
, stmt
);
5717 if (zero_iter_bb
== NULL
)
5719 first_zero_iter
= i
;
5720 zero_iter_bb
= create_empty_bb (entry_bb
);
5721 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5722 *gsi
= gsi_after_labels (zero_iter_bb
);
5723 stmt
= gimple_build_assign (fd
->loop
.n2
,
5724 build_zero_cst (type
));
5725 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5726 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5729 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5730 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5731 e
->flags
= EDGE_TRUE_VALUE
;
5732 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5733 if (l2_dom_bb
== NULL
)
5734 l2_dom_bb
= entry_bb
;
5736 *gsi
= gsi_last_bb (entry_bb
);
5739 if (POINTER_TYPE_P (itype
))
5740 itype
= signed_type_for (itype
);
5741 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5743 t
= fold_build2 (PLUS_EXPR
, itype
,
5744 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5745 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5746 fold_convert (itype
, fd
->loops
[i
].n2
));
5747 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5748 fold_convert (itype
, fd
->loops
[i
].n1
));
5749 /* ?? We could probably use CEIL_DIV_EXPR instead of
5750 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5751 generate the same code in the end because generically we
5752 don't know that the values involved must be negative for
5754 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5755 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5756 fold_build1 (NEGATE_EXPR
, itype
, t
),
5757 fold_build1 (NEGATE_EXPR
, itype
,
5758 fold_convert (itype
,
5759 fd
->loops
[i
].step
)));
5761 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5762 fold_convert (itype
, fd
->loops
[i
].step
));
5763 t
= fold_convert (type
, t
);
5764 if (TREE_CODE (t
) == INTEGER_CST
)
5768 counts
[i
] = create_tmp_reg (type
, ".count");
5769 expand_omp_build_assign (gsi
, counts
[i
], t
);
5771 if (SSA_VAR_P (fd
->loop
.n2
))
5776 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5777 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5783 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5785 V3 = N31 + (T % count3) * STEP3;
5787 V2 = N21 + (T % count2) * STEP2;
5789 V1 = N11 + T * STEP1;
5790 if this loop doesn't have an inner loop construct combined with it.
5791 If it does have an inner loop construct combined with it and the
5792 iteration count isn't known constant, store values from counts array
5793 into its _looptemp_ temporaries instead. */
5796 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5797 tree
*counts
, gimple inner_stmt
, tree startvar
)
5800 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5802 /* If fd->loop.n2 is constant, then no propagation of the counts
5803 is needed, they are constant. */
5804 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5807 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5808 ? gimple_omp_parallel_clauses (inner_stmt
)
5809 : gimple_omp_for_clauses (inner_stmt
);
5810 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5811 isn't supposed to be handled, as the inner loop doesn't
5813 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5814 gcc_assert (innerc
);
5815 for (i
= 0; i
< fd
->collapse
; i
++)
5817 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5818 OMP_CLAUSE__LOOPTEMP_
);
5819 gcc_assert (innerc
);
5822 tree tem
= OMP_CLAUSE_DECL (innerc
);
5823 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5824 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5825 false, GSI_CONTINUE_LINKING
);
5826 gimple stmt
= gimple_build_assign (tem
, t
);
5827 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5833 tree type
= TREE_TYPE (fd
->loop
.v
);
5834 tree tem
= create_tmp_reg (type
, ".tem");
5835 gimple stmt
= gimple_build_assign (tem
, startvar
);
5836 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5838 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5840 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5842 if (POINTER_TYPE_P (vtype
))
5843 itype
= signed_type_for (vtype
);
5845 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5848 t
= fold_convert (itype
, t
);
5849 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5850 fold_convert (itype
, fd
->loops
[i
].step
));
5851 if (POINTER_TYPE_P (vtype
))
5852 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5854 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5855 t
= force_gimple_operand_gsi (gsi
, t
,
5856 DECL_P (fd
->loops
[i
].v
)
5857 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5859 GSI_CONTINUE_LINKING
);
5860 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5861 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5864 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5865 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5866 false, GSI_CONTINUE_LINKING
);
5867 stmt
= gimple_build_assign (tem
, t
);
5868 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5874 /* Helper function for expand_omp_for_*. Generate code like:
5877 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5881 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5888 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5889 basic_block body_bb
)
5891 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5893 gimple_stmt_iterator gsi
;
5899 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5901 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5903 bb
= create_empty_bb (last_bb
);
5904 add_bb_to_loop (bb
, last_bb
->loop_father
);
5905 gsi
= gsi_start_bb (bb
);
5907 if (i
< fd
->collapse
- 1)
5909 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5910 e
->probability
= REG_BR_PROB_BASE
/ 8;
5912 t
= fd
->loops
[i
+ 1].n1
;
5913 t
= force_gimple_operand_gsi (&gsi
, t
,
5914 DECL_P (fd
->loops
[i
+ 1].v
)
5915 && TREE_ADDRESSABLE (fd
->loops
[i
5918 GSI_CONTINUE_LINKING
);
5919 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5920 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5925 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5927 if (POINTER_TYPE_P (vtype
))
5928 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5930 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5931 t
= force_gimple_operand_gsi (&gsi
, t
,
5932 DECL_P (fd
->loops
[i
].v
)
5933 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5934 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5935 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5936 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5940 t
= fd
->loops
[i
].n2
;
5941 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5942 false, GSI_CONTINUE_LINKING
);
5943 tree v
= fd
->loops
[i
].v
;
5944 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5945 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5946 false, GSI_CONTINUE_LINKING
);
5947 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5948 stmt
= gimple_build_cond_empty (t
);
5949 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5950 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5951 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5954 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5962 /* A subroutine of expand_omp_for. Generate code for a parallel
5963 loop with any schedule. Given parameters:
5965 for (V = N1; V cond N2; V += STEP) BODY;
5967 where COND is "<" or ">", we generate pseudocode
5969 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5970 if (more) goto L0; else goto L3;
5977 if (V cond iend) goto L1; else goto L2;
5979 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5982 If this is a combined omp parallel loop, instead of the call to
5983 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5984 If this is gimple_omp_for_combined_p loop, then instead of assigning
5985 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5986 inner GIMPLE_OMP_FOR and V += STEP; and
5987 if (V cond iend) goto L1; else goto L2; are removed.
5989 For collapsed loops, given parameters:
5991 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5992 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5993 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5996 we generate pseudocode
5998 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6003 count3 = (adj + N32 - N31) / STEP3;
6004 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6009 count2 = (adj + N22 - N21) / STEP2;
6010 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6015 count1 = (adj + N12 - N11) / STEP1;
6016 count = count1 * count2 * count3;
6021 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6022 if (more) goto L0; else goto L3;
6026 V3 = N31 + (T % count3) * STEP3;
6028 V2 = N21 + (T % count2) * STEP2;
6030 V1 = N11 + T * STEP1;
6035 if (V < iend) goto L10; else goto L2;
6038 if (V3 cond3 N32) goto L1; else goto L11;
6042 if (V2 cond2 N22) goto L1; else goto L12;
6048 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6054 expand_omp_for_generic (struct omp_region
*region
,
6055 struct omp_for_data
*fd
,
6056 enum built_in_function start_fn
,
6057 enum built_in_function next_fn
,
6060 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
) != GF_OMP_FOR_KIND_OACC_LOOP
);
6062 tree type
, istart0
, iend0
, iend
;
6063 tree t
, vmain
, vback
, bias
= NULL_TREE
;
6064 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
6065 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
6066 gimple_stmt_iterator gsi
;
6068 bool in_combined_parallel
= is_combined_parallel (region
);
6069 bool broken_loop
= region
->cont
== NULL
;
6071 tree
*counts
= NULL
;
6074 gcc_assert (!broken_loop
|| !in_combined_parallel
);
6075 gcc_assert (fd
->iter_type
== long_integer_type_node
6076 || !in_combined_parallel
);
6078 type
= TREE_TYPE (fd
->loop
.v
);
6079 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
6080 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
6081 TREE_ADDRESSABLE (istart0
) = 1;
6082 TREE_ADDRESSABLE (iend0
) = 1;
6084 /* See if we need to bias by LLONG_MIN. */
6085 if (fd
->iter_type
== long_long_unsigned_type_node
6086 && TREE_CODE (type
) == INTEGER_TYPE
6087 && !TYPE_UNSIGNED (type
))
6091 if (fd
->loop
.cond_code
== LT_EXPR
)
6094 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6098 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6101 if (TREE_CODE (n1
) != INTEGER_CST
6102 || TREE_CODE (n2
) != INTEGER_CST
6103 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
6104 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
6107 entry_bb
= region
->entry
;
6108 cont_bb
= region
->cont
;
6110 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6111 gcc_assert (broken_loop
6112 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6113 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6114 l1_bb
= single_succ (l0_bb
);
6117 l2_bb
= create_empty_bb (cont_bb
);
6118 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
6119 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6123 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6124 exit_bb
= region
->exit
;
6126 gsi
= gsi_last_bb (entry_bb
);
6128 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6129 if (fd
->collapse
> 1)
6131 gcc_assert (gimple_omp_for_kind (gsi_stmt (gsi
))
6132 != GF_OMP_FOR_KIND_OACC_LOOP
);
6134 int first_zero_iter
= -1;
6135 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
6137 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6138 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6139 zero_iter_bb
, first_zero_iter
,
6144 /* Some counts[i] vars might be uninitialized if
6145 some loop has zero iterations. But the body shouldn't
6146 be executed in that case, so just avoid uninit warnings. */
6147 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
6148 if (SSA_VAR_P (counts
[i
]))
6149 TREE_NO_WARNING (counts
[i
]) = 1;
6151 e
= split_block (entry_bb
, gsi_stmt (gsi
));
6153 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
6154 gsi
= gsi_last_bb (entry_bb
);
6155 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
6156 get_immediate_dominator (CDI_DOMINATORS
,
6160 if (in_combined_parallel
)
6162 gcc_assert (gimple_omp_for_kind (gsi_stmt (gsi
))
6163 != GF_OMP_FOR_KIND_OACC_LOOP
);
6165 /* In a combined parallel loop, emit a call to
6166 GOMP_loop_foo_next. */
6167 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6168 build_fold_addr_expr (istart0
),
6169 build_fold_addr_expr (iend0
));
6173 tree t0
, t1
, t2
, t3
, t4
;
6174 /* If this is not a combined parallel loop, emit a call to
6175 GOMP_loop_foo_start in ENTRY_BB. */
6176 t4
= build_fold_addr_expr (iend0
);
6177 t3
= build_fold_addr_expr (istart0
);
6178 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
6181 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6183 gcc_assert (gimple_omp_for_kind (gsi_stmt (gsi
))
6184 != GF_OMP_FOR_KIND_OACC_LOOP
);
6186 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6187 OMP_CLAUSE__LOOPTEMP_
);
6188 gcc_assert (innerc
);
6189 t0
= OMP_CLAUSE_DECL (innerc
);
6190 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6191 OMP_CLAUSE__LOOPTEMP_
);
6192 gcc_assert (innerc
);
6193 t1
= OMP_CLAUSE_DECL (innerc
);
6195 if (POINTER_TYPE_P (TREE_TYPE (t0
))
6196 && TYPE_PRECISION (TREE_TYPE (t0
))
6197 != TYPE_PRECISION (fd
->iter_type
))
6199 /* Avoid casting pointers to integer of a different size. */
6200 tree itype
= signed_type_for (type
);
6201 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
6202 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
6206 t1
= fold_convert (fd
->iter_type
, t1
);
6207 t0
= fold_convert (fd
->iter_type
, t0
);
6211 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
6212 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
6214 if (fd
->iter_type
== long_integer_type_node
)
6218 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6219 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6220 6, t0
, t1
, t2
, t
, t3
, t4
);
6223 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6224 5, t0
, t1
, t2
, t3
, t4
);
6232 /* The GOMP_loop_ull_*start functions have additional boolean
6233 argument, true for < loops and false for > loops.
6234 In Fortran, the C bool type can be different from
6235 boolean_type_node. */
6236 bfn_decl
= builtin_decl_explicit (start_fn
);
6237 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
6238 t5
= build_int_cst (c_bool_type
,
6239 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
6242 tree bfn_decl
= builtin_decl_explicit (start_fn
);
6243 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6244 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
6247 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6248 6, t5
, t0
, t1
, t2
, t3
, t4
);
6251 if (TREE_TYPE (t
) != boolean_type_node
)
6252 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6253 t
, build_int_cst (TREE_TYPE (t
), 0));
6254 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6255 true, GSI_SAME_STMT
);
6256 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6258 /* Remove the GIMPLE_OMP_FOR statement. */
6259 gsi_remove (&gsi
, true);
6261 /* Iteration setup for sequential loop goes in L0_BB. */
6262 tree startvar
= fd
->loop
.v
;
6263 tree endvar
= NULL_TREE
;
6265 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6267 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
6268 && gimple_omp_for_kind (inner_stmt
)
6269 == GF_OMP_FOR_KIND_SIMD
);
6270 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
6271 OMP_CLAUSE__LOOPTEMP_
);
6272 gcc_assert (innerc
);
6273 startvar
= OMP_CLAUSE_DECL (innerc
);
6274 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6275 OMP_CLAUSE__LOOPTEMP_
);
6276 gcc_assert (innerc
);
6277 endvar
= OMP_CLAUSE_DECL (innerc
);
6280 gsi
= gsi_start_bb (l0_bb
);
6283 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6284 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6285 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6286 t
= fold_convert (TREE_TYPE (startvar
), t
);
6287 t
= force_gimple_operand_gsi (&gsi
, t
,
6289 && TREE_ADDRESSABLE (startvar
),
6290 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6291 stmt
= gimple_build_assign (startvar
, t
);
6292 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6296 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6297 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6298 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6299 t
= fold_convert (TREE_TYPE (startvar
), t
);
6300 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6301 false, GSI_CONTINUE_LINKING
);
6304 stmt
= gimple_build_assign (endvar
, iend
);
6305 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6306 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
6307 stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
6309 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, iend
,
6311 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6313 if (fd
->collapse
> 1)
6314 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6318 /* Code to control the increment and predicate for the sequential
6319 loop goes in the CONT_BB. */
6320 gsi
= gsi_last_bb (cont_bb
);
6321 stmt
= gsi_stmt (gsi
);
6322 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6323 vmain
= gimple_omp_continue_control_use (stmt
);
6324 vback
= gimple_omp_continue_control_def (stmt
);
6326 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6328 if (POINTER_TYPE_P (type
))
6329 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
6331 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
6332 t
= force_gimple_operand_gsi (&gsi
, t
,
6334 && TREE_ADDRESSABLE (vback
),
6335 NULL_TREE
, true, GSI_SAME_STMT
);
6336 stmt
= gimple_build_assign (vback
, t
);
6337 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6339 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6340 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
6342 stmt
= gimple_build_cond_empty (t
);
6343 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6346 /* Remove GIMPLE_OMP_CONTINUE. */
6347 gsi_remove (&gsi
, true);
6349 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6350 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
6352 /* Emit code to get the next parallel iteration in L2_BB. */
6353 gsi
= gsi_start_bb (l2_bb
);
6355 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6356 build_fold_addr_expr (istart0
),
6357 build_fold_addr_expr (iend0
));
6358 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6359 false, GSI_CONTINUE_LINKING
);
6360 if (TREE_TYPE (t
) != boolean_type_node
)
6361 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6362 t
, build_int_cst (TREE_TYPE (t
), 0));
6363 stmt
= gimple_build_cond_empty (t
);
6364 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6367 /* Add the loop cleanup function. */
6368 gsi
= gsi_last_bb (exit_bb
);
6369 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6370 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
6371 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6372 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
6374 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
6375 stmt
= gimple_build_call (t
, 0);
6376 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6377 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
6378 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
6379 gsi_remove (&gsi
, true);
6381 /* Connect the new blocks. */
6382 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
6383 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
6389 e
= find_edge (cont_bb
, l3_bb
);
6390 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
6392 phis
= phi_nodes (l3_bb
);
6393 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6395 gimple phi
= gsi_stmt (gsi
);
6396 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
6397 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
6401 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
6402 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
6403 e
= find_edge (cont_bb
, l1_bb
);
6404 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6409 else if (fd
->collapse
> 1)
6412 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6415 e
->flags
= EDGE_TRUE_VALUE
;
6418 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6419 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
6423 e
= find_edge (cont_bb
, l2_bb
);
6424 e
->flags
= EDGE_FALLTHRU
;
6426 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6428 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
6429 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
6430 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
6431 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
6432 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
6433 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
6434 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
6435 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
6437 struct loop
*outer_loop
= alloc_loop ();
6438 outer_loop
->header
= l0_bb
;
6439 outer_loop
->latch
= l2_bb
;
6440 add_loop (outer_loop
, l0_bb
->loop_father
);
6442 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6444 struct loop
*loop
= alloc_loop ();
6445 loop
->header
= l1_bb
;
6446 /* The loop may have multiple latches. */
6447 add_loop (loop
, outer_loop
);
6453 /* A subroutine of expand_omp_for. Generate code for a parallel
6454 loop with static schedule and no specified chunk size. Given
6457 for (V = N1; V cond N2; V += STEP) BODY;
6459 where COND is "<" or ">", we generate pseudocode
6461 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6466 if ((__typeof (V)) -1 > 0 && cond is >)
6467 n = -(adj + N2 - N1) / -STEP;
6469 n = (adj + N2 - N1) / STEP;
6472 if (threadid < tt) goto L3; else goto L4;
6477 s0 = q * threadid + tt;
6480 if (s0 >= e0) goto L2; else goto L0;
6486 if (V cond e) goto L1;
6491 expand_omp_for_static_nochunk (struct omp_region
*region
,
6492 struct omp_for_data
*fd
,
6495 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6496 tree type
, itype
, vmain
, vback
;
6497 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6498 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6500 gimple_stmt_iterator gsi
;
6503 bool broken_loop
= region
->cont
== NULL
;
6504 tree
*counts
= NULL
;
6507 gcc_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6508 != GF_OMP_FOR_KIND_OACC_LOOP
)
6511 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6512 if (POINTER_TYPE_P (type
))
6513 itype
= signed_type_for (type
);
6515 entry_bb
= region
->entry
;
6516 cont_bb
= region
->cont
;
6517 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6518 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6519 gcc_assert (broken_loop
6520 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6521 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6522 body_bb
= single_succ (seq_start_bb
);
6525 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6526 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6528 exit_bb
= region
->exit
;
6530 /* Iteration space partitioning goes in ENTRY_BB. */
6531 gsi
= gsi_last_bb (entry_bb
);
6532 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6534 if (fd
->collapse
> 1)
6536 int first_zero_iter
= -1;
6537 basic_block l2_dom_bb
= NULL
;
6539 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6540 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6541 fin_bb
, first_zero_iter
,
6545 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6546 t
= integer_one_node
;
6548 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6549 fold_convert (type
, fd
->loop
.n1
),
6550 fold_convert (type
, fd
->loop
.n2
));
6551 if (fd
->collapse
== 1
6552 && TYPE_UNSIGNED (type
)
6553 && (t
== NULL_TREE
|| !integer_onep (t
)))
6555 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6556 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6557 true, GSI_SAME_STMT
);
6558 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6559 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6560 true, GSI_SAME_STMT
);
6561 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6562 NULL_TREE
, NULL_TREE
);
6563 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6564 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6565 expand_omp_regimplify_p
, NULL
, NULL
)
6566 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6567 expand_omp_regimplify_p
, NULL
, NULL
))
6569 gsi
= gsi_for_stmt (stmt
);
6570 gimple_regimplify_operands (stmt
, &gsi
);
6572 ep
= split_block (entry_bb
, stmt
);
6573 ep
->flags
= EDGE_TRUE_VALUE
;
6574 entry_bb
= ep
->dest
;
6575 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6576 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6577 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6578 if (gimple_in_ssa_p (cfun
))
6580 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6581 for (gsi
= gsi_start_phis (fin_bb
);
6582 !gsi_end_p (gsi
); gsi_next (&gsi
))
6584 gimple phi
= gsi_stmt (gsi
);
6585 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6586 ep
, UNKNOWN_LOCATION
);
6589 gsi
= gsi_last_bb (entry_bb
);
6592 switch (gimple_omp_for_kind (fd
->for_stmt
))
6594 case GF_OMP_FOR_KIND_FOR
:
6595 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6596 nthreads
= build_call_expr (nthreads
, 0);
6597 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6598 threadid
= build_call_expr (threadid
, 0);
6600 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6601 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6602 nthreads
= build_call_expr (nthreads
, 0);
6603 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6604 threadid
= build_call_expr (threadid
, 0);
6606 case GF_OMP_FOR_KIND_OACC_LOOP
:
6607 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6608 nthreads
= build_call_expr (nthreads
, 0);
6609 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
6610 threadid
= build_call_expr (threadid
, 0);
6615 nthreads
= fold_convert (itype
, nthreads
);
6616 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
6617 true, GSI_SAME_STMT
);
6618 threadid
= fold_convert (itype
, threadid
);
6619 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
6620 true, GSI_SAME_STMT
);
6624 step
= fd
->loop
.step
;
6625 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6627 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
6628 != GF_OMP_FOR_KIND_OACC_LOOP
);
6630 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6631 OMP_CLAUSE__LOOPTEMP_
);
6632 gcc_assert (innerc
);
6633 n1
= OMP_CLAUSE_DECL (innerc
);
6634 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6635 OMP_CLAUSE__LOOPTEMP_
);
6636 gcc_assert (innerc
);
6637 n2
= OMP_CLAUSE_DECL (innerc
);
6639 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6640 true, NULL_TREE
, true, GSI_SAME_STMT
);
6641 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6642 true, NULL_TREE
, true, GSI_SAME_STMT
);
6643 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6644 true, NULL_TREE
, true, GSI_SAME_STMT
);
6646 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6647 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6648 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6649 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6650 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6651 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6652 fold_build1 (NEGATE_EXPR
, itype
, t
),
6653 fold_build1 (NEGATE_EXPR
, itype
, step
));
6655 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6656 t
= fold_convert (itype
, t
);
6657 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6659 q
= create_tmp_reg (itype
, "q");
6660 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6661 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6662 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6664 tt
= create_tmp_reg (itype
, "tt");
6665 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6666 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6667 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6669 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6670 stmt
= gimple_build_cond_empty (t
);
6671 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6673 second_bb
= split_block (entry_bb
, stmt
)->dest
;
6674 gsi
= gsi_last_bb (second_bb
);
6675 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6677 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6679 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
6680 build_int_cst (itype
, 1));
6681 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6683 third_bb
= split_block (second_bb
, stmt
)->dest
;
6684 gsi
= gsi_last_bb (third_bb
);
6685 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6687 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6688 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6689 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6691 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6692 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6694 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6695 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6697 /* Remove the GIMPLE_OMP_FOR statement. */
6698 gsi_remove (&gsi
, true);
6700 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6701 gsi
= gsi_start_bb (seq_start_bb
);
6703 tree startvar
= fd
->loop
.v
;
6704 tree endvar
= NULL_TREE
;
6706 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6708 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6709 ? gimple_omp_parallel_clauses (inner_stmt
)
6710 : gimple_omp_for_clauses (inner_stmt
);
6711 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6712 gcc_assert (innerc
);
6713 startvar
= OMP_CLAUSE_DECL (innerc
);
6714 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6715 OMP_CLAUSE__LOOPTEMP_
);
6716 gcc_assert (innerc
);
6717 endvar
= OMP_CLAUSE_DECL (innerc
);
6719 t
= fold_convert (itype
, s0
);
6720 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6721 if (POINTER_TYPE_P (type
))
6722 t
= fold_build_pointer_plus (n1
, t
);
6724 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6725 t
= fold_convert (TREE_TYPE (startvar
), t
);
6726 t
= force_gimple_operand_gsi (&gsi
, t
,
6728 && TREE_ADDRESSABLE (startvar
),
6729 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6730 stmt
= gimple_build_assign (startvar
, t
);
6731 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6733 t
= fold_convert (itype
, e0
);
6734 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6735 if (POINTER_TYPE_P (type
))
6736 t
= fold_build_pointer_plus (n1
, t
);
6738 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6739 t
= fold_convert (TREE_TYPE (startvar
), t
);
6740 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6741 false, GSI_CONTINUE_LINKING
);
6744 stmt
= gimple_build_assign (endvar
, e
);
6745 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6746 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6747 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6749 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6751 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6753 if (fd
->collapse
> 1)
6754 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6758 /* The code controlling the sequential loop replaces the
6759 GIMPLE_OMP_CONTINUE. */
6760 gsi
= gsi_last_bb (cont_bb
);
6761 stmt
= gsi_stmt (gsi
);
6762 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6763 vmain
= gimple_omp_continue_control_use (stmt
);
6764 vback
= gimple_omp_continue_control_def (stmt
);
6766 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6768 if (POINTER_TYPE_P (type
))
6769 t
= fold_build_pointer_plus (vmain
, step
);
6771 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6772 t
= force_gimple_operand_gsi (&gsi
, t
,
6774 && TREE_ADDRESSABLE (vback
),
6775 NULL_TREE
, true, GSI_SAME_STMT
);
6776 stmt
= gimple_build_assign (vback
, t
);
6777 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6779 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6780 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6782 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6785 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6786 gsi_remove (&gsi
, true);
6788 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6789 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6792 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6793 gsi
= gsi_last_bb (exit_bb
);
6794 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6796 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
6797 != GF_OMP_FOR_KIND_OACC_LOOP
);
6799 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6800 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6802 gsi_remove (&gsi
, true);
6804 /* Connect all the blocks. */
6805 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6806 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6807 ep
= find_edge (entry_bb
, second_bb
);
6808 ep
->flags
= EDGE_TRUE_VALUE
;
6809 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6810 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6811 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6815 ep
= find_edge (cont_bb
, body_bb
);
6816 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6818 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
6819 != GF_OMP_FOR_KIND_OACC_LOOP
);
6824 else if (fd
->collapse
> 1)
6827 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6830 ep
->flags
= EDGE_TRUE_VALUE
;
6831 find_edge (cont_bb
, fin_bb
)->flags
6832 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6835 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6836 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6837 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6839 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6840 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6841 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6842 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6844 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6846 struct loop
*loop
= alloc_loop ();
6847 loop
->header
= body_bb
;
6848 if (collapse_bb
== NULL
)
6849 loop
->latch
= cont_bb
;
6850 add_loop (loop
, body_bb
->loop_father
);
6855 /* A subroutine of expand_omp_for. Generate code for a parallel
6856 loop with static schedule and a specified chunk size. Given
6859 for (V = N1; V cond N2; V += STEP) BODY;
6861 where COND is "<" or ">", we generate pseudocode
6863 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6868 if ((__typeof (V)) -1 > 0 && cond is >)
6869 n = -(adj + N2 - N1) / -STEP;
6871 n = (adj + N2 - N1) / STEP;
6873 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6874 here so that V is defined
6875 if the loop is not entered
6877 s0 = (trip * nthreads + threadid) * CHUNK;
6878 e0 = min(s0 + CHUNK, n);
6879 if (s0 < n) goto L1; else goto L4;
6886 if (V cond e) goto L2; else goto L3;
6894 expand_omp_for_static_chunk (struct omp_region
*region
,
6895 struct omp_for_data
*fd
, gimple inner_stmt
)
6897 tree n
, s0
, e0
, e
, t
;
6898 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6899 tree type
, itype
, vmain
, vback
, vextra
;
6900 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6901 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6902 gimple_stmt_iterator gsi
;
6905 bool broken_loop
= region
->cont
== NULL
;
6906 tree
*counts
= NULL
;
6909 gcc_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6910 != GF_OMP_FOR_KIND_OACC_LOOP
)
6913 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6914 if (POINTER_TYPE_P (type
))
6915 itype
= signed_type_for (type
);
6917 entry_bb
= region
->entry
;
6918 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6920 iter_part_bb
= se
->dest
;
6921 cont_bb
= region
->cont
;
6922 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6923 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6924 gcc_assert (broken_loop
6925 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6926 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6927 body_bb
= single_succ (seq_start_bb
);
6930 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6931 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6932 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6934 exit_bb
= region
->exit
;
6936 /* Trip and adjustment setup goes in ENTRY_BB. */
6937 gsi
= gsi_last_bb (entry_bb
);
6938 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6940 if (fd
->collapse
> 1)
6942 int first_zero_iter
= -1;
6943 basic_block l2_dom_bb
= NULL
;
6945 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6946 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6947 fin_bb
, first_zero_iter
,
6951 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6952 t
= integer_one_node
;
6954 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6955 fold_convert (type
, fd
->loop
.n1
),
6956 fold_convert (type
, fd
->loop
.n2
));
6957 if (fd
->collapse
== 1
6958 && TYPE_UNSIGNED (type
)
6959 && (t
== NULL_TREE
|| !integer_onep (t
)))
6961 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6962 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6963 true, GSI_SAME_STMT
);
6964 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6965 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6966 true, GSI_SAME_STMT
);
6967 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6968 NULL_TREE
, NULL_TREE
);
6969 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6970 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6971 expand_omp_regimplify_p
, NULL
, NULL
)
6972 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6973 expand_omp_regimplify_p
, NULL
, NULL
))
6975 gsi
= gsi_for_stmt (stmt
);
6976 gimple_regimplify_operands (stmt
, &gsi
);
6978 se
= split_block (entry_bb
, stmt
);
6979 se
->flags
= EDGE_TRUE_VALUE
;
6980 entry_bb
= se
->dest
;
6981 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6982 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6983 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6984 if (gimple_in_ssa_p (cfun
))
6986 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6987 for (gsi
= gsi_start_phis (fin_bb
);
6988 !gsi_end_p (gsi
); gsi_next (&gsi
))
6990 gimple phi
= gsi_stmt (gsi
);
6991 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6992 se
, UNKNOWN_LOCATION
);
6995 gsi
= gsi_last_bb (entry_bb
);
6998 switch (gimple_omp_for_kind (fd
->for_stmt
))
7000 case GF_OMP_FOR_KIND_FOR
:
7001 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7002 nthreads
= build_call_expr (nthreads
, 0);
7003 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7004 threadid
= build_call_expr (threadid
, 0);
7006 case GF_OMP_FOR_KIND_DISTRIBUTE
:
7007 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
7008 nthreads
= build_call_expr (nthreads
, 0);
7009 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
7010 threadid
= build_call_expr (threadid
, 0);
7012 case GF_OMP_FOR_KIND_OACC_LOOP
:
7013 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
7014 nthreads
= build_call_expr (nthreads
, 0);
7015 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
7016 threadid
= build_call_expr (threadid
, 0);
7021 nthreads
= fold_convert (itype
, nthreads
);
7022 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
7023 true, GSI_SAME_STMT
);
7024 threadid
= fold_convert (itype
, threadid
);
7025 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
7026 true, GSI_SAME_STMT
);
7030 step
= fd
->loop
.step
;
7031 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7033 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
7034 != GF_OMP_FOR_KIND_OACC_LOOP
);
7036 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7037 OMP_CLAUSE__LOOPTEMP_
);
7038 gcc_assert (innerc
);
7039 n1
= OMP_CLAUSE_DECL (innerc
);
7040 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7041 OMP_CLAUSE__LOOPTEMP_
);
7042 gcc_assert (innerc
);
7043 n2
= OMP_CLAUSE_DECL (innerc
);
7045 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
7046 true, NULL_TREE
, true, GSI_SAME_STMT
);
7047 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
7048 true, NULL_TREE
, true, GSI_SAME_STMT
);
7049 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
7050 true, NULL_TREE
, true, GSI_SAME_STMT
);
7052 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
7053 true, NULL_TREE
, true, GSI_SAME_STMT
);
7055 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
7056 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
7057 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
7058 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
7059 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
7060 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7061 fold_build1 (NEGATE_EXPR
, itype
, t
),
7062 fold_build1 (NEGATE_EXPR
, itype
, step
));
7064 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
7065 t
= fold_convert (itype
, t
);
7066 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7067 true, GSI_SAME_STMT
);
7069 trip_var
= create_tmp_reg (itype
, ".trip");
7070 if (gimple_in_ssa_p (cfun
))
7072 trip_init
= make_ssa_name (trip_var
, NULL
);
7073 trip_main
= make_ssa_name (trip_var
, NULL
);
7074 trip_back
= make_ssa_name (trip_var
, NULL
);
7078 trip_init
= trip_var
;
7079 trip_main
= trip_var
;
7080 trip_back
= trip_var
;
7083 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
7084 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7086 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
7087 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7088 if (POINTER_TYPE_P (type
))
7089 t
= fold_build_pointer_plus (n1
, t
);
7091 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7092 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7093 true, GSI_SAME_STMT
);
7095 /* Remove the GIMPLE_OMP_FOR. */
7096 gsi_remove (&gsi
, true);
7098 /* Iteration space partitioning goes in ITER_PART_BB. */
7099 gsi
= gsi_last_bb (iter_part_bb
);
7101 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
7102 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
7103 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
7104 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7105 false, GSI_CONTINUE_LINKING
);
7107 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
7108 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
7109 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7110 false, GSI_CONTINUE_LINKING
);
7112 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
7113 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
7115 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7116 gsi
= gsi_start_bb (seq_start_bb
);
7118 tree startvar
= fd
->loop
.v
;
7119 tree endvar
= NULL_TREE
;
7121 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7123 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
7124 ? gimple_omp_parallel_clauses (inner_stmt
)
7125 : gimple_omp_for_clauses (inner_stmt
);
7126 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7127 gcc_assert (innerc
);
7128 startvar
= OMP_CLAUSE_DECL (innerc
);
7129 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7130 OMP_CLAUSE__LOOPTEMP_
);
7131 gcc_assert (innerc
);
7132 endvar
= OMP_CLAUSE_DECL (innerc
);
7135 t
= fold_convert (itype
, s0
);
7136 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7137 if (POINTER_TYPE_P (type
))
7138 t
= fold_build_pointer_plus (n1
, t
);
7140 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7141 t
= fold_convert (TREE_TYPE (startvar
), t
);
7142 t
= force_gimple_operand_gsi (&gsi
, t
,
7144 && TREE_ADDRESSABLE (startvar
),
7145 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7146 stmt
= gimple_build_assign (startvar
, t
);
7147 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7149 t
= fold_convert (itype
, e0
);
7150 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7151 if (POINTER_TYPE_P (type
))
7152 t
= fold_build_pointer_plus (n1
, t
);
7154 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7155 t
= fold_convert (TREE_TYPE (startvar
), t
);
7156 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7157 false, GSI_CONTINUE_LINKING
);
7160 stmt
= gimple_build_assign (endvar
, e
);
7161 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7162 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
7163 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
7165 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
7167 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7169 if (fd
->collapse
> 1)
7170 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
7174 /* The code controlling the sequential loop goes in CONT_BB,
7175 replacing the GIMPLE_OMP_CONTINUE. */
7176 gsi
= gsi_last_bb (cont_bb
);
7177 stmt
= gsi_stmt (gsi
);
7178 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7179 vmain
= gimple_omp_continue_control_use (stmt
);
7180 vback
= gimple_omp_continue_control_def (stmt
);
7182 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7184 if (POINTER_TYPE_P (type
))
7185 t
= fold_build_pointer_plus (vmain
, step
);
7187 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
7188 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
7189 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7190 true, GSI_SAME_STMT
);
7191 stmt
= gimple_build_assign (vback
, t
);
7192 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7194 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
7195 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
7197 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
7200 /* Remove GIMPLE_OMP_CONTINUE. */
7201 gsi_remove (&gsi
, true);
7203 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
7204 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
7206 /* Trip update code goes into TRIP_UPDATE_BB. */
7207 gsi
= gsi_start_bb (trip_update_bb
);
7209 t
= build_int_cst (itype
, 1);
7210 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
7211 stmt
= gimple_build_assign (trip_back
, t
);
7212 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7215 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7216 gsi
= gsi_last_bb (exit_bb
);
7217 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
7219 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
7220 != GF_OMP_FOR_KIND_OACC_LOOP
);
7222 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
7223 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
7225 gsi_remove (&gsi
, true);
7227 /* Connect the new blocks. */
7228 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
7229 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
7233 se
= find_edge (cont_bb
, body_bb
);
7234 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7236 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
7237 != GF_OMP_FOR_KIND_OACC_LOOP
);
7242 else if (fd
->collapse
> 1)
7245 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
7248 se
->flags
= EDGE_TRUE_VALUE
;
7249 find_edge (cont_bb
, trip_update_bb
)->flags
7250 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
7252 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
7255 if (gimple_in_ssa_p (cfun
))
7257 gimple_stmt_iterator psi
;
7263 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
7265 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7266 remove arguments of the phi nodes in fin_bb. We need to create
7267 appropriate phi nodes in iter_part_bb instead. */
7268 se
= single_pred_edge (fin_bb
);
7269 re
= single_succ_edge (trip_update_bb
);
7270 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
7271 ene
= single_succ_edge (entry_bb
);
7273 psi
= gsi_start_phis (fin_bb
);
7274 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
7275 gsi_next (&psi
), ++i
)
7278 source_location locus
;
7280 phi
= gsi_stmt (psi
);
7281 t
= gimple_phi_result (phi
);
7282 gcc_assert (t
== redirect_edge_var_map_result (vm
));
7283 nphi
= create_phi_node (t
, iter_part_bb
);
7285 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
7286 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
7288 /* A special case -- fd->loop.v is not yet computed in
7289 iter_part_bb, we need to use vextra instead. */
7290 if (t
== fd
->loop
.v
)
7292 add_phi_arg (nphi
, t
, ene
, locus
);
7293 locus
= redirect_edge_var_map_location (vm
);
7294 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
7296 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
7297 redirect_edge_var_map_clear (re
);
7300 psi
= gsi_start_phis (fin_bb
);
7301 if (gsi_end_p (psi
))
7303 remove_phi_node (&psi
, false);
7306 /* Make phi node for trip. */
7307 phi
= create_phi_node (trip_main
, iter_part_bb
);
7308 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
7310 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
7315 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
7316 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
7317 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
7318 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
7319 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
7320 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
7321 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
7322 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
7323 recompute_dominator (CDI_DOMINATORS
, body_bb
));
7327 struct loop
*trip_loop
= alloc_loop ();
7328 trip_loop
->header
= iter_part_bb
;
7329 trip_loop
->latch
= trip_update_bb
;
7330 add_loop (trip_loop
, iter_part_bb
->loop_father
);
7332 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7334 struct loop
*loop
= alloc_loop ();
7335 loop
->header
= body_bb
;
7336 if (collapse_bb
== NULL
)
7337 loop
->latch
= cont_bb
;
7338 add_loop (loop
, trip_loop
);
7343 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7345 for (V = N1; V cond N2; V += STEP) BODY;
7347 where COND is "<" or ">" or "!=", we generate pseudocode
7349 for (ind_var = low; ind_var < high; ind_var++)
7351 V = n1 + (ind_var * STEP)
7356 In the above pseudocode, low and high are function parameters of the
7357 child function. In the function below, we are inserting a temp.
7358 variable that will be making a call to two OMP functions that will not be
7359 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7360 with _Cilk_for). These functions are replaced with low and high
7361 by the function that handles taskreg. */
7365 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
7367 bool broken_loop
= region
->cont
== NULL
;
7368 basic_block entry_bb
= region
->entry
;
7369 basic_block cont_bb
= region
->cont
;
7371 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7372 gcc_assert (broken_loop
7373 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7374 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7375 basic_block l1_bb
, l2_bb
;
7379 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7380 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7381 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7382 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7386 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7387 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7388 l2_bb
= single_succ (l1_bb
);
7390 basic_block exit_bb
= region
->exit
;
7391 basic_block l2_dom_bb
= NULL
;
7393 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
7395 /* Below statements until the "tree high_val = ..." are pseudo statements
7396 used to pass information to be used by expand_omp_taskreg.
7397 low_val and high_val will be replaced by the __low and __high
7398 parameter from the child function.
7400 The call_exprs part is a place-holder, it is mainly used
7401 to distinctly identify to the top-level part that this is
7402 where we should put low and high (reasoning given in header
7406 = gimple_omp_parallel_child_fn (last_stmt (region
->outer
->entry
));
7407 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
7408 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
7410 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
7412 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
7415 gcc_assert (low_val
&& high_val
);
7417 tree type
= TREE_TYPE (low_val
);
7418 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
7419 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7421 /* Not needed in SSA form right now. */
7422 gcc_assert (!gimple_in_ssa_p (cfun
));
7423 if (l2_dom_bb
== NULL
)
7429 gimple stmt
= gimple_build_assign (ind_var
, n1
);
7431 /* Replace the GIMPLE_OMP_FOR statement. */
7432 gsi_replace (&gsi
, stmt
, true);
7436 /* Code to control the increment goes in the CONT_BB. */
7437 gsi
= gsi_last_bb (cont_bb
);
7438 stmt
= gsi_stmt (gsi
);
7439 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7440 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, ind_var
, ind_var
,
7441 build_one_cst (type
));
7443 /* Replace GIMPLE_OMP_CONTINUE. */
7444 gsi_replace (&gsi
, stmt
, true);
7447 /* Emit the condition in L1_BB. */
7448 gsi
= gsi_after_labels (l1_bb
);
7449 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
7450 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
7452 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
7453 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7454 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7456 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7457 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
7458 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
7459 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7461 /* The condition is always '<' since the runtime will fill in the low
7463 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
7464 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7466 /* Remove GIMPLE_OMP_RETURN. */
7467 gsi
= gsi_last_bb (exit_bb
);
7468 gsi_remove (&gsi
, true);
7470 /* Connect the new blocks. */
7471 remove_edge (FALLTHRU_EDGE (entry_bb
));
7476 remove_edge (BRANCH_EDGE (entry_bb
));
7477 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7479 e
= BRANCH_EDGE (l1_bb
);
7480 ne
= FALLTHRU_EDGE (l1_bb
);
7481 e
->flags
= EDGE_TRUE_VALUE
;
7485 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7487 ne
= single_succ_edge (l1_bb
);
7488 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7491 ne
->flags
= EDGE_FALSE_VALUE
;
7492 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7493 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7495 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7496 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7497 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7501 struct loop
*loop
= alloc_loop ();
7502 loop
->header
= l1_bb
;
7503 loop
->latch
= cont_bb
;
7504 add_loop (loop
, l1_bb
->loop_father
);
7505 loop
->safelen
= INT_MAX
;
7508 /* Pick the correct library function based on the precision of the
7509 induction variable type. */
7510 tree lib_fun
= NULL_TREE
;
7511 if (TYPE_PRECISION (type
) == 32)
7512 lib_fun
= cilk_for_32_fndecl
;
7513 else if (TYPE_PRECISION (type
) == 64)
7514 lib_fun
= cilk_for_64_fndecl
;
7518 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7520 /* WS_ARGS contains the library function flavor to call:
7521 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7522 user-defined grain value. If the user does not define one, then zero
7523 is passed in by the parser. */
7524 vec_alloc (region
->ws_args
, 2);
7525 region
->ws_args
->quick_push (lib_fun
);
7526 region
->ws_args
->quick_push (fd
->chunk_size
);
7529 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7530 loop. Given parameters:
7532 for (V = N1; V cond N2; V += STEP) BODY;
7534 where COND is "<" or ">", we generate pseudocode
7542 if (V cond N2) goto L0; else goto L2;
7545 For collapsed loops, given parameters:
7547 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7548 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7549 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7552 we generate pseudocode
7558 count3 = (adj + N32 - N31) / STEP3;
7563 count2 = (adj + N22 - N21) / STEP2;
7568 count1 = (adj + N12 - N11) / STEP1;
7569 count = count1 * count2 * count3;
7579 V2 += (V3 cond3 N32) ? 0 : STEP2;
7580 V3 = (V3 cond3 N32) ? V3 : N31;
7581 V1 += (V2 cond2 N22) ? 0 : STEP1;
7582 V2 = (V2 cond2 N22) ? V2 : N21;
7584 if (V < count) goto L0; else goto L2;
7590 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7593 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7594 gimple_stmt_iterator gsi
;
7596 bool broken_loop
= region
->cont
== NULL
;
7598 tree
*counts
= NULL
;
7600 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7601 OMP_CLAUSE_SAFELEN
);
7602 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7603 OMP_CLAUSE__SIMDUID_
);
7606 type
= TREE_TYPE (fd
->loop
.v
);
7607 entry_bb
= region
->entry
;
7608 cont_bb
= region
->cont
;
7609 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7610 gcc_assert (broken_loop
7611 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7612 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7615 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7616 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7617 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7618 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7622 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7623 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7624 l2_bb
= single_succ (l1_bb
);
7626 exit_bb
= region
->exit
;
7629 gsi
= gsi_last_bb (entry_bb
);
7631 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7632 /* Not needed in SSA form right now. */
7633 gcc_assert (!gimple_in_ssa_p (cfun
));
7634 if (fd
->collapse
> 1)
7636 int first_zero_iter
= -1;
7637 basic_block zero_iter_bb
= l2_bb
;
7639 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7640 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7641 zero_iter_bb
, first_zero_iter
,
7644 if (l2_dom_bb
== NULL
)
7649 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7651 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7652 OMP_CLAUSE__LOOPTEMP_
);
7653 gcc_assert (innerc
);
7654 n1
= OMP_CLAUSE_DECL (innerc
);
7655 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7656 OMP_CLAUSE__LOOPTEMP_
);
7657 gcc_assert (innerc
);
7658 n2
= OMP_CLAUSE_DECL (innerc
);
7659 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7660 fold_convert (type
, n1
));
7661 if (fd
->collapse
> 1)
7664 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7670 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7671 fold_convert (type
, fd
->loop
.n1
));
7672 if (fd
->collapse
> 1)
7673 for (i
= 0; i
< fd
->collapse
; i
++)
7675 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7676 if (POINTER_TYPE_P (itype
))
7677 itype
= signed_type_for (itype
);
7678 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7679 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7683 /* Remove the GIMPLE_OMP_FOR statement. */
7684 gsi_remove (&gsi
, true);
7688 /* Code to control the increment goes in the CONT_BB. */
7689 gsi
= gsi_last_bb (cont_bb
);
7690 stmt
= gsi_stmt (gsi
);
7691 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7693 if (POINTER_TYPE_P (type
))
7694 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7696 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7697 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7699 if (fd
->collapse
> 1)
7701 i
= fd
->collapse
- 1;
7702 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7704 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7705 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7709 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7711 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7714 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7716 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7718 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7719 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7720 if (POINTER_TYPE_P (itype2
))
7721 itype2
= signed_type_for (itype2
);
7722 t
= build3 (COND_EXPR
, itype2
,
7723 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7725 fold_convert (itype
, fd
->loops
[i
].n2
)),
7726 build_int_cst (itype2
, 0),
7727 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7728 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7729 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7731 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7732 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7734 t
= build3 (COND_EXPR
, itype
,
7735 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7737 fold_convert (itype
, fd
->loops
[i
].n2
)),
7739 fold_convert (itype
, fd
->loops
[i
].n1
));
7740 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7744 /* Remove GIMPLE_OMP_CONTINUE. */
7745 gsi_remove (&gsi
, true);
7748 /* Emit the condition in L1_BB. */
7749 gsi
= gsi_start_bb (l1_bb
);
7751 t
= fold_convert (type
, n2
);
7752 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7753 false, GSI_CONTINUE_LINKING
);
7754 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7755 stmt
= gimple_build_cond_empty (t
);
7756 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7757 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
7759 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
7762 gsi
= gsi_for_stmt (stmt
);
7763 gimple_regimplify_operands (stmt
, &gsi
);
7766 /* Remove GIMPLE_OMP_RETURN. */
7767 gsi
= gsi_last_bb (exit_bb
);
7768 gsi_remove (&gsi
, true);
7770 /* Connect the new blocks. */
7771 remove_edge (FALLTHRU_EDGE (entry_bb
));
7775 remove_edge (BRANCH_EDGE (entry_bb
));
7776 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7778 e
= BRANCH_EDGE (l1_bb
);
7779 ne
= FALLTHRU_EDGE (l1_bb
);
7780 e
->flags
= EDGE_TRUE_VALUE
;
7784 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7786 ne
= single_succ_edge (l1_bb
);
7787 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7790 ne
->flags
= EDGE_FALSE_VALUE
;
7791 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7792 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7794 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7795 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7796 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7800 struct loop
*loop
= alloc_loop ();
7801 loop
->header
= l1_bb
;
7802 loop
->latch
= cont_bb
;
7803 add_loop (loop
, l1_bb
->loop_father
);
7804 if (safelen
== NULL_TREE
)
7805 loop
->safelen
= INT_MAX
;
7808 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7809 if (TREE_CODE (safelen
) != INTEGER_CST
)
7811 else if (!tree_fits_uhwi_p (safelen
)
7812 || tree_to_uhwi (safelen
) > INT_MAX
)
7813 loop
->safelen
= INT_MAX
;
7815 loop
->safelen
= tree_to_uhwi (safelen
);
7816 if (loop
->safelen
== 1)
7821 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7822 cfun
->has_simduid_loops
= true;
7824 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7826 if ((flag_tree_loop_vectorize
7827 || (!global_options_set
.x_flag_tree_loop_vectorize
7828 && !global_options_set
.x_flag_tree_vectorize
))
7829 && flag_tree_loop_optimize
7830 && loop
->safelen
> 1)
7832 loop
->force_vectorize
= true;
7833 cfun
->has_force_vectorize_loops
= true;
7839 /* Expand the OpenMP loop defined by REGION. */
7842 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7844 struct omp_for_data fd
;
7845 struct omp_for_data_loop
*loops
;
7848 = (struct omp_for_data_loop
*)
7849 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7850 * sizeof (struct omp_for_data_loop
));
7851 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
7852 region
->sched_kind
= fd
.sched_kind
;
7854 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7855 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7856 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7859 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7860 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7861 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7864 /* If there isn't a continue then this is a degerate case where
7865 the introduction of abnormal edges during lowering will prevent
7866 original loops from being detected. Fix that up. */
7867 loops_state_set (LOOPS_NEED_FIXUP
);
7869 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7870 expand_omp_simd (region
, &fd
);
7871 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7872 expand_cilk_for (region
, &fd
);
7873 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7874 && !fd
.have_ordered
)
7876 if (fd
.chunk_size
== NULL
)
7877 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7879 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7883 int fn_index
, start_ix
, next_ix
;
7885 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7886 == GF_OMP_FOR_KIND_FOR
);
7887 if (fd
.chunk_size
== NULL
7888 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7889 fd
.chunk_size
= integer_zero_node
;
7890 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7891 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7892 ? 3 : fd
.sched_kind
;
7893 fn_index
+= fd
.have_ordered
* 4;
7894 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7895 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7896 if (fd
.iter_type
== long_long_unsigned_type_node
)
7898 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7899 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7900 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7901 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7903 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7904 (enum built_in_function
) next_ix
, inner_stmt
);
7907 if (gimple_in_ssa_p (cfun
))
7908 update_ssa (TODO_update_ssa_only_virtuals
);
7912 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7914 v = GOMP_sections_start (n);
7931 v = GOMP_sections_next ();
7936 If this is a combined parallel sections, replace the call to
7937 GOMP_sections_start with call to GOMP_sections_next. */
7940 expand_omp_sections (struct omp_region
*region
)
7942 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7944 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7945 gimple_stmt_iterator si
, switch_si
;
7946 gimple sections_stmt
, stmt
, cont
;
7949 struct omp_region
*inner
;
7951 bool exit_reachable
= region
->cont
!= NULL
;
7953 gcc_assert (region
->exit
!= NULL
);
7954 entry_bb
= region
->entry
;
7955 l0_bb
= single_succ (entry_bb
);
7956 l1_bb
= region
->cont
;
7957 l2_bb
= region
->exit
;
7958 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7959 l2
= gimple_block_label (l2_bb
);
7962 /* This can happen if there are reductions. */
7963 len
= EDGE_COUNT (l0_bb
->succs
);
7964 gcc_assert (len
> 0);
7965 e
= EDGE_SUCC (l0_bb
, len
- 1);
7966 si
= gsi_last_bb (e
->dest
);
7969 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7970 l2
= gimple_block_label (e
->dest
);
7972 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7974 si
= gsi_last_bb (e
->dest
);
7976 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7978 l2
= gimple_block_label (e
->dest
);
7984 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7986 default_bb
= create_empty_bb (l0_bb
);
7988 /* We will build a switch() with enough cases for all the
7989 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7990 and a default case to abort if something goes wrong. */
7991 len
= EDGE_COUNT (l0_bb
->succs
);
7993 /* Use vec::quick_push on label_vec throughout, since we know the size
7995 auto_vec
<tree
> label_vec (len
);
7997 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7998 GIMPLE_OMP_SECTIONS statement. */
7999 si
= gsi_last_bb (entry_bb
);
8000 sections_stmt
= gsi_stmt (si
);
8001 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
8002 vin
= gimple_omp_sections_control (sections_stmt
);
8003 if (!is_combined_parallel (region
))
8005 /* If we are not inside a combined parallel+sections region,
8006 call GOMP_sections_start. */
8007 t
= build_int_cst (unsigned_type_node
, len
- 1);
8008 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
8009 stmt
= gimple_build_call (u
, 1, t
);
8013 /* Otherwise, call GOMP_sections_next. */
8014 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8015 stmt
= gimple_build_call (u
, 0);
8017 gimple_call_set_lhs (stmt
, vin
);
8018 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8019 gsi_remove (&si
, true);
8021 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8023 switch_si
= gsi_last_bb (l0_bb
);
8024 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
8027 cont
= last_stmt (l1_bb
);
8028 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
8029 vmain
= gimple_omp_continue_control_use (cont
);
8030 vnext
= gimple_omp_continue_control_def (cont
);
8038 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
8039 label_vec
.quick_push (t
);
8042 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8043 for (inner
= region
->inner
, casei
= 1;
8045 inner
= inner
->next
, i
++, casei
++)
8047 basic_block s_entry_bb
, s_exit_bb
;
8049 /* Skip optional reduction region. */
8050 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
8057 s_entry_bb
= inner
->entry
;
8058 s_exit_bb
= inner
->exit
;
8060 t
= gimple_block_label (s_entry_bb
);
8061 u
= build_int_cst (unsigned_type_node
, casei
);
8062 u
= build_case_label (u
, NULL
, t
);
8063 label_vec
.quick_push (u
);
8065 si
= gsi_last_bb (s_entry_bb
);
8066 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
8067 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
8068 gsi_remove (&si
, true);
8069 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
8071 if (s_exit_bb
== NULL
)
8074 si
= gsi_last_bb (s_exit_bb
);
8075 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8076 gsi_remove (&si
, true);
8078 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
8081 /* Error handling code goes in DEFAULT_BB. */
8082 t
= gimple_block_label (default_bb
);
8083 u
= build_case_label (NULL
, NULL
, t
);
8084 make_edge (l0_bb
, default_bb
, 0);
8085 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
8087 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
8088 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
8089 gsi_remove (&switch_si
, true);
8091 si
= gsi_start_bb (default_bb
);
8092 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
8093 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
8099 /* Code to get the next section goes in L1_BB. */
8100 si
= gsi_last_bb (l1_bb
);
8101 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
8103 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8104 stmt
= gimple_build_call (bfn_decl
, 0);
8105 gimple_call_set_lhs (stmt
, vnext
);
8106 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8107 gsi_remove (&si
, true);
8109 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
8112 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8113 si
= gsi_last_bb (l2_bb
);
8114 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
8115 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
8116 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
8117 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
8119 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
8120 stmt
= gimple_build_call (t
, 0);
8121 if (gimple_omp_return_lhs (gsi_stmt (si
)))
8122 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
8123 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8124 gsi_remove (&si
, true);
8126 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
8130 /* Expand code for an OpenMP single directive. We've already expanded
8131 much of the code, here we simply place the GOMP_barrier call. */
8134 expand_omp_single (struct omp_region
*region
)
8136 basic_block entry_bb
, exit_bb
;
8137 gimple_stmt_iterator si
;
8139 entry_bb
= region
->entry
;
8140 exit_bb
= region
->exit
;
8142 si
= gsi_last_bb (entry_bb
);
8143 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
8144 gsi_remove (&si
, true);
8145 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8147 si
= gsi_last_bb (exit_bb
);
8148 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
8150 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
8151 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
8153 gsi_remove (&si
, true);
8154 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8158 /* Generic expansion for OpenMP synchronization directives: master,
8159 ordered and critical. All we need to do here is remove the entry
8160 and exit markers for REGION. */
8163 expand_omp_synch (struct omp_region
*region
)
8165 basic_block entry_bb
, exit_bb
;
8166 gimple_stmt_iterator si
;
8168 entry_bb
= region
->entry
;
8169 exit_bb
= region
->exit
;
8171 si
= gsi_last_bb (entry_bb
);
8172 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
8173 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
8174 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
8175 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
8176 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
8177 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
8178 gsi_remove (&si
, true);
8179 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8183 si
= gsi_last_bb (exit_bb
);
8184 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8185 gsi_remove (&si
, true);
8186 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8190 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8191 operation as a normal volatile load. */
8194 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
8195 tree loaded_val
, int index
)
8197 enum built_in_function tmpbase
;
8198 gimple_stmt_iterator gsi
;
8199 basic_block store_bb
;
8202 tree decl
, call
, type
, itype
;
8204 gsi
= gsi_last_bb (load_bb
);
8205 stmt
= gsi_stmt (gsi
);
8206 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8207 loc
= gimple_location (stmt
);
8209 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8210 is smaller than word size, then expand_atomic_load assumes that the load
8211 is atomic. We could avoid the builtin entirely in this case. */
8213 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8214 decl
= builtin_decl_explicit (tmpbase
);
8215 if (decl
== NULL_TREE
)
8218 type
= TREE_TYPE (loaded_val
);
8219 itype
= TREE_TYPE (TREE_TYPE (decl
));
8221 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
8222 build_int_cst (NULL
,
8223 gimple_omp_atomic_seq_cst_p (stmt
)
8225 : MEMMODEL_RELAXED
));
8226 if (!useless_type_conversion_p (type
, itype
))
8227 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8228 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8230 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8231 gsi_remove (&gsi
, true);
8233 store_bb
= single_succ (load_bb
);
8234 gsi
= gsi_last_bb (store_bb
);
8235 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8236 gsi_remove (&gsi
, true);
8238 if (gimple_in_ssa_p (cfun
))
8239 update_ssa (TODO_update_ssa_no_phi
);
8244 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8245 operation as a normal volatile store. */
8248 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
8249 tree loaded_val
, tree stored_val
, int index
)
8251 enum built_in_function tmpbase
;
8252 gimple_stmt_iterator gsi
;
8253 basic_block store_bb
= single_succ (load_bb
);
8256 tree decl
, call
, type
, itype
;
8260 gsi
= gsi_last_bb (load_bb
);
8261 stmt
= gsi_stmt (gsi
);
8262 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8264 /* If the load value is needed, then this isn't a store but an exchange. */
8265 exchange
= gimple_omp_atomic_need_value_p (stmt
);
8267 gsi
= gsi_last_bb (store_bb
);
8268 stmt
= gsi_stmt (gsi
);
8269 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
8270 loc
= gimple_location (stmt
);
8272 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8273 is smaller than word size, then expand_atomic_store assumes that the store
8274 is atomic. We could avoid the builtin entirely in this case. */
8276 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
8277 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
8278 decl
= builtin_decl_explicit (tmpbase
);
8279 if (decl
== NULL_TREE
)
8282 type
= TREE_TYPE (stored_val
);
8284 /* Dig out the type of the function's second argument. */
8285 itype
= TREE_TYPE (decl
);
8286 itype
= TYPE_ARG_TYPES (itype
);
8287 itype
= TREE_CHAIN (itype
);
8288 itype
= TREE_VALUE (itype
);
8289 imode
= TYPE_MODE (itype
);
8291 if (exchange
&& !can_atomic_exchange_p (imode
, true))
8294 if (!useless_type_conversion_p (itype
, type
))
8295 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
8296 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
8297 build_int_cst (NULL
,
8298 gimple_omp_atomic_seq_cst_p (stmt
)
8300 : MEMMODEL_RELAXED
));
8303 if (!useless_type_conversion_p (type
, itype
))
8304 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8305 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8308 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8309 gsi_remove (&gsi
, true);
8311 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8312 gsi
= gsi_last_bb (load_bb
);
8313 gsi_remove (&gsi
, true);
8315 if (gimple_in_ssa_p (cfun
))
8316 update_ssa (TODO_update_ssa_no_phi
);
8321 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8322 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8323 size of the data type, and thus usable to find the index of the builtin
8324 decl. Returns false if the expression is not of the proper form. */
8327 expand_omp_atomic_fetch_op (basic_block load_bb
,
8328 tree addr
, tree loaded_val
,
8329 tree stored_val
, int index
)
8331 enum built_in_function oldbase
, newbase
, tmpbase
;
8332 tree decl
, itype
, call
;
8334 basic_block store_bb
= single_succ (load_bb
);
8335 gimple_stmt_iterator gsi
;
8338 enum tree_code code
;
8339 bool need_old
, need_new
;
8343 /* We expect to find the following sequences:
8346 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8349 val = tmp OP something; (or: something OP tmp)
8350 GIMPLE_OMP_STORE (val)
8352 ???FIXME: Allow a more flexible sequence.
8353 Perhaps use data flow to pick the statements.
8357 gsi
= gsi_after_labels (store_bb
);
8358 stmt
= gsi_stmt (gsi
);
8359 loc
= gimple_location (stmt
);
8360 if (!is_gimple_assign (stmt
))
8363 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
8365 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
8366 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
8367 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
8368 gcc_checking_assert (!need_old
|| !need_new
);
8370 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
8373 /* Check for one of the supported fetch-op operations. */
8374 code
= gimple_assign_rhs_code (stmt
);
8378 case POINTER_PLUS_EXPR
:
8379 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
8380 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
8383 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
8384 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
8387 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
8388 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
8391 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
8392 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
8395 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
8396 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
8402 /* Make sure the expression is of the proper form. */
8403 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
8404 rhs
= gimple_assign_rhs2 (stmt
);
8405 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
8406 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
8407 rhs
= gimple_assign_rhs1 (stmt
);
8411 tmpbase
= ((enum built_in_function
)
8412 ((need_new
? newbase
: oldbase
) + index
+ 1));
8413 decl
= builtin_decl_explicit (tmpbase
);
8414 if (decl
== NULL_TREE
)
8416 itype
= TREE_TYPE (TREE_TYPE (decl
));
8417 imode
= TYPE_MODE (itype
);
8419 /* We could test all of the various optabs involved, but the fact of the
8420 matter is that (with the exception of i486 vs i586 and xadd) all targets
8421 that support any atomic operaton optab also implements compare-and-swap.
8422 Let optabs.c take care of expanding any compare-and-swap loop. */
8423 if (!can_compare_and_swap_p (imode
, true))
8426 gsi
= gsi_last_bb (load_bb
);
8427 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8429 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8430 It only requires that the operation happen atomically. Thus we can
8431 use the RELAXED memory model. */
8432 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
8433 fold_convert_loc (loc
, itype
, rhs
),
8434 build_int_cst (NULL
,
8435 seq_cst
? MEMMODEL_SEQ_CST
8436 : MEMMODEL_RELAXED
));
8438 if (need_old
|| need_new
)
8440 lhs
= need_old
? loaded_val
: stored_val
;
8441 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
8442 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
8445 call
= fold_convert_loc (loc
, void_type_node
, call
);
8446 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8447 gsi_remove (&gsi
, true);
8449 gsi
= gsi_last_bb (store_bb
);
8450 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8451 gsi_remove (&gsi
, true);
8452 gsi
= gsi_last_bb (store_bb
);
8453 gsi_remove (&gsi
, true);
8455 if (gimple_in_ssa_p (cfun
))
8456 update_ssa (TODO_update_ssa_no_phi
);
8461 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8465 newval = rhs; // with oldval replacing *addr in rhs
8466 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8467 if (oldval != newval)
8470 INDEX is log2 of the size of the data type, and thus usable to find the
8471 index of the builtin decl. */
8474 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
8475 tree addr
, tree loaded_val
, tree stored_val
,
8478 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
8479 tree type
, itype
, cmpxchg
, iaddr
;
8480 gimple_stmt_iterator si
;
8481 basic_block loop_header
= single_succ (load_bb
);
8484 enum built_in_function fncode
;
8486 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8487 order to use the RELAXED memory model effectively. */
8488 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8490 cmpxchg
= builtin_decl_explicit (fncode
);
8491 if (cmpxchg
== NULL_TREE
)
8493 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8494 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
8496 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
8499 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8500 si
= gsi_last_bb (load_bb
);
8501 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8503 /* For floating-point values, we'll need to view-convert them to integers
8504 so that we can perform the atomic compare and swap. Simplify the
8505 following code by always setting up the "i"ntegral variables. */
8506 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
8510 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
8513 = force_gimple_operand_gsi (&si
,
8514 fold_convert (TREE_TYPE (iaddr
), addr
),
8515 false, NULL_TREE
, true, GSI_SAME_STMT
);
8516 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
8517 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8518 loadedi
= create_tmp_var (itype
, NULL
);
8519 if (gimple_in_ssa_p (cfun
))
8520 loadedi
= make_ssa_name (loadedi
, NULL
);
8525 loadedi
= loaded_val
;
8528 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8529 tree loaddecl
= builtin_decl_explicit (fncode
);
8532 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8533 build_call_expr (loaddecl
, 2, iaddr
,
8534 build_int_cst (NULL_TREE
,
8535 MEMMODEL_RELAXED
)));
8537 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8538 build_int_cst (TREE_TYPE (iaddr
), 0));
8541 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8544 /* Move the value to the LOADEDI temporary. */
8545 if (gimple_in_ssa_p (cfun
))
8547 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8548 phi
= create_phi_node (loadedi
, loop_header
);
8549 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8553 gsi_insert_before (&si
,
8554 gimple_build_assign (loadedi
, initial
),
8556 if (loadedi
!= loaded_val
)
8558 gimple_stmt_iterator gsi2
;
8561 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8562 gsi2
= gsi_start_bb (loop_header
);
8563 if (gimple_in_ssa_p (cfun
))
8566 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8567 true, GSI_SAME_STMT
);
8568 stmt
= gimple_build_assign (loaded_val
, x
);
8569 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8573 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8574 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8575 true, GSI_SAME_STMT
);
8578 gsi_remove (&si
, true);
8580 si
= gsi_last_bb (store_bb
);
8581 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8584 storedi
= stored_val
;
8587 force_gimple_operand_gsi (&si
,
8588 build1 (VIEW_CONVERT_EXPR
, itype
,
8589 stored_val
), true, NULL_TREE
, true,
8592 /* Build the compare&swap statement. */
8593 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8594 new_storedi
= force_gimple_operand_gsi (&si
,
8595 fold_convert (TREE_TYPE (loadedi
),
8598 true, GSI_SAME_STMT
);
8600 if (gimple_in_ssa_p (cfun
))
8604 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
8605 stmt
= gimple_build_assign (old_vali
, loadedi
);
8606 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8608 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8609 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8612 /* Note that we always perform the comparison as an integer, even for
8613 floating point. This allows the atomic operation to properly
8614 succeed even with NaNs and -0.0. */
8615 stmt
= gimple_build_cond_empty
8616 (build2 (NE_EXPR
, boolean_type_node
,
8617 new_storedi
, old_vali
));
8618 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8621 e
= single_succ_edge (store_bb
);
8622 e
->flags
&= ~EDGE_FALLTHRU
;
8623 e
->flags
|= EDGE_FALSE_VALUE
;
8625 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8627 /* Copy the new value to loadedi (we already did that before the condition
8628 if we are not in SSA). */
8629 if (gimple_in_ssa_p (cfun
))
8631 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8632 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8635 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8636 gsi_remove (&si
, true);
8638 struct loop
*loop
= alloc_loop ();
8639 loop
->header
= loop_header
;
8640 loop
->latch
= store_bb
;
8641 add_loop (loop
, loop_header
->loop_father
);
8643 if (gimple_in_ssa_p (cfun
))
8644 update_ssa (TODO_update_ssa_no_phi
);
8649 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8651 GOMP_atomic_start ();
8655 The result is not globally atomic, but works so long as all parallel
8656 references are within #pragma omp atomic directives. According to
8657 responses received from omp@openmp.org, appears to be within spec.
8658 Which makes sense, since that's how several other compilers handle
8659 this situation as well.
8660 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8661 expanding. STORED_VAL is the operand of the matching
8662 GIMPLE_OMP_ATOMIC_STORE.
8665 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8669 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8674 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8675 tree addr
, tree loaded_val
, tree stored_val
)
8677 gimple_stmt_iterator si
;
8681 si
= gsi_last_bb (load_bb
);
8682 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8684 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8685 t
= build_call_expr (t
, 0);
8686 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8688 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8689 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8690 gsi_remove (&si
, true);
8692 si
= gsi_last_bb (store_bb
);
8693 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8695 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8697 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8699 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8700 t
= build_call_expr (t
, 0);
8701 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8702 gsi_remove (&si
, true);
8704 if (gimple_in_ssa_p (cfun
))
8705 update_ssa (TODO_update_ssa_no_phi
);
8709 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8710 using expand_omp_atomic_fetch_op. If it failed, we try to
8711 call expand_omp_atomic_pipeline, and if it fails too, the
8712 ultimate fallback is wrapping the operation in a mutex
8713 (expand_omp_atomic_mutex). REGION is the atomic region built
8714 by build_omp_regions_1(). */
8717 expand_omp_atomic (struct omp_region
*region
)
8719 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8720 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
8721 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8722 tree addr
= gimple_omp_atomic_load_rhs (load
);
8723 tree stored_val
= gimple_omp_atomic_store_val (store
);
8724 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8725 HOST_WIDE_INT index
;
8727 /* Make sure the type is one of the supported sizes. */
8728 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8729 index
= exact_log2 (index
);
8730 if (index
>= 0 && index
<= 4)
8732 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8734 /* __sync builtins require strict data alignment. */
8735 if (exact_log2 (align
) >= index
)
8738 if (loaded_val
== stored_val
8739 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8740 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8741 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8742 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8746 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8747 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8748 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8749 && store_bb
== single_succ (load_bb
)
8750 && first_stmt (store_bb
) == store
8751 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8755 /* When possible, use specialized atomic update functions. */
8756 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8757 && store_bb
== single_succ (load_bb
)
8758 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8759 loaded_val
, stored_val
, index
))
8762 /* If we don't have specialized __sync builtins, try and implement
8763 as a compare and swap loop. */
8764 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8765 loaded_val
, stored_val
, index
))
8770 /* The ultimate fallback is wrapping the operation in a mutex. */
8771 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8775 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8778 expand_omp_target (struct omp_region
*region
)
8780 basic_block entry_bb
, exit_bb
, new_bb
;
8781 struct function
*child_cfun
;
8782 tree child_fn
, block
, t
;
8783 gimple_stmt_iterator gsi
;
8784 gimple entry_stmt
, stmt
;
8786 bool offloaded
, data_region
;
8787 tree (*gimple_omp_child_fn
) (const_gimple
);
8788 tree (*gimple_omp_data_arg
) (const_gimple
);
8790 entry_stmt
= last_stmt (region
->entry
);
8791 new_bb
= region
->entry
;
8793 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
8794 data_region
= false;
8795 switch (region
->type
)
8797 case GIMPLE_OACC_KERNELS
:
8798 gimple_omp_child_fn
= gimple_oacc_kernels_child_fn
;
8799 gimple_omp_data_arg
= gimple_oacc_kernels_data_arg
;
8801 case GIMPLE_OACC_PARALLEL
:
8802 gimple_omp_child_fn
= gimple_oacc_parallel_child_fn
;
8803 gimple_omp_data_arg
= gimple_oacc_parallel_data_arg
;
8805 case GIMPLE_OMP_TARGET
:
8806 switch (gimple_omp_target_kind (entry_stmt
))
8808 case GF_OMP_TARGET_KIND_DATA
:
8809 case GF_OMP_TARGET_KIND_OACC_DATA
:
8812 case GF_OMP_TARGET_KIND_REGION
:
8813 case GF_OMP_TARGET_KIND_UPDATE
:
8814 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8815 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8821 gimple_omp_child_fn
= gimple_omp_target_child_fn
;
8822 gimple_omp_data_arg
= gimple_omp_target_data_arg
;
8828 child_fn
= NULL_TREE
;
8832 child_fn
= gimple_omp_child_fn (entry_stmt
);
8833 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8836 /* Supported by expand_omp_taskreg, but not here. */
8837 if (child_cfun
!= NULL
)
8838 gcc_assert (!child_cfun
->cfg
);
8839 gcc_assert (!gimple_in_ssa_p (cfun
));
8841 entry_bb
= region
->entry
;
8842 exit_bb
= region
->exit
;
8846 unsigned srcidx
, dstidx
, num
;
8848 /* If the offloading region needs data sent from the parent
8849 function, then the very first statement (except possible
8850 tree profile counter updates) of the offloading body
8851 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8852 &.OMP_DATA_O is passed as an argument to the child function,
8853 we need to replace it with the argument as seen by the child
8856 In most cases, this will end up being the identity assignment
8857 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8858 a function call that has been inlined, the original PARM_DECL
8859 .OMP_DATA_I may have been converted into a different local
8860 variable. In which case, we need to keep the assignment. */
8861 if (gimple_omp_data_arg (entry_stmt
))
8863 basic_block entry_succ_bb
= single_succ (entry_bb
);
8864 gimple_stmt_iterator gsi
;
8866 gimple tgtcopy_stmt
= NULL
;
8867 tree sender
= TREE_VEC_ELT (gimple_omp_data_arg (entry_stmt
), 0);
8869 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8871 gcc_assert (!gsi_end_p (gsi
));
8872 stmt
= gsi_stmt (gsi
);
8873 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8876 if (gimple_num_ops (stmt
) == 2)
8878 tree arg
= gimple_assign_rhs1 (stmt
);
8880 /* We're ignoring the subcode because we're
8881 effectively doing a STRIP_NOPS. */
8883 if (TREE_CODE (arg
) == ADDR_EXPR
8884 && TREE_OPERAND (arg
, 0) == sender
)
8886 tgtcopy_stmt
= stmt
;
8892 gcc_assert (tgtcopy_stmt
!= NULL
);
8893 arg
= DECL_ARGUMENTS (child_fn
);
8895 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8896 gsi_remove (&gsi
, true);
8899 /* Declare local variables needed in CHILD_CFUN. */
8900 block
= DECL_INITIAL (child_fn
);
8901 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8902 /* The gimplifier could record temporaries in the offloading block
8903 rather than in containing function's local_decls chain,
8904 which would mean cgraph missed finalizing them. Do it now. */
8905 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8906 if (TREE_CODE (t
) == VAR_DECL
8908 && !DECL_EXTERNAL (t
))
8909 varpool_node::finalize_decl (t
);
8910 DECL_SAVED_TREE (child_fn
) = NULL
;
8911 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8912 gimple_set_body (child_fn
, NULL
);
8913 TREE_USED (block
) = 1;
8915 /* Reset DECL_CONTEXT on function arguments. */
8916 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8917 DECL_CONTEXT (t
) = child_fn
;
8919 /* Split ENTRY_BB at GIMPLE_*,
8920 so that it can be moved to the child function. */
8921 gsi
= gsi_last_bb (entry_bb
);
8922 stmt
= gsi_stmt (gsi
);
8924 gimple_code (stmt
) == gimple_code (entry_stmt
));
8925 gsi_remove (&gsi
, true);
8926 e
= split_block (entry_bb
, stmt
);
8928 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8930 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8933 gsi
= gsi_last_bb (exit_bb
);
8934 gcc_assert (!gsi_end_p (gsi
)
8935 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8936 stmt
= gimple_build_return (NULL
);
8937 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8938 gsi_remove (&gsi
, true);
8941 /* Move the offloading region into CHILD_CFUN. */
8943 block
= gimple_block (entry_stmt
);
8945 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8947 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8948 /* When the OMP expansion process cannot guarantee an up-to-date
8949 loop tree arrange for the child function to fixup loops. */
8950 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8951 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8953 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8954 num
= vec_safe_length (child_cfun
->local_decls
);
8955 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8957 t
= (*child_cfun
->local_decls
)[srcidx
];
8958 if (DECL_CONTEXT (t
) == cfun
->decl
)
8960 if (srcidx
!= dstidx
)
8961 (*child_cfun
->local_decls
)[dstidx
] = t
;
8965 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8967 /* Inform the callgraph about the new function. */
8968 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8969 cgraph_node::add_new_function (child_fn
, true);
8971 /* Add the new function to the offload table. */
8972 vec_safe_push (offload_funcs
, child_fn
);
8974 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8975 fixed in a following pass. */
8976 push_cfun (child_cfun
);
8977 cgraph_edge::rebuild_edges ();
8979 /* Some EH regions might become dead, see PR34608. If
8980 pass_cleanup_cfg isn't the first pass to happen with the
8981 new child, these dead EH edges might cause problems.
8982 Clean them up now. */
8983 if (flag_exceptions
)
8986 bool changed
= false;
8988 FOR_EACH_BB_FN (bb
, cfun
)
8989 changed
|= gimple_purge_dead_eh_edges (bb
);
8991 cleanup_tree_cfg ();
8996 /* Emit a library call to launch the offloading region, or do data
8998 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8999 enum built_in_function start_ix
;
9000 location_t clause_loc
;
9001 tree (*gimple_omp_clauses
) (const_gimple
);
9003 switch (region
->type
)
9005 case GIMPLE_OACC_KERNELS
:
9006 gimple_omp_clauses
= gimple_oacc_kernels_clauses
;
9007 start_ix
= BUILT_IN_GOACC_KERNELS
;
9009 case GIMPLE_OACC_PARALLEL
:
9010 gimple_omp_clauses
= gimple_oacc_parallel_clauses
;
9011 start_ix
= BUILT_IN_GOACC_PARALLEL
;
9013 case GIMPLE_OMP_TARGET
:
9014 gimple_omp_clauses
= gimple_omp_target_clauses
;
9015 switch (gimple_omp_target_kind (entry_stmt
))
9017 case GF_OMP_TARGET_KIND_REGION
:
9018 start_ix
= BUILT_IN_GOMP_TARGET
;
9020 case GF_OMP_TARGET_KIND_DATA
:
9021 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
9023 case GF_OMP_TARGET_KIND_UPDATE
:
9024 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
9026 case GF_OMP_TARGET_KIND_OACC_DATA
:
9027 start_ix
= BUILT_IN_GOACC_DATA_START
;
9029 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9030 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
9032 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9033 start_ix
= BUILT_IN_GOACC_UPDATE
;
9043 clauses
= gimple_omp_clauses (entry_stmt
);
9045 /* By default, the value of DEVICE is -1 (let runtime library choose)
9046 and there is no conditional. */
9048 device
= build_int_cst (integer_type_node
, -1);
9050 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
9052 cond
= OMP_CLAUSE_IF_EXPR (c
);
9054 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
9057 /* Even if we pass it to all library function calls, it is currently only
9058 defined/used for the OpenMP target ones. */
9059 gcc_assert (start_ix
== BUILT_IN_GOMP_TARGET
9060 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
9061 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
);
9063 device
= OMP_CLAUSE_DEVICE_ID (c
);
9064 clause_loc
= OMP_CLAUSE_LOCATION (c
);
9067 clause_loc
= gimple_location (entry_stmt
);
9069 /* Ensure 'device' is of the correct type. */
9070 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
9072 /* If we found the clause 'if (cond)', build
9073 (cond ? device : -2). */
9076 cond
= gimple_boolify (cond
);
9078 basic_block cond_bb
, then_bb
, else_bb
;
9082 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
9085 e
= split_block (new_bb
, NULL
);
9089 gsi
= gsi_last_bb (new_bb
);
9091 e
= split_block (new_bb
, gsi_stmt (gsi
));
9097 then_bb
= create_empty_bb (cond_bb
);
9098 else_bb
= create_empty_bb (then_bb
);
9099 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
9100 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
9102 stmt
= gimple_build_cond_empty (cond
);
9103 gsi
= gsi_last_bb (cond_bb
);
9104 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9106 gsi
= gsi_start_bb (then_bb
);
9107 stmt
= gimple_build_assign (tmp_var
, device
);
9108 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9110 gsi
= gsi_start_bb (else_bb
);
9111 stmt
= gimple_build_assign (tmp_var
,
9112 build_int_cst (integer_type_node
, -2));
9113 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9115 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
9116 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
9117 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
9118 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
9119 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
9120 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
9125 gsi
= gsi_last_bb (new_bb
);
9126 t
= gimple_omp_data_arg (entry_stmt
);
9129 t1
= size_zero_node
;
9130 t2
= build_zero_cst (ptr_type_node
);
9136 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
9137 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
9138 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
9139 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
9140 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
9144 tree openmp_target
= get_offload_symbol_decl ();
9146 /* The maximum number used by any start_ix, without varargs. */
9147 unsigned int argcnt
= 12;
9149 vec_alloc (args
, argcnt
);
9150 args
->quick_push (device
);
9152 args
->quick_push (build_fold_addr_expr (child_fn
));
9153 args
->quick_push (build_fold_addr_expr (openmp_target
));
9154 args
->quick_push (t1
);
9155 args
->quick_push (t2
);
9156 args
->quick_push (t3
);
9157 args
->quick_push (t4
);
9160 case BUILT_IN_GOACC_DATA_START
:
9161 case BUILT_IN_GOMP_TARGET
:
9162 case BUILT_IN_GOMP_TARGET_DATA
:
9163 case BUILT_IN_GOMP_TARGET_UPDATE
:
9165 case BUILT_IN_GOACC_KERNELS
:
9166 case BUILT_IN_GOACC_PARALLEL
:
9168 tree t_num_gangs
, t_num_workers
, t_vector_length
;
9170 /* Default values for num_gangs, num_workers, and vector_length. */
9171 t_num_gangs
= t_num_workers
= t_vector_length
9172 = fold_convert_loc (gimple_location (entry_stmt
),
9173 integer_type_node
, integer_one_node
);
9174 /* ..., but if present, use the value specified by the respective
9175 clause, making sure that are of the correct type. */
9176 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
9178 t_num_gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9180 OMP_CLAUSE_NUM_GANGS_EXPR (c
));
9181 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_WORKERS
);
9183 t_num_workers
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9185 OMP_CLAUSE_NUM_WORKERS_EXPR (c
));
9186 c
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
9188 t_vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9190 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c
));
9191 args
->quick_push (t_num_gangs
);
9192 args
->quick_push (t_num_workers
);
9193 args
->quick_push (t_vector_length
);
9196 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9197 case BUILT_IN_GOACC_UPDATE
:
9202 /* Default values for t_async. */
9203 /* TODO: XXX FIX -2. */
9204 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
9206 build_int_cst (integer_type_node
, -2));
9207 /* ..., but if present, use the value specified by the respective
9208 clause, making sure that is of the correct type. */
9209 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
9211 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9213 OMP_CLAUSE_ASYNC_EXPR (c
));
9215 args
->quick_push (t_async
);
9216 /* Save the index, and... */
9217 t_wait_idx
= args
->length ();
9218 /* ... push a default value. */
9219 args
->quick_push (fold_convert_loc (gimple_location (entry_stmt
),
9221 integer_zero_node
));
9222 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
9227 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
9229 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
9231 args
->safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9233 OMP_CLAUSE_WAIT_EXPR (c
)));
9238 /* Now that we know the number, replace the default value. */
9239 args
->ordered_remove (t_wait_idx
);
9240 args
->quick_insert (t_wait_idx
,
9241 fold_convert_loc (gimple_location (entry_stmt
),
9243 build_int_cst (integer_type_node
, n
)));
9251 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), *args
);
9253 gimple_set_location (g
, gimple_location (entry_stmt
));
9254 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
9258 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
9259 gsi_remove (&gsi
, true);
9264 gsi
= gsi_last_bb (region
->exit
);
9266 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
9267 gsi_remove (&gsi
, true);
9272 /* Expand the parallel region tree rooted at REGION. Expansion
9273 proceeds in depth-first order. Innermost regions are expanded
9274 first. This way, parallel regions that require a new function to
9275 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9276 internal dependencies in their body. */
9279 expand_omp (struct omp_region
*region
)
9283 location_t saved_location
;
9284 gimple inner_stmt
= NULL
;
9286 /* First, determine whether this is a combined parallel+workshare
9288 if (region
->type
== GIMPLE_OMP_PARALLEL
)
9289 determine_parallel_type (region
);
9291 if (region
->type
== GIMPLE_OMP_FOR
9292 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
9293 inner_stmt
= last_stmt (region
->inner
->entry
);
9296 expand_omp (region
->inner
);
9298 saved_location
= input_location
;
9299 if (gimple_has_location (last_stmt (region
->entry
)))
9300 input_location
= gimple_location (last_stmt (region
->entry
));
9302 switch (region
->type
)
9304 case GIMPLE_OMP_PARALLEL
:
9305 case GIMPLE_OMP_TASK
:
9306 expand_omp_taskreg (region
);
9309 case GIMPLE_OMP_FOR
:
9310 expand_omp_for (region
, inner_stmt
);
9313 case GIMPLE_OMP_SECTIONS
:
9314 expand_omp_sections (region
);
9317 case GIMPLE_OMP_SECTION
:
9318 /* Individual omp sections are handled together with their
9319 parent GIMPLE_OMP_SECTIONS region. */
9322 case GIMPLE_OMP_SINGLE
:
9323 expand_omp_single (region
);
9326 case GIMPLE_OMP_MASTER
:
9327 case GIMPLE_OMP_TASKGROUP
:
9328 case GIMPLE_OMP_ORDERED
:
9329 case GIMPLE_OMP_CRITICAL
:
9330 case GIMPLE_OMP_TEAMS
:
9331 expand_omp_synch (region
);
9334 case GIMPLE_OMP_ATOMIC_LOAD
:
9335 expand_omp_atomic (region
);
9338 case GIMPLE_OACC_KERNELS
:
9339 case GIMPLE_OACC_PARALLEL
:
9340 case GIMPLE_OMP_TARGET
:
9341 expand_omp_target (region
);
9348 input_location
= saved_location
;
9349 region
= region
->next
;
9354 /* Helper for build_omp_regions. Scan the dominator tree starting at
9355 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9356 true, the function ends once a single tree is built (otherwise, whole
9357 forest of OMP constructs may be built). */
9360 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
9363 gimple_stmt_iterator gsi
;
9367 gsi
= gsi_last_bb (bb
);
9368 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
9370 struct omp_region
*region
;
9371 enum gimple_code code
;
9373 stmt
= gsi_stmt (gsi
);
9374 code
= gimple_code (stmt
);
9375 if (code
== GIMPLE_OMP_RETURN
)
9377 /* STMT is the return point out of region PARENT. Mark it
9378 as the exit point and make PARENT the immediately
9379 enclosing region. */
9380 gcc_assert (parent
);
9383 parent
= parent
->outer
;
9385 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
9387 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9388 GIMPLE_OMP_RETURN, but matches with
9389 GIMPLE_OMP_ATOMIC_LOAD. */
9390 gcc_assert (parent
);
9391 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
9394 parent
= parent
->outer
;
9397 else if (code
== GIMPLE_OMP_CONTINUE
)
9399 gcc_assert (parent
);
9402 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
9404 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9405 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9408 else if (code
== GIMPLE_OMP_TARGET
9409 && (gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
9410 || (gimple_omp_target_kind (stmt
)
9411 == GF_OMP_TARGET_KIND_OACC_UPDATE
)))
9412 new_omp_region (bb
, code
, parent
);
9415 /* Otherwise, this directive becomes the parent for a new
9417 region
= new_omp_region (bb
, code
, parent
);
9422 if (single_tree
&& !parent
)
9425 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
9427 son
= next_dom_son (CDI_DOMINATORS
, son
))
9428 build_omp_regions_1 (son
, parent
, single_tree
);
9431 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9435 build_omp_regions_root (basic_block root
)
9437 gcc_assert (root_omp_region
== NULL
);
9438 build_omp_regions_1 (root
, NULL
, true);
9439 gcc_assert (root_omp_region
!= NULL
);
9442 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9445 omp_expand_local (basic_block head
)
9447 build_omp_regions_root (head
);
9448 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
9450 fprintf (dump_file
, "\nOMP region tree\n\n");
9451 dump_omp_region (dump_file
, root_omp_region
, 0);
9452 fprintf (dump_file
, "\n");
9455 remove_exit_barriers (root_omp_region
);
9456 expand_omp (root_omp_region
);
9458 free_omp_regions ();
9461 /* Scan the CFG and build a tree of OMP regions. Return the root of
9462 the OMP region tree. */
9465 build_omp_regions (void)
9467 gcc_assert (root_omp_region
== NULL
);
9468 calculate_dominance_info (CDI_DOMINATORS
);
9469 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
9472 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9475 execute_expand_omp (void)
9477 build_omp_regions ();
9479 if (!root_omp_region
)
9484 fprintf (dump_file
, "\nOMP region tree\n\n");
9485 dump_omp_region (dump_file
, root_omp_region
, 0);
9486 fprintf (dump_file
, "\n");
9489 remove_exit_barriers (root_omp_region
);
9491 expand_omp (root_omp_region
);
9493 cleanup_tree_cfg ();
9495 free_omp_regions ();
9500 /* OMP expansion -- the default pass, run before creation of SSA form. */
9504 const pass_data pass_data_expand_omp
=
9506 GIMPLE_PASS
, /* type */
9507 "ompexp", /* name */
9508 OPTGROUP_NONE
, /* optinfo_flags */
9509 TV_NONE
, /* tv_id */
9510 PROP_gimple_any
, /* properties_required */
9511 0, /* properties_provided */
9512 0, /* properties_destroyed */
9513 0, /* todo_flags_start */
9514 0, /* todo_flags_finish */
9517 class pass_expand_omp
: public gimple_opt_pass
9520 pass_expand_omp (gcc::context
*ctxt
)
9521 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
9524 /* opt_pass methods: */
9525 virtual bool gate (function
*)
9527 return ((flag_openacc
!= 0 || flag_openmp
!= 0 || flag_openmp_simd
!= 0
9528 || flag_cilkplus
!= 0) && !seen_error ());
9531 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
9533 }; // class pass_expand_omp
9538 make_pass_expand_omp (gcc::context
*ctxt
)
9540 return new pass_expand_omp (ctxt
);
9543 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9544 convert it to gimple. */
9546 omp_gimple_assign_with_ops (tree_code op
, tree dest
, tree src
, gimple_seq
*seq
)
9550 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
9552 stmt
= gimple_build_assign_with_ops (op
, dest
, dest
, src
);
9553 gimple_seq_add_stmt (seq
, stmt
);
9557 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9558 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9559 gimplify_assign (t
, rdest
, seq
);
9562 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9563 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9564 gimplify_assign (t
, idest
, seq
);
9567 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)), NULL
);
9568 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9569 gimplify_assign (t
, rsrc
, seq
);
9572 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)), NULL
);
9573 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9574 gimplify_assign (t
, isrc
, seq
);
9577 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9578 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9581 gcc_assert (op
== PLUS_EXPR
|| op
== MULT_EXPR
);
9583 if (op
== PLUS_EXPR
)
9585 stmt
= gimple_build_assign_with_ops (op
, r
, rdest
, rsrc
);
9586 gimple_seq_add_stmt (seq
, stmt
);
9588 stmt
= gimple_build_assign_with_ops (op
, i
, idest
, isrc
);
9589 gimple_seq_add_stmt (seq
, stmt
);
9591 else if (op
== MULT_EXPR
)
9593 /* Let x = a + ib = dest, y = c + id = src.
9594 x * y = (ac - bd) + i(ad + bc) */
9595 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9596 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9597 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9598 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9600 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, ac
, rdest
, rsrc
);
9601 gimple_seq_add_stmt (seq
, stmt
);
9603 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, bd
, idest
, isrc
);
9604 gimple_seq_add_stmt (seq
, stmt
);
9606 stmt
= gimple_build_assign_with_ops (MINUS_EXPR
, r
, ac
, bd
);
9607 gimple_seq_add_stmt (seq
, stmt
);
9609 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, ad
, rdest
, isrc
);
9610 gimple_seq_add_stmt (seq
, stmt
);
9612 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, bd
, idest
, rsrc
);
9613 gimple_seq_add_stmt (seq
, stmt
);
9615 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, i
, ad
, bc
);
9616 gimple_seq_add_stmt (seq
, stmt
);
9619 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
9620 gimplify_assign (dest
, result
, seq
);
9623 /* Helper function to initialize local data for the reduction arrays.
9624 The reduction arrays need to be placed inside the calling function
9625 for accelerators, or else the host won't be able to preform the final
9626 reduction. FIXME: This function assumes that there are
9627 vector_length threads in total. */
9630 initialize_reduction_data (tree clauses
, tree nthreads
, gimple_seq
*stmt_seqp
,
9633 gcc_assert (is_gimple_omp_oacc_specifically (ctx
->stmt
));
9638 tree (*gimple_omp_clauses
) (const_gimple
);
9639 void (*gimple_omp_set_clauses
) (gimple
, tree
);
9641 /* Find the innermost PARALLEL openmp context. FIXME: OpenACC kernels
9642 may require extra care unless they are converted to openmp for loops. */
9644 if (gimple_code (ctx
->stmt
) == GIMPLE_OACC_PARALLEL
)
9649 gimple_omp_clauses
= gimple_oacc_parallel_clauses
;
9650 gimple_omp_set_clauses
= gimple_oacc_parallel_set_clauses
;
9652 /* Extract the clauses. */
9653 oc
= gimple_omp_clauses (octx
->stmt
);
9655 /* Find the last outer clause. */
9656 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
9659 /* Allocate arrays for each reduction variable. */
9660 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9662 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9665 tree var
= OMP_CLAUSE_DECL (c
);
9666 tree type
= get_base_type (var
);
9667 tree array
= lookup_reduction (omp_get_id (var
), ctx
);
9670 /* Calculate size of the reduction array. */
9671 t
= create_tmp_var (TREE_TYPE (nthreads
), NULL
);
9672 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, t
, nthreads
,
9673 fold_convert (TREE_TYPE (nthreads
),
9674 TYPE_SIZE_UNIT (type
)));
9675 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9677 size
= create_tmp_var (sizetype
, NULL
);
9678 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
9680 /* Now allocate memory for it. FIXME: Allocating memory for the
9681 reduction array may be unnecessary once the final reduction is able
9682 to be preformed on the accelerator. Instead of allocating memory on
9683 the host side, it could just be allocated on the accelerator. */
9684 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
9685 stmt
= gimple_build_call (call
, 1, size
);
9686 gimple_call_set_lhs (stmt
, array
);
9687 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9689 /* Map this array into the accelerator. */
9691 /* Add the reduction array to the list of clauses. */
9692 /* FIXME: Currently, these variables must be placed in the outer
9693 most clause so that copy-out works. */
9695 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
9696 OMP_CLAUSE_MAP_KIND (t
) = OMP_CLAUSE_MAP_FORCE_FROM
;
9697 OMP_CLAUSE_DECL (t
) = x
;
9698 OMP_CLAUSE_CHAIN (t
) = NULL
;
9700 OMP_CLAUSE_CHAIN (oc
) = t
;
9702 gimple_omp_set_clauses (octx
->stmt
, t
);
9703 OMP_CLAUSE_SIZE (t
) = size
;
9708 /* Helper function to process the array of partial reductions. Nthreads
9709 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9710 cannot be used here, because nthreads on the host may be different than
9711 on the accelerator. */
9714 finalize_reduction_data (tree clauses
, tree nthreads
, gimple_seq
*stmt_seqp
,
9717 gcc_assert (is_gimple_omp_oacc_specifically (ctx
->stmt
));
9719 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
9724 let var = the original reduction variable
9725 let array = reduction variable array
9727 for (i = 0; i < nthreads; i++)
9731 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
9732 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
9733 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
9735 /* Create and initialize an index variable. */
9736 tree ix
= create_tmp_var (sizetype
, NULL
);
9737 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
9740 /* Insert the loop header label here. */
9741 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
9743 /* Exit loop if ix >= nthreads. */
9744 x
= create_tmp_var (sizetype
, NULL
);
9745 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
9746 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
9747 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9749 /* Insert the loop body label here. */
9750 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
9752 /* Collapse each reduction array, one element at a time. */
9753 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9755 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9758 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
9760 /* reduction(-:var) sums up the partial results, so it acts
9761 identically to reduction(+:var). */
9762 if (reduction_code
== MINUS_EXPR
)
9763 reduction_code
= PLUS_EXPR
;
9765 /* Set up reduction variable var. */
9766 var
= OMP_CLAUSE_DECL (c
);
9767 type
= get_base_type (var
);
9768 array
= lookup_reduction (omp_get_id (OMP_CLAUSE_DECL (c
)), ctx
);
9770 /* Calculate the array offset. */
9771 tree offset
= create_tmp_var (sizetype
, NULL
);
9772 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
9773 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, offset
, offset
, ix
);
9774 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9776 tree ptr
= create_tmp_var (TREE_TYPE (array
), NULL
);
9777 stmt
= gimple_build_assign_with_ops (POINTER_PLUS_EXPR
, ptr
, array
,
9779 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9781 /* Extract array[ix] into mem. */
9782 tree mem
= create_tmp_var (type
, NULL
);
9783 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
9785 /* Find the original reduction variable. */
9786 if (is_reference (var
))
9787 var
= build_simple_mem_ref (var
);
9789 tree t
= create_tmp_var (type
, NULL
);
9791 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
9792 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9794 /* var = var op mem */
9795 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
9797 case TRUTH_ANDIF_EXPR
:
9798 case TRUTH_ORIF_EXPR
:
9799 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
9801 gimplify_and_add (t
, stmt_seqp
);
9804 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9805 omp_gimple_assign_with_ops (OMP_CLAUSE_REDUCTION_CODE (c
),
9809 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
9810 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
9811 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9814 /* Increment the induction variable. */
9815 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
9816 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, ix
, ix
, one
);
9817 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9819 /* Go back to the top of the loop. */
9820 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
9822 /* Place the loop exit label here. */
9823 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
9826 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9827 scan that for reductions. */
9830 process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
9831 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
9833 gcc_assert (is_gimple_omp_oacc_specifically (ctx
->stmt
));
9835 gimple_stmt_iterator gsi
;
9836 gimple_seq inner
= NULL
;
9839 /* A collapse clause may have inserted a new bind block. */
9840 gsi
= gsi_start (*body
);
9841 while (!gsi_end_p (gsi
))
9843 stmt
= gsi_stmt (gsi
);
9844 if (gimple_code (stmt
) == GIMPLE_BIND
)
9846 inner
= gimple_bind_body (stmt
);
9848 gsi
= gsi_start (*body
);
9850 else if (gimple_code (stmt
) == GIMPLE_OMP_FOR
)
9856 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9858 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
9860 bool reduction_found
= false;
9862 stmt
= gsi_stmt (gsi
);
9864 switch (gimple_code (stmt
))
9866 /* FIXME: A reduction may also appear in an oacc parallel. */
9867 case GIMPLE_OMP_FOR
:
9868 clauses
= gimple_omp_for_clauses (stmt
);
9870 /* Search for a reduction clause. */
9871 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9872 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
9874 reduction_found
= true;
9878 if (!reduction_found
)
9881 ctx
= maybe_lookup_ctx (stmt
);
9884 /* Extract the number of threads. */
9885 nthreads
= create_tmp_var (sizetype
, NULL
);
9886 t
= oacc_max_threads (ctx
);
9887 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
9889 /* Determine if this is kernel will be executed on the host. */
9890 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
9891 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
9892 stmt
= gimple_build_call (call
, 0);
9893 gimple_call_set_lhs (stmt
, acc_device
);
9894 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9896 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9897 acc_device_host
= create_tmp_var (integer_type_node
,
9898 ".acc_device_host");
9899 gimplify_assign (acc_device_host
, build_int_cst (integer_type_node
,
9903 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9904 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9906 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9908 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9909 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9910 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9913 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9915 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9916 gimplify_assign (acc_device_host
, build_int_cst (integer_type_node
,
9920 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9921 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9923 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9925 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9926 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9927 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9930 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9932 initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
, ctx
);
9933 finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
9936 // Scan for other directives which support reduction here.
9942 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
9944 /* If ctx is a worksharing context inside of a cancellable parallel
9945 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
9946 and conditional branch to parallel's cancel_label to handle
9947 cancellation in the implicit barrier. */
9950 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
9952 gimple omp_return
= gimple_seq_last_stmt (*body
);
9953 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
9954 if (gimple_omp_return_nowait_p (omp_return
))
9957 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
9958 && ctx
->outer
->cancellable
)
9960 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
9961 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
9962 tree lhs
= create_tmp_var (c_bool_type
, NULL
);
9963 gimple_omp_return_set_lhs (omp_return
, lhs
);
9964 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
9965 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
9966 fold_convert (c_bool_type
,
9967 boolean_false_node
),
9968 ctx
->outer
->cancel_label
, fallthru_label
);
9969 gimple_seq_add_stmt (body
, g
);
9970 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
9974 /* Lower the OpenMP sections directive in the current statement in GSI_P.
9975 CTX is the enclosing OMP context for the current statement. */
9978 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9980 tree block
, control
;
9981 gimple_stmt_iterator tgsi
;
9982 gimple stmt
, new_stmt
, bind
, t
;
9983 gimple_seq ilist
, dlist
, olist
, new_body
;
9985 stmt
= gsi_stmt (*gsi_p
);
9987 push_gimplify_context ();
9991 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
9992 &ilist
, &dlist
, ctx
, NULL
);
9994 new_body
= gimple_omp_body (stmt
);
9995 gimple_omp_set_body (stmt
, NULL
);
9996 tgsi
= gsi_start (new_body
);
9997 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
10002 sec_start
= gsi_stmt (tgsi
);
10003 sctx
= maybe_lookup_ctx (sec_start
);
10006 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
10007 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
10008 GSI_CONTINUE_LINKING
);
10009 gimple_omp_set_body (sec_start
, NULL
);
10011 if (gsi_one_before_end_p (tgsi
))
10013 gimple_seq l
= NULL
;
10014 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
10016 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
10017 gimple_omp_section_set_last (sec_start
);
10020 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
10021 GSI_CONTINUE_LINKING
);
10024 block
= make_node (BLOCK
);
10025 bind
= gimple_build_bind (NULL
, new_body
, block
);
10028 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
10030 block
= make_node (BLOCK
);
10031 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10032 gsi_replace (gsi_p
, new_stmt
, true);
10034 pop_gimplify_context (new_stmt
);
10035 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10036 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10037 if (BLOCK_VARS (block
))
10038 TREE_USED (block
) = 1;
10041 gimple_seq_add_seq (&new_body
, ilist
);
10042 gimple_seq_add_stmt (&new_body
, stmt
);
10043 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
10044 gimple_seq_add_stmt (&new_body
, bind
);
10046 control
= create_tmp_var (unsigned_type_node
, ".section");
10047 t
= gimple_build_omp_continue (control
, control
);
10048 gimple_omp_sections_set_control (stmt
, control
);
10049 gimple_seq_add_stmt (&new_body
, t
);
10051 gimple_seq_add_seq (&new_body
, olist
);
10052 if (ctx
->cancellable
)
10053 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10054 gimple_seq_add_seq (&new_body
, dlist
);
10056 new_body
= maybe_catch_exception (new_body
);
10058 t
= gimple_build_omp_return
10059 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
10060 OMP_CLAUSE_NOWAIT
));
10061 gimple_seq_add_stmt (&new_body
, t
);
10062 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
10064 gimple_bind_set_body (new_stmt
, new_body
);
10068 /* A subroutine of lower_omp_single. Expand the simple form of
10069 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10071 if (GOMP_single_start ())
10073 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10075 FIXME. It may be better to delay expanding the logic of this until
10076 pass_expand_omp. The expanded logic may make the job more difficult
10077 to a synchronization analysis pass. */
10080 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
10082 location_t loc
= gimple_location (single_stmt
);
10083 tree tlabel
= create_artificial_label (loc
);
10084 tree flabel
= create_artificial_label (loc
);
10088 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
10089 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
10090 call
= gimple_build_call (decl
, 0);
10091 gimple_call_set_lhs (call
, lhs
);
10092 gimple_seq_add_stmt (pre_p
, call
);
10094 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
10095 fold_convert_loc (loc
, TREE_TYPE (lhs
),
10096 boolean_true_node
),
10098 gimple_seq_add_stmt (pre_p
, cond
);
10099 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
10100 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10101 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
10105 /* A subroutine of lower_omp_single. Expand the simple form of
10106 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10108 #pragma omp single copyprivate (a, b, c)
10110 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10113 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10119 GOMP_single_copy_end (©out);
10130 FIXME. It may be better to delay expanding the logic of this until
10131 pass_expand_omp. The expanded logic may make the job more difficult
10132 to a synchronization analysis pass. */
10135 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
10137 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
10138 gimple_seq copyin_seq
;
10139 location_t loc
= gimple_location (single_stmt
);
10141 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
10143 ptr_type
= build_pointer_type (ctx
->record_type
);
10144 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
10146 l0
= create_artificial_label (loc
);
10147 l1
= create_artificial_label (loc
);
10148 l2
= create_artificial_label (loc
);
10150 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
10151 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
10152 t
= fold_convert_loc (loc
, ptr_type
, t
);
10153 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
10155 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
10156 build_int_cst (ptr_type
, 0));
10157 t
= build3 (COND_EXPR
, void_type_node
, t
,
10158 build_and_jump (&l0
), build_and_jump (&l1
));
10159 gimplify_and_add (t
, pre_p
);
10161 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
10163 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10166 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
10169 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10170 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
10171 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
10172 gimplify_and_add (t
, pre_p
);
10174 t
= build_and_jump (&l2
);
10175 gimplify_and_add (t
, pre_p
);
10177 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
10179 gimple_seq_add_seq (pre_p
, copyin_seq
);
10181 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
10185 /* Expand code for an OpenMP single directive. */
10188 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10191 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
10192 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
10194 push_gimplify_context ();
10196 block
= make_node (BLOCK
);
10197 bind
= gimple_build_bind (NULL
, NULL
, block
);
10198 gsi_replace (gsi_p
, bind
, true);
10201 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
10202 &bind_body
, &dlist
, ctx
, NULL
);
10203 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
10205 gimple_seq_add_stmt (&bind_body
, single_stmt
);
10207 if (ctx
->record_type
)
10208 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
10210 lower_omp_single_simple (single_stmt
, &bind_body
);
10212 gimple_omp_set_body (single_stmt
, NULL
);
10214 gimple_seq_add_seq (&bind_body
, dlist
);
10216 bind_body
= maybe_catch_exception (bind_body
);
10218 t
= gimple_build_omp_return
10219 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
10220 OMP_CLAUSE_NOWAIT
));
10221 gimple_seq_add_stmt (&bind_body_tail
, t
);
10222 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
10223 if (ctx
->record_type
)
10225 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
10226 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10227 TREE_THIS_VOLATILE (clobber
) = 1;
10228 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
10229 clobber
), GSI_SAME_STMT
);
10231 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
10232 gimple_bind_set_body (bind
, bind_body
);
10234 pop_gimplify_context (bind
);
10236 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10237 BLOCK_VARS (block
) = ctx
->block_vars
;
10238 if (BLOCK_VARS (block
))
10239 TREE_USED (block
) = 1;
10243 /* Expand code for an OpenMP master directive. */
10246 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10248 tree block
, lab
= NULL
, x
, bfn_decl
;
10249 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
10250 location_t loc
= gimple_location (stmt
);
10253 push_gimplify_context ();
10255 block
= make_node (BLOCK
);
10256 bind
= gimple_build_bind (NULL
, NULL
, block
);
10257 gsi_replace (gsi_p
, bind
, true);
10258 gimple_bind_add_stmt (bind
, stmt
);
10260 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
10261 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
10262 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
10263 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
10265 gimplify_and_add (x
, &tseq
);
10266 gimple_bind_add_seq (bind
, tseq
);
10268 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10269 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10270 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10271 gimple_omp_set_body (stmt
, NULL
);
10273 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
10275 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10277 pop_gimplify_context (bind
);
10279 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10280 BLOCK_VARS (block
) = ctx
->block_vars
;
10284 /* Expand code for an OpenMP taskgroup directive. */
10287 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10289 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
10290 tree block
= make_node (BLOCK
);
10292 bind
= gimple_build_bind (NULL
, NULL
, block
);
10293 gsi_replace (gsi_p
, bind
, true);
10294 gimple_bind_add_stmt (bind
, stmt
);
10296 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
10298 gimple_bind_add_stmt (bind
, x
);
10300 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10301 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10302 gimple_omp_set_body (stmt
, NULL
);
10304 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10306 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10307 BLOCK_VARS (block
) = ctx
->block_vars
;
10311 /* Expand code for an OpenMP ordered directive. */
10314 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10317 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
10319 push_gimplify_context ();
10321 block
= make_node (BLOCK
);
10322 bind
= gimple_build_bind (NULL
, NULL
, block
);
10323 gsi_replace (gsi_p
, bind
, true);
10324 gimple_bind_add_stmt (bind
, stmt
);
10326 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
10328 gimple_bind_add_stmt (bind
, x
);
10330 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10331 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10332 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10333 gimple_omp_set_body (stmt
, NULL
);
10335 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
10336 gimple_bind_add_stmt (bind
, x
);
10338 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10340 pop_gimplify_context (bind
);
10342 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10343 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10347 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10348 substitution of a couple of function calls. But in the NAMED case,
10349 requires that languages coordinate a symbol name. It is therefore
10350 best put here in common code. */
10352 static GTY((param1_is (tree
), param2_is (tree
)))
10353 splay_tree critical_name_mutexes
;
10356 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10359 tree name
, lock
, unlock
;
10360 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
10361 location_t loc
= gimple_location (stmt
);
10364 name
= gimple_omp_critical_name (stmt
);
10370 if (!critical_name_mutexes
)
10371 critical_name_mutexes
10372 = splay_tree_new_ggc (splay_tree_compare_pointers
,
10373 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
10374 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
10376 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
10381 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
10383 new_str
= ACONCAT ((".gomp_critical_user_",
10384 IDENTIFIER_POINTER (name
), NULL
));
10385 DECL_NAME (decl
) = get_identifier (new_str
);
10386 TREE_PUBLIC (decl
) = 1;
10387 TREE_STATIC (decl
) = 1;
10388 DECL_COMMON (decl
) = 1;
10389 DECL_ARTIFICIAL (decl
) = 1;
10390 DECL_IGNORED_P (decl
) = 1;
10391 varpool_node::finalize_decl (decl
);
10393 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
10394 (splay_tree_value
) decl
);
10397 decl
= (tree
) n
->value
;
10399 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
10400 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
10402 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
10403 unlock
= build_call_expr_loc (loc
, unlock
, 1,
10404 build_fold_addr_expr_loc (loc
, decl
));
10408 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
10409 lock
= build_call_expr_loc (loc
, lock
, 0);
10411 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
10412 unlock
= build_call_expr_loc (loc
, unlock
, 0);
10415 push_gimplify_context ();
10417 block
= make_node (BLOCK
);
10418 bind
= gimple_build_bind (NULL
, NULL
, block
);
10419 gsi_replace (gsi_p
, bind
, true);
10420 gimple_bind_add_stmt (bind
, stmt
);
10422 tbody
= gimple_bind_body (bind
);
10423 gimplify_and_add (lock
, &tbody
);
10424 gimple_bind_set_body (bind
, tbody
);
10426 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10427 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10428 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10429 gimple_omp_set_body (stmt
, NULL
);
10431 tbody
= gimple_bind_body (bind
);
10432 gimplify_and_add (unlock
, &tbody
);
10433 gimple_bind_set_body (bind
, tbody
);
10435 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10437 pop_gimplify_context (bind
);
10438 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10439 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10443 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10444 for a lastprivate clause. Given a loop control predicate of (V
10445 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10446 is appended to *DLIST, iterator initialization is appended to
10450 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
10451 gimple_seq
*dlist
, struct omp_context
*ctx
)
10453 tree clauses
, cond
, vinit
;
10454 enum tree_code cond_code
;
10457 cond_code
= fd
->loop
.cond_code
;
10458 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
10460 /* When possible, use a strict equality expression. This can let VRP
10461 type optimizations deduce the value and remove a copy. */
10462 if (tree_fits_shwi_p (fd
->loop
.step
))
10464 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
10465 if (step
== 1 || step
== -1)
10466 cond_code
= EQ_EXPR
;
10469 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
10471 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
10473 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
10474 if (!gimple_seq_empty_p (stmts
))
10476 gimple_seq_add_seq (&stmts
, *dlist
);
10479 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10480 vinit
= fd
->loop
.n1
;
10481 if (cond_code
== EQ_EXPR
10482 && tree_fits_shwi_p (fd
->loop
.n2
)
10483 && ! integer_zerop (fd
->loop
.n2
))
10484 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
10486 vinit
= unshare_expr (vinit
);
10488 /* Initialize the iterator variable, so that threads that don't execute
10489 any iterations don't execute the lastprivate clauses by accident. */
10490 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
10495 /* Lower code for an OpenMP loop directive. */
10498 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10500 tree
*rhs_p
, block
;
10501 struct omp_for_data fd
, *fdp
= NULL
;
10502 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
10503 gimple_seq omp_for_body
, body
, dlist
;
10506 push_gimplify_context ();
10508 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
10510 block
= make_node (BLOCK
);
10511 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10512 /* Replace at gsi right away, so that 'stmt' is no member
10513 of a sequence anymore as we're going to add to to a different
10515 gsi_replace (gsi_p
, new_stmt
, true);
10517 /* Move declaration of temporaries in the loop body before we make
10519 omp_for_body
= gimple_omp_body (stmt
);
10520 if (!gimple_seq_empty_p (omp_for_body
)
10521 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
10523 gimple inner_bind
= gimple_seq_first_stmt (omp_for_body
);
10524 tree vars
= gimple_bind_vars (inner_bind
);
10525 gimple_bind_append_vars (new_stmt
, vars
);
10526 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10527 keep them on the inner_bind and it's block. */
10528 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
10529 if (gimple_bind_block (inner_bind
))
10530 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
10533 if (gimple_omp_for_combined_into_p (stmt
))
10535 gcc_assert (gimple_omp_for_kind (stmt
) != GF_OMP_FOR_KIND_OACC_LOOP
);
10537 extract_omp_for_data (stmt
, &fd
, NULL
);
10540 /* We need two temporaries with fd.loop.v type (istart/iend)
10541 and then (fd.collapse - 1) temporaries with the same
10542 type for count2 ... countN-1 vars if not constant. */
10544 tree type
= fd
.iter_type
;
10545 if (fd
.collapse
> 1
10546 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
10547 count
+= fd
.collapse
- 1;
10548 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
10549 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
10550 tree clauses
= *pc
;
10553 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
10554 OMP_CLAUSE__LOOPTEMP_
);
10555 for (i
= 0; i
< count
; i
++)
10560 gcc_assert (outerc
);
10561 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
10562 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
10563 OMP_CLAUSE__LOOPTEMP_
);
10567 temp
= create_tmp_var (type
, NULL
);
10568 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
10570 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
10571 OMP_CLAUSE_DECL (*pc
) = temp
;
10572 pc
= &OMP_CLAUSE_CHAIN (*pc
);
10577 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10580 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
10582 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
10584 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10586 /* Lower the header expressions. At this point, we can assume that
10587 the header is of the form:
10589 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10591 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10592 using the .omp_data_s mapping, if needed. */
10593 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
10595 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
10596 if (!is_gimple_min_invariant (*rhs_p
))
10597 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10599 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
10600 if (!is_gimple_min_invariant (*rhs_p
))
10601 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10603 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
10604 if (!is_gimple_min_invariant (*rhs_p
))
10605 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10608 /* Once lowered, extract the bounds and clauses. */
10609 extract_omp_for_data (stmt
, &fd
, NULL
);
10611 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
10613 gimple_seq_add_stmt (&body
, stmt
);
10614 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
10616 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
10619 /* After the loop, add exit clauses. */
10620 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
10622 if (ctx
->cancellable
)
10623 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
10625 gimple_seq_add_seq (&body
, dlist
);
10627 body
= maybe_catch_exception (body
);
10629 /* Region exit marker goes at the end of the loop body. */
10630 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
10631 maybe_add_implicit_barrier_cancel (ctx
, &body
);
10632 pop_gimplify_context (new_stmt
);
10634 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10635 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
10636 if (BLOCK_VARS (block
))
10637 TREE_USED (block
) = 1;
10639 gimple_bind_set_body (new_stmt
, body
);
10640 gimple_omp_set_body (stmt
, NULL
);
10641 gimple_omp_for_set_pre_body (stmt
, NULL
);
10644 /* Callback for walk_stmts. Check if the current statement only contains
10645 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10648 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
10649 bool *handled_ops_p
,
10650 struct walk_stmt_info
*wi
)
10652 int *info
= (int *) wi
->info
;
10653 gimple stmt
= gsi_stmt (*gsi_p
);
10655 *handled_ops_p
= true;
10656 switch (gimple_code (stmt
))
10660 case GIMPLE_OMP_FOR
:
10661 case GIMPLE_OMP_SECTIONS
:
10662 *info
= *info
== 0 ? 1 : -1;
10671 struct omp_taskcopy_context
10673 /* This field must be at the beginning, as we do "inheritance": Some
10674 callback functions for tree-inline.c (e.g., omp_copy_decl)
10675 receive a copy_body_data pointer that is up-casted to an
10676 omp_context pointer. */
10682 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
10684 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
10686 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
10687 return create_tmp_var (TREE_TYPE (var
), NULL
);
10693 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
10695 tree name
, new_fields
= NULL
, type
, f
;
10697 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
10698 name
= DECL_NAME (TYPE_NAME (orig_type
));
10699 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
10700 TYPE_DECL
, name
, type
);
10701 TYPE_NAME (type
) = name
;
10703 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
10705 tree new_f
= copy_node (f
);
10706 DECL_CONTEXT (new_f
) = type
;
10707 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
10708 TREE_CHAIN (new_f
) = new_fields
;
10709 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10710 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10711 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
10713 new_fields
= new_f
;
10714 tcctx
->cb
.decl_map
->put (f
, new_f
);
10716 TYPE_FIELDS (type
) = nreverse (new_fields
);
10717 layout_type (type
);
10721 /* Create task copyfn. */
10724 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
10726 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
10728 struct function
*child_cfun
;
10729 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
10730 tree record_type
, srecord_type
, bind
, list
;
10731 bool record_needs_remap
= false, srecord_needs_remap
= false;
10733 struct omp_taskcopy_context tcctx
;
10734 location_t loc
= gimple_location (task_stmt
);
10736 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
10737 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
10738 gcc_assert (child_cfun
->cfg
== NULL
);
10739 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
10741 /* Reset DECL_CONTEXT on function arguments. */
10742 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
10743 DECL_CONTEXT (t
) = child_fn
;
10745 /* Populate the function. */
10746 push_gimplify_context ();
10747 push_cfun (child_cfun
);
10749 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
10750 TREE_SIDE_EFFECTS (bind
) = 1;
10752 DECL_SAVED_TREE (child_fn
) = bind
;
10753 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
10755 /* Remap src and dst argument types if needed. */
10756 record_type
= ctx
->record_type
;
10757 srecord_type
= ctx
->srecord_type
;
10758 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
10759 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10761 record_needs_remap
= true;
10764 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
10765 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10767 srecord_needs_remap
= true;
10771 if (record_needs_remap
|| srecord_needs_remap
)
10773 memset (&tcctx
, '\0', sizeof (tcctx
));
10774 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
10775 tcctx
.cb
.dst_fn
= child_fn
;
10776 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
10777 gcc_checking_assert (tcctx
.cb
.src_node
);
10778 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
10779 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
10780 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
10781 tcctx
.cb
.eh_lp_nr
= 0;
10782 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
10783 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
10786 if (record_needs_remap
)
10787 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
10788 if (srecord_needs_remap
)
10789 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
10792 tcctx
.cb
.decl_map
= NULL
;
10794 arg
= DECL_ARGUMENTS (child_fn
);
10795 TREE_TYPE (arg
) = build_pointer_type (record_type
);
10796 sarg
= DECL_CHAIN (arg
);
10797 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
10799 /* First pass: initialize temporaries used in record_type and srecord_type
10800 sizes and field offsets. */
10801 if (tcctx
.cb
.decl_map
)
10802 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10803 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10807 decl
= OMP_CLAUSE_DECL (c
);
10808 p
= tcctx
.cb
.decl_map
->get (decl
);
10811 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10812 sf
= (tree
) n
->value
;
10813 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10814 src
= build_simple_mem_ref_loc (loc
, sarg
);
10815 src
= omp_build_component_ref (src
, sf
);
10816 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
10817 append_to_statement_list (t
, &list
);
10820 /* Second pass: copy shared var pointers and copy construct non-VLA
10821 firstprivate vars. */
10822 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10823 switch (OMP_CLAUSE_CODE (c
))
10825 case OMP_CLAUSE_SHARED
:
10826 decl
= OMP_CLAUSE_DECL (c
);
10827 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10830 f
= (tree
) n
->value
;
10831 if (tcctx
.cb
.decl_map
)
10832 f
= *tcctx
.cb
.decl_map
->get (f
);
10833 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10834 sf
= (tree
) n
->value
;
10835 if (tcctx
.cb
.decl_map
)
10836 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10837 src
= build_simple_mem_ref_loc (loc
, sarg
);
10838 src
= omp_build_component_ref (src
, sf
);
10839 dst
= build_simple_mem_ref_loc (loc
, arg
);
10840 dst
= omp_build_component_ref (dst
, f
);
10841 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10842 append_to_statement_list (t
, &list
);
10844 case OMP_CLAUSE_FIRSTPRIVATE
:
10845 decl
= OMP_CLAUSE_DECL (c
);
10846 if (is_variable_sized (decl
))
10848 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10851 f
= (tree
) n
->value
;
10852 if (tcctx
.cb
.decl_map
)
10853 f
= *tcctx
.cb
.decl_map
->get (f
);
10854 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10857 sf
= (tree
) n
->value
;
10858 if (tcctx
.cb
.decl_map
)
10859 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10860 src
= build_simple_mem_ref_loc (loc
, sarg
);
10861 src
= omp_build_component_ref (src
, sf
);
10862 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
10863 src
= build_simple_mem_ref_loc (loc
, src
);
10867 dst
= build_simple_mem_ref_loc (loc
, arg
);
10868 dst
= omp_build_component_ref (dst
, f
);
10869 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10870 append_to_statement_list (t
, &list
);
10872 case OMP_CLAUSE_PRIVATE
:
10873 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
10875 decl
= OMP_CLAUSE_DECL (c
);
10876 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10877 f
= (tree
) n
->value
;
10878 if (tcctx
.cb
.decl_map
)
10879 f
= *tcctx
.cb
.decl_map
->get (f
);
10880 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10883 sf
= (tree
) n
->value
;
10884 if (tcctx
.cb
.decl_map
)
10885 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10886 src
= build_simple_mem_ref_loc (loc
, sarg
);
10887 src
= omp_build_component_ref (src
, sf
);
10888 if (use_pointer_for_field (decl
, NULL
))
10889 src
= build_simple_mem_ref_loc (loc
, src
);
10893 dst
= build_simple_mem_ref_loc (loc
, arg
);
10894 dst
= omp_build_component_ref (dst
, f
);
10895 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10896 append_to_statement_list (t
, &list
);
10902 /* Last pass: handle VLA firstprivates. */
10903 if (tcctx
.cb
.decl_map
)
10904 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10905 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10909 decl
= OMP_CLAUSE_DECL (c
);
10910 if (!is_variable_sized (decl
))
10912 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10915 f
= (tree
) n
->value
;
10916 f
= *tcctx
.cb
.decl_map
->get (f
);
10917 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
10918 ind
= DECL_VALUE_EXPR (decl
);
10919 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
10920 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
10921 n
= splay_tree_lookup (ctx
->sfield_map
,
10922 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10923 sf
= (tree
) n
->value
;
10924 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10925 src
= build_simple_mem_ref_loc (loc
, sarg
);
10926 src
= omp_build_component_ref (src
, sf
);
10927 src
= build_simple_mem_ref_loc (loc
, src
);
10928 dst
= build_simple_mem_ref_loc (loc
, arg
);
10929 dst
= omp_build_component_ref (dst
, f
);
10930 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10931 append_to_statement_list (t
, &list
);
10932 n
= splay_tree_lookup (ctx
->field_map
,
10933 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10934 df
= (tree
) n
->value
;
10935 df
= *tcctx
.cb
.decl_map
->get (df
);
10936 ptr
= build_simple_mem_ref_loc (loc
, arg
);
10937 ptr
= omp_build_component_ref (ptr
, df
);
10938 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
10939 build_fold_addr_expr_loc (loc
, dst
));
10940 append_to_statement_list (t
, &list
);
10943 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
10944 append_to_statement_list (t
, &list
);
10946 if (tcctx
.cb
.decl_map
)
10947 delete tcctx
.cb
.decl_map
;
10948 pop_gimplify_context (NULL
);
10949 BIND_EXPR_BODY (bind
) = list
;
10954 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
10958 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
10960 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
10961 OMP_CLAUSE_DEPEND
);
10962 gcc_assert (clauses
);
10963 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10964 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
10965 switch (OMP_CLAUSE_DEPEND_KIND (c
))
10967 case OMP_CLAUSE_DEPEND_IN
:
10970 case OMP_CLAUSE_DEPEND_OUT
:
10971 case OMP_CLAUSE_DEPEND_INOUT
:
10975 gcc_unreachable ();
10977 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
10978 tree array
= create_tmp_var (type
, NULL
);
10979 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
10981 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
10982 gimple_seq_add_stmt (iseq
, g
);
10983 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
10985 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
10986 gimple_seq_add_stmt (iseq
, g
);
10987 for (i
= 0; i
< 2; i
++)
10989 if ((i
? n_in
: n_out
) == 0)
10991 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10992 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
10993 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
10995 tree t
= OMP_CLAUSE_DECL (c
);
10996 t
= fold_convert (ptr_type_node
, t
);
10997 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
10998 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
10999 NULL_TREE
, NULL_TREE
);
11000 g
= gimple_build_assign (r
, t
);
11001 gimple_seq_add_stmt (iseq
, g
);
11004 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
11005 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
11006 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
11007 OMP_CLAUSE_CHAIN (c
) = *p
;
11009 tree clobber
= build_constructor (type
, NULL
);
11010 TREE_THIS_VOLATILE (clobber
) = 1;
11011 g
= gimple_build_assign (array
, clobber
);
11012 gimple_seq_add_stmt (oseq
, g
);
11015 /* Lower the OpenMP parallel or task directive in the current statement
11016 in GSI_P. CTX holds context information for the directive. */
11019 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11023 gimple stmt
= gsi_stmt (*gsi_p
);
11024 gimple par_bind
, bind
, dep_bind
= NULL
;
11025 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
11026 location_t loc
= gimple_location (stmt
);
11028 clauses
= gimple_omp_taskreg_clauses (stmt
);
11029 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
11030 par_body
= gimple_bind_body (par_bind
);
11031 child_fn
= ctx
->cb
.dst_fn
;
11032 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11033 && !gimple_omp_parallel_combined_p (stmt
))
11035 struct walk_stmt_info wi
;
11038 memset (&wi
, 0, sizeof (wi
));
11040 wi
.val_only
= true;
11041 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
11043 gimple_omp_parallel_set_combined_p (stmt
, true);
11045 gimple_seq dep_ilist
= NULL
;
11046 gimple_seq dep_olist
= NULL
;
11047 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11048 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
11050 push_gimplify_context ();
11051 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11052 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
11055 if (ctx
->srecord_type
)
11056 create_task_copyfn (stmt
, ctx
);
11058 push_gimplify_context ();
11063 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
11064 lower_omp (&par_body
, ctx
);
11065 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
11066 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
11068 /* Declare all the variables created by mapping and the variables
11069 declared in the scope of the parallel body. */
11070 record_vars_into (ctx
->block_vars
, child_fn
);
11071 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
11073 if (ctx
->record_type
)
11076 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
11077 : ctx
->record_type
, ".omp_data_o");
11078 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11079 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11080 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
11085 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
11086 lower_send_shared_vars (&ilist
, &olist
, ctx
);
11088 if (ctx
->record_type
)
11090 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
11091 TREE_THIS_VOLATILE (clobber
) = 1;
11092 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11096 /* Once all the expansions are done, sequence all the different
11097 fragments inside gimple_omp_body. */
11101 if (ctx
->record_type
)
11103 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11104 /* fixup_child_record_type might have changed receiver_decl's type. */
11105 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11106 gimple_seq_add_stmt (&new_body
,
11107 gimple_build_assign (ctx
->receiver_decl
, t
));
11110 gimple_seq_add_seq (&new_body
, par_ilist
);
11111 gimple_seq_add_seq (&new_body
, par_body
);
11112 gimple_seq_add_seq (&new_body
, par_rlist
);
11113 if (ctx
->cancellable
)
11114 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
11115 gimple_seq_add_seq (&new_body
, par_olist
);
11116 new_body
= maybe_catch_exception (new_body
);
11117 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11118 gimple_omp_set_body (stmt
, new_body
);
11120 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
11121 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
11122 gimple_bind_add_seq (bind
, ilist
);
11123 gimple_bind_add_stmt (bind
, stmt
);
11124 gimple_bind_add_seq (bind
, olist
);
11126 pop_gimplify_context (NULL
);
11130 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11131 gimple_bind_add_stmt (dep_bind
, bind
);
11132 gimple_bind_add_seq (dep_bind
, dep_olist
);
11133 pop_gimplify_context (dep_bind
);
11137 /* Lower the GIMPLE_OMP_TARGET in the current statement
11138 in GSI_P. CTX holds context information for the directive. */
11141 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11144 tree child_fn
, t
, c
;
11145 gimple stmt
= gsi_stmt (*gsi_p
);
11146 gimple tgt_bind
, bind
;
11147 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
11148 location_t loc
= gimple_location (stmt
);
11149 bool offloaded
, data_region
;
11150 unsigned int map_cnt
= 0;
11151 tree (*gimple_omp_clauses
) (const_gimple
);
11152 void (*gimple_omp_set_data_arg
) (gimple
, tree
);
11154 offloaded
= is_gimple_omp_offloaded (stmt
);
11155 data_region
= false;
11156 switch (gimple_code (stmt
))
11158 case GIMPLE_OACC_KERNELS
:
11159 gimple_omp_clauses
= gimple_oacc_kernels_clauses
;
11160 gimple_omp_set_data_arg
= gimple_oacc_kernels_set_data_arg
;
11162 case GIMPLE_OACC_PARALLEL
:
11163 gimple_omp_clauses
= gimple_oacc_parallel_clauses
;
11164 gimple_omp_set_data_arg
= gimple_oacc_parallel_set_data_arg
;
11166 case GIMPLE_OMP_TARGET
:
11167 switch (gimple_omp_target_kind (stmt
))
11169 case GF_OMP_TARGET_KIND_DATA
:
11170 case GF_OMP_TARGET_KIND_OACC_DATA
:
11171 data_region
= true;
11173 case GF_OMP_TARGET_KIND_REGION
:
11174 case GF_OMP_TARGET_KIND_UPDATE
:
11175 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11176 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11179 gcc_unreachable ();
11182 gimple_omp_clauses
= gimple_omp_target_clauses
;
11183 gimple_omp_set_data_arg
= gimple_omp_target_set_data_arg
;
11186 gcc_unreachable ();
11189 clauses
= gimple_omp_clauses (stmt
);
11195 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
11196 tgt_body
= gimple_bind_body (tgt_bind
);
11198 else if (data_region
)
11199 tgt_body
= gimple_omp_body (stmt
);
11200 child_fn
= ctx
->cb
.dst_fn
;
11202 push_gimplify_context ();
11206 switch (gimple_code (stmt
))
11208 case GIMPLE_OACC_KERNELS
:
11209 case GIMPLE_OACC_PARALLEL
:
11210 process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
11216 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11217 switch (OMP_CLAUSE_CODE (c
))
11223 case OMP_CLAUSE_MAP
:
11224 #ifdef ENABLE_CHECKING
11225 /* First check what we're prepared to handle in the following. */
11226 switch (OMP_CLAUSE_MAP_KIND (c
))
11228 case OMP_CLAUSE_MAP_ALLOC
:
11229 case OMP_CLAUSE_MAP_TO
:
11230 case OMP_CLAUSE_MAP_FROM
:
11231 case OMP_CLAUSE_MAP_TOFROM
:
11232 case OMP_CLAUSE_MAP_POINTER
:
11233 case OMP_CLAUSE_MAP_TO_PSET
:
11235 case OMP_CLAUSE_MAP_FORCE_ALLOC
:
11236 case OMP_CLAUSE_MAP_FORCE_TO
:
11237 case OMP_CLAUSE_MAP_FORCE_FROM
:
11238 case OMP_CLAUSE_MAP_FORCE_TOFROM
:
11239 case OMP_CLAUSE_MAP_FORCE_PRESENT
:
11240 case OMP_CLAUSE_MAP_FORCE_DEALLOC
:
11241 case OMP_CLAUSE_MAP_FORCE_DEVICEPTR
:
11242 gcc_assert (is_gimple_omp_oacc_specifically (stmt
));
11245 gcc_unreachable ();
11249 case OMP_CLAUSE_TO
:
11250 case OMP_CLAUSE_FROM
:
11251 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
)
11252 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_TARGET
11253 && (gimple_omp_target_kind (stmt
)
11254 == GF_OMP_TARGET_KIND_UPDATE
));
11255 var
= OMP_CLAUSE_DECL (c
);
11258 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11259 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11264 if (DECL_SIZE (var
)
11265 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
11267 tree var2
= DECL_VALUE_EXPR (var
);
11268 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
11269 var2
= TREE_OPERAND (var2
, 0);
11270 gcc_assert (DECL_P (var2
));
11274 if (!maybe_lookup_field (var
, ctx
))
11279 x
= build_receiver_ref (var
, true, ctx
);
11280 tree new_var
= lookup_decl (var
, ctx
);
11281 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
);
11282 gcc_assert ((OMP_CLAUSE_MAP_KIND (c
)
11283 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
11284 || TREE_CODE (TREE_TYPE (var
)) != ARRAY_TYPE
);
11285 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
11286 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11287 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11288 x
= build_simple_mem_ref (x
);
11289 SET_DECL_VALUE_EXPR (new_var
, x
);
11290 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11297 target_nesting_level
++;
11298 lower_omp (&tgt_body
, ctx
);
11299 target_nesting_level
--;
11301 else if (data_region
)
11302 lower_omp (&tgt_body
, ctx
);
11306 /* Declare all the variables created by mapping and the variables
11307 declared in the scope of the target body. */
11308 record_vars_into (ctx
->block_vars
, child_fn
);
11309 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
11314 if (ctx
->record_type
)
11317 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
11318 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11319 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11320 t
= make_tree_vec (3);
11321 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
11322 TREE_VEC_ELT (t
, 1)
11323 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
11324 ".omp_data_sizes");
11325 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
11326 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
11327 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
11330 if (is_gimple_omp_oacc_specifically (stmt
))
11332 tkind_type
= short_unsigned_type_node
;
11337 tkind_type
= unsigned_char_type_node
;
11340 TREE_VEC_ELT (t
, 2)
11341 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
11342 ".omp_data_kinds");
11343 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
11344 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
11345 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
11346 gimple_omp_set_data_arg (stmt
, t
);
11348 vec
<constructor_elt
, va_gc
> *vsize
;
11349 vec
<constructor_elt
, va_gc
> *vkind
;
11350 vec_alloc (vsize
, map_cnt
);
11351 vec_alloc (vkind
, map_cnt
);
11352 unsigned int map_idx
= 0;
11354 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11355 switch (OMP_CLAUSE_CODE (c
))
11361 case OMP_CLAUSE_MAP
:
11362 case OMP_CLAUSE_TO
:
11363 case OMP_CLAUSE_FROM
:
11365 ovar
= OMP_CLAUSE_DECL (c
);
11366 if (!DECL_P (ovar
))
11368 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11369 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11371 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
11372 == get_base_address (ovar
));
11373 nc
= OMP_CLAUSE_CHAIN (c
);
11374 ovar
= OMP_CLAUSE_DECL (nc
);
11378 tree x
= build_sender_ref (ovar
, ctx
);
11380 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
11381 gimplify_assign (x
, v
, &ilist
);
11387 if (DECL_SIZE (ovar
)
11388 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
11390 tree ovar2
= DECL_VALUE_EXPR (ovar
);
11391 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
11392 ovar2
= TREE_OPERAND (ovar2
, 0);
11393 gcc_assert (DECL_P (ovar2
));
11396 if (!maybe_lookup_field (ovar
, ctx
))
11400 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
11401 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
11402 talign
= DECL_ALIGN_UNIT (ovar
);
11405 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11406 tree x
= build_sender_ref (ovar
, ctx
);
11407 gcc_assert (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11408 || (OMP_CLAUSE_MAP_KIND (c
)
11409 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
11410 || TREE_CODE (TREE_TYPE (ovar
)) != ARRAY_TYPE
);
11411 if (maybe_lookup_reduction (var
, ctx
))
11413 gcc_assert (gimple_code (stmt
) == GIMPLE_OACC_KERNELS
11414 || gimple_code (stmt
) == GIMPLE_OACC_PARALLEL
);
11415 gimplify_assign (x
, var
, &ilist
);
11417 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11418 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
11419 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11420 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
11422 gcc_assert (offloaded
);
11424 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
11425 mark_addressable (avar
);
11426 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
11427 talign
= DECL_ALIGN_UNIT (avar
);
11428 avar
= build_fold_addr_expr (avar
);
11429 gimplify_assign (x
, avar
, &ilist
);
11431 else if (is_gimple_reg (var
))
11433 gcc_assert (offloaded
);
11434 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
11435 mark_addressable (avar
);
11436 enum omp_clause_map_kind map_kind
11437 = OMP_CLAUSE_MAP_KIND (c
);
11438 if ((!(map_kind
& OMP_CLAUSE_MAP_SPECIAL
)
11439 && (map_kind
& OMP_CLAUSE_MAP_TO
))
11440 || map_kind
== OMP_CLAUSE_MAP_POINTER
11441 || map_kind
== OMP_CLAUSE_MAP_TO_PSET
11442 || map_kind
== OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
11443 gimplify_assign (avar
, var
, &ilist
);
11444 avar
= build_fold_addr_expr (avar
);
11445 gimplify_assign (x
, avar
, &ilist
);
11446 if (((!(map_kind
& OMP_CLAUSE_MAP_SPECIAL
)
11447 && (map_kind
& OMP_CLAUSE_MAP_FROM
))
11448 || map_kind
== OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
11449 && !TYPE_READONLY (TREE_TYPE (var
)))
11451 x
= build_sender_ref (ovar
, ctx
);
11452 x
= build_simple_mem_ref (x
);
11453 gimplify_assign (var
, x
, &olist
);
11458 var
= build_fold_addr_expr (var
);
11459 gimplify_assign (x
, var
, &ilist
);
11462 tree s
= OMP_CLAUSE_SIZE (c
);
11463 if (s
== NULL_TREE
)
11464 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11465 s
= fold_convert (size_type_node
, s
);
11466 tree purpose
= size_int (map_idx
++);
11467 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11468 if (TREE_CODE (s
) != INTEGER_CST
)
11469 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11471 unsigned HOST_WIDE_INT tkind
;
11472 switch (OMP_CLAUSE_CODE (c
))
11474 case OMP_CLAUSE_MAP
:
11475 tkind
= OMP_CLAUSE_MAP_KIND (c
);
11477 case OMP_CLAUSE_TO
:
11478 tkind
= OMP_CLAUSE_MAP_TO
;
11480 case OMP_CLAUSE_FROM
:
11481 tkind
= OMP_CLAUSE_MAP_FROM
;
11484 gcc_unreachable ();
11486 gcc_assert (tkind
< (HOST_WIDE_INT_C (1U) << talign_shift
));
11487 talign
= ceil_log2 (talign
);
11488 tkind
|= talign
<< talign_shift
;
11489 gcc_assert (tkind
<= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11490 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
11491 build_int_cstu (tkind_type
, tkind
));
11496 gcc_assert (map_idx
== map_cnt
);
11498 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
11499 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
11500 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
11501 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
11502 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
11504 gimple_seq initlist
= NULL
;
11505 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
11506 TREE_VEC_ELT (t
, 1)),
11507 &initlist
, true, NULL_TREE
);
11508 gimple_seq_add_seq (&ilist
, initlist
);
11510 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
11512 TREE_THIS_VOLATILE (clobber
) = 1;
11513 gimple_seq_add_stmt (&olist
,
11514 gimple_build_assign (TREE_VEC_ELT (t
, 1),
11518 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
11519 TREE_THIS_VOLATILE (clobber
) = 1;
11520 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11524 /* Once all the expansions are done, sequence all the different
11525 fragments inside gimple_omp_body. */
11530 && ctx
->record_type
)
11532 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11533 /* fixup_child_record_type might have changed receiver_decl's type. */
11534 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11535 gimple_seq_add_stmt (&new_body
,
11536 gimple_build_assign (ctx
->receiver_decl
, t
));
11541 gimple_seq_add_seq (&new_body
, tgt_body
);
11542 new_body
= maybe_catch_exception (new_body
);
11544 else if (data_region
)
11545 new_body
= tgt_body
;
11546 if (offloaded
|| data_region
)
11548 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11549 gimple_omp_set_body (stmt
, new_body
);
11552 bind
= gimple_build_bind (NULL
, NULL
,
11553 tgt_bind
? gimple_bind_block (tgt_bind
)
11555 gsi_replace (gsi_p
, bind
, true);
11556 gimple_bind_add_seq (bind
, irlist
);
11557 gimple_bind_add_seq (bind
, ilist
);
11558 gimple_bind_add_stmt (bind
, stmt
);
11559 gimple_bind_add_seq (bind
, olist
);
11560 gimple_bind_add_seq (bind
, orlist
);
11562 pop_gimplify_context (NULL
);
11565 /* Expand code for an OpenMP teams directive. */
11568 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11570 gimple teams_stmt
= gsi_stmt (*gsi_p
);
11571 push_gimplify_context ();
11573 tree block
= make_node (BLOCK
);
11574 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
11575 gsi_replace (gsi_p
, bind
, true);
11576 gimple_seq bind_body
= NULL
;
11577 gimple_seq dlist
= NULL
;
11578 gimple_seq olist
= NULL
;
11580 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11581 OMP_CLAUSE_NUM_TEAMS
);
11582 if (num_teams
== NULL_TREE
)
11583 num_teams
= build_int_cst (unsigned_type_node
, 0);
11586 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
11587 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
11588 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
11590 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11591 OMP_CLAUSE_THREAD_LIMIT
);
11592 if (thread_limit
== NULL_TREE
)
11593 thread_limit
= build_int_cst (unsigned_type_node
, 0);
11596 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
11597 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
11598 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
11602 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
11603 &bind_body
, &dlist
, ctx
, NULL
);
11604 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
11605 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
11606 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
11608 location_t loc
= gimple_location (teams_stmt
);
11609 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
11610 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
11611 gimple_set_location (call
, loc
);
11612 gimple_seq_add_stmt (&bind_body
, call
);
11614 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
11615 gimple_omp_set_body (teams_stmt
, NULL
);
11616 gimple_seq_add_seq (&bind_body
, olist
);
11617 gimple_seq_add_seq (&bind_body
, dlist
);
11618 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
11619 gimple_bind_set_body (bind
, bind_body
);
11621 pop_gimplify_context (bind
);
11623 gimple_bind_append_vars (bind
, ctx
->block_vars
);
11624 BLOCK_VARS (block
) = ctx
->block_vars
;
11625 if (BLOCK_VARS (block
))
11626 TREE_USED (block
) = 1;
11630 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11631 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11632 of OpenMP context, but with task_shared_vars set. */
11635 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
11640 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11641 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
11644 if (task_shared_vars
11646 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
11649 /* If a global variable has been privatized, TREE_CONSTANT on
11650 ADDR_EXPR might be wrong. */
11651 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
11652 recompute_tree_invariant_for_addr_expr (t
);
11654 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
11659 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11661 gimple stmt
= gsi_stmt (*gsi_p
);
11662 struct walk_stmt_info wi
;
11664 if (gimple_has_location (stmt
))
11665 input_location
= gimple_location (stmt
);
11667 if (task_shared_vars
)
11668 memset (&wi
, '\0', sizeof (wi
));
11670 /* If we have issued syntax errors, avoid doing any heavy lifting.
11671 Just replace the OpenMP directives with a NOP to avoid
11672 confusing RTL expansion. */
11673 if (seen_error () && is_gimple_omp (stmt
))
11675 gsi_replace (gsi_p
, gimple_build_nop (), true);
11679 switch (gimple_code (stmt
))
11682 if ((ctx
|| task_shared_vars
)
11683 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
11684 ctx
? NULL
: &wi
, NULL
)
11685 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
11686 ctx
? NULL
: &wi
, NULL
)))
11687 gimple_regimplify_operands (stmt
, gsi_p
);
11690 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
11692 case GIMPLE_EH_FILTER
:
11693 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
11696 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
11697 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
11699 case GIMPLE_TRANSACTION
:
11700 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
11703 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
11705 case GIMPLE_OMP_PARALLEL
:
11706 case GIMPLE_OMP_TASK
:
11707 ctx
= maybe_lookup_ctx (stmt
);
11709 if (ctx
->cancellable
)
11710 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11711 lower_omp_taskreg (gsi_p
, ctx
);
11713 case GIMPLE_OMP_FOR
:
11714 ctx
= maybe_lookup_ctx (stmt
);
11716 if (ctx
->cancellable
)
11717 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11718 lower_omp_for (gsi_p
, ctx
);
11720 case GIMPLE_OMP_SECTIONS
:
11721 ctx
= maybe_lookup_ctx (stmt
);
11723 if (ctx
->cancellable
)
11724 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11725 lower_omp_sections (gsi_p
, ctx
);
11727 case GIMPLE_OMP_SINGLE
:
11728 ctx
= maybe_lookup_ctx (stmt
);
11730 lower_omp_single (gsi_p
, ctx
);
11732 case GIMPLE_OMP_MASTER
:
11733 ctx
= maybe_lookup_ctx (stmt
);
11735 lower_omp_master (gsi_p
, ctx
);
11737 case GIMPLE_OMP_TASKGROUP
:
11738 ctx
= maybe_lookup_ctx (stmt
);
11740 lower_omp_taskgroup (gsi_p
, ctx
);
11742 case GIMPLE_OMP_ORDERED
:
11743 ctx
= maybe_lookup_ctx (stmt
);
11745 lower_omp_ordered (gsi_p
, ctx
);
11747 case GIMPLE_OMP_CRITICAL
:
11748 ctx
= maybe_lookup_ctx (stmt
);
11750 lower_omp_critical (gsi_p
, ctx
);
11752 case GIMPLE_OMP_ATOMIC_LOAD
:
11753 if ((ctx
|| task_shared_vars
)
11754 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
11755 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
11756 gimple_regimplify_operands (stmt
, gsi_p
);
11758 case GIMPLE_OACC_KERNELS
:
11759 case GIMPLE_OACC_PARALLEL
:
11760 case GIMPLE_OMP_TARGET
:
11761 ctx
= maybe_lookup_ctx (stmt
);
11763 if (is_gimple_omp_oacc_specifically (stmt
))
11764 gcc_assert (!ctx
->cancellable
);
11765 lower_omp_target (gsi_p
, ctx
);
11767 case GIMPLE_OMP_TEAMS
:
11768 ctx
= maybe_lookup_ctx (stmt
);
11770 lower_omp_teams (gsi_p
, ctx
);
11774 fndecl
= gimple_call_fndecl (stmt
);
11776 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
11777 switch (DECL_FUNCTION_CODE (fndecl
))
11779 case BUILT_IN_GOMP_BARRIER
:
11783 case BUILT_IN_GOMP_CANCEL
:
11784 case BUILT_IN_GOMP_CANCELLATION_POINT
:
11787 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
11788 cctx
= cctx
->outer
;
11789 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
11790 if (!cctx
->cancellable
)
11792 if (DECL_FUNCTION_CODE (fndecl
)
11793 == BUILT_IN_GOMP_CANCELLATION_POINT
)
11795 stmt
= gimple_build_nop ();
11796 gsi_replace (gsi_p
, stmt
, false);
11800 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
11802 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
11803 gimple_call_set_fndecl (stmt
, fndecl
);
11804 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
11807 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)), NULL
);
11808 gimple_call_set_lhs (stmt
, lhs
);
11809 tree fallthru_label
;
11810 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
11812 g
= gimple_build_label (fallthru_label
);
11813 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11814 g
= gimple_build_cond (NE_EXPR
, lhs
,
11815 fold_convert (TREE_TYPE (lhs
),
11816 boolean_false_node
),
11817 cctx
->cancel_label
, fallthru_label
);
11818 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11825 if ((ctx
|| task_shared_vars
)
11826 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
11829 /* Just remove clobbers, this should happen only if we have
11830 "privatized" local addressable variables in SIMD regions,
11831 the clobber isn't needed in that case and gimplifying address
11832 of the ARRAY_REF into a pointer and creating MEM_REF based
11833 clobber would create worse code than we get with the clobber
11835 if (gimple_clobber_p (stmt
))
11837 gsi_replace (gsi_p
, gimple_build_nop (), true);
11840 gimple_regimplify_operands (stmt
, gsi_p
);
11847 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
11849 location_t saved_location
= input_location
;
11850 gimple_stmt_iterator gsi
;
11851 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11852 lower_omp_1 (&gsi
, ctx
);
11853 /* During gimplification, we haven't folded statments inside offloading
11854 regions (gimplify.c:maybe_fold_stmt); do that now. */
11855 if (target_nesting_level
)
11856 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11858 input_location
= saved_location
;
11861 /* Main entry point. */
11863 static unsigned int
11864 execute_lower_omp (void)
11870 /* This pass always runs, to provide PROP_gimple_lomp.
11871 But often, there is nothing to do. */
11872 if (flag_openacc
== 0 && flag_openmp
== 0 && flag_openmp_simd
== 0
11873 && flag_cilkplus
== 0)
11876 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
11877 delete_omp_context
);
11879 body
= gimple_body (current_function_decl
);
11880 scan_omp (&body
, NULL
);
11881 gcc_assert (taskreg_nesting_level
== 0);
11882 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
11883 finish_taskreg_scan (ctx
);
11884 taskreg_contexts
.release ();
11886 if (all_contexts
->root
)
11888 if (task_shared_vars
)
11889 push_gimplify_context ();
11890 lower_omp (&body
, NULL
);
11891 if (task_shared_vars
)
11892 pop_gimplify_context (NULL
);
11897 splay_tree_delete (all_contexts
);
11898 all_contexts
= NULL
;
11900 BITMAP_FREE (task_shared_vars
);
11906 const pass_data pass_data_lower_omp
=
11908 GIMPLE_PASS
, /* type */
11909 "omplower", /* name */
11910 OPTGROUP_NONE
, /* optinfo_flags */
11911 TV_NONE
, /* tv_id */
11912 PROP_gimple_any
, /* properties_required */
11913 PROP_gimple_lomp
, /* properties_provided */
11914 0, /* properties_destroyed */
11915 0, /* todo_flags_start */
11916 0, /* todo_flags_finish */
11919 class pass_lower_omp
: public gimple_opt_pass
11922 pass_lower_omp (gcc::context
*ctxt
)
11923 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
11926 /* opt_pass methods: */
11927 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
11929 }; // class pass_lower_omp
11931 } // anon namespace
11934 make_pass_lower_omp (gcc::context
*ctxt
)
11936 return new pass_lower_omp (ctxt
);
11939 /* The following is a utility to diagnose structured block violations.
11940 It is not part of the "omplower" pass, as that's invoked too late. It
11941 should be invoked by the respective front ends after gimplification. */
11943 static splay_tree all_labels
;
11945 /* Check for mismatched contexts and generate an error if needed. Return
11946 true if an error is detected. */
11949 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
11950 gimple branch_ctx
, gimple label_ctx
)
11952 gcc_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
11953 gcc_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
11955 if (label_ctx
== branch_ctx
)
11958 const char* kind
= NULL
;
11963 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
11964 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
11966 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
11967 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
11968 kind
= "Cilk Plus";
11972 if ((branch_ctx
&& is_gimple_omp_oacc_specifically (branch_ctx
))
11973 || (label_ctx
&& is_gimple_omp_oacc_specifically (label_ctx
)))
11975 gcc_assert (kind
== NULL
);
11981 gcc_assert (flag_openmp
);
11986 Previously we kept track of the label's entire context in diagnose_sb_[12]
11987 so we could traverse it and issue a correct "exit" or "enter" error
11988 message upon a structured block violation.
11990 We built the context by building a list with tree_cons'ing, but there is
11991 no easy counterpart in gimple tuples. It seems like far too much work
11992 for issuing exit/enter error messages. If someone really misses the
11993 distinct error message... patches welcome.
11997 /* Try to avoid confusing the user by producing and error message
11998 with correct "exit" or "enter" verbiage. We prefer "exit"
11999 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12000 if (branch_ctx
== NULL
)
12006 if (TREE_VALUE (label_ctx
) == branch_ctx
)
12011 label_ctx
= TREE_CHAIN (label_ctx
);
12016 error ("invalid exit from %s structured block", kind
);
12018 error ("invalid entry to %s structured block", kind
);
12021 /* If it's obvious we have an invalid entry, be specific about the error. */
12022 if (branch_ctx
== NULL
)
12023 error ("invalid entry to %s structured block", kind
);
12026 /* Otherwise, be vague and lazy, but efficient. */
12027 error ("invalid branch to/from %s structured block", kind
);
12030 gsi_replace (gsi_p
, gimple_build_nop (), false);
12034 /* Pass 1: Create a minimal tree of structured blocks, and record
12035 where each label is found. */
12038 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12039 struct walk_stmt_info
*wi
)
12041 gimple context
= (gimple
) wi
->info
;
12042 gimple inner_context
;
12043 gimple stmt
= gsi_stmt (*gsi_p
);
12045 *handled_ops_p
= true;
12047 switch (gimple_code (stmt
))
12051 case GIMPLE_OACC_KERNELS
:
12052 case GIMPLE_OACC_PARALLEL
:
12053 case GIMPLE_OMP_PARALLEL
:
12054 case GIMPLE_OMP_TASK
:
12055 case GIMPLE_OMP_SECTIONS
:
12056 case GIMPLE_OMP_SINGLE
:
12057 case GIMPLE_OMP_SECTION
:
12058 case GIMPLE_OMP_MASTER
:
12059 case GIMPLE_OMP_ORDERED
:
12060 case GIMPLE_OMP_CRITICAL
:
12061 case GIMPLE_OMP_TARGET
:
12062 case GIMPLE_OMP_TEAMS
:
12063 case GIMPLE_OMP_TASKGROUP
:
12064 /* The minimal context here is just the current construct. */
12065 inner_context
= stmt
;
12066 wi
->info
= inner_context
;
12067 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12068 wi
->info
= context
;
12071 case GIMPLE_OMP_FOR
:
12072 inner_context
= stmt
;
12073 wi
->info
= inner_context
;
12074 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12076 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
12077 diagnose_sb_1
, NULL
, wi
);
12078 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12079 wi
->info
= context
;
12083 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
12084 (splay_tree_value
) context
);
12094 /* Pass 2: Check each branch and see if its context differs from that of
12095 the destination label's context. */
12098 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12099 struct walk_stmt_info
*wi
)
12101 gimple context
= (gimple
) wi
->info
;
12103 gimple stmt
= gsi_stmt (*gsi_p
);
12105 *handled_ops_p
= true;
12107 switch (gimple_code (stmt
))
12111 case GIMPLE_OACC_KERNELS
:
12112 case GIMPLE_OACC_PARALLEL
:
12113 case GIMPLE_OMP_PARALLEL
:
12114 case GIMPLE_OMP_TASK
:
12115 case GIMPLE_OMP_SECTIONS
:
12116 case GIMPLE_OMP_SINGLE
:
12117 case GIMPLE_OMP_SECTION
:
12118 case GIMPLE_OMP_MASTER
:
12119 case GIMPLE_OMP_ORDERED
:
12120 case GIMPLE_OMP_CRITICAL
:
12121 case GIMPLE_OMP_TARGET
:
12122 case GIMPLE_OMP_TEAMS
:
12123 case GIMPLE_OMP_TASKGROUP
:
12125 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12126 wi
->info
= context
;
12129 case GIMPLE_OMP_FOR
:
12131 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12133 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
12134 diagnose_sb_2
, NULL
, wi
);
12135 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12136 wi
->info
= context
;
12141 tree lab
= gimple_cond_true_label (stmt
);
12144 n
= splay_tree_lookup (all_labels
,
12145 (splay_tree_key
) lab
);
12146 diagnose_sb_0 (gsi_p
, context
,
12147 n
? (gimple
) n
->value
: NULL
);
12149 lab
= gimple_cond_false_label (stmt
);
12152 n
= splay_tree_lookup (all_labels
,
12153 (splay_tree_key
) lab
);
12154 diagnose_sb_0 (gsi_p
, context
,
12155 n
? (gimple
) n
->value
: NULL
);
12162 tree lab
= gimple_goto_dest (stmt
);
12163 if (TREE_CODE (lab
) != LABEL_DECL
)
12166 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12167 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
12171 case GIMPLE_SWITCH
:
12174 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
12176 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
12177 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12178 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
12184 case GIMPLE_RETURN
:
12185 diagnose_sb_0 (gsi_p
, context
, NULL
);
12195 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12198 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
12201 gimple last
= last_stmt (bb
);
12202 enum gimple_code code
= gimple_code (last
);
12203 struct omp_region
*cur_region
= *region
;
12204 bool fallthru
= false;
12208 case GIMPLE_OACC_KERNELS
:
12209 case GIMPLE_OACC_PARALLEL
:
12210 case GIMPLE_OMP_PARALLEL
:
12211 case GIMPLE_OMP_TASK
:
12212 case GIMPLE_OMP_FOR
:
12213 case GIMPLE_OMP_SINGLE
:
12214 case GIMPLE_OMP_TEAMS
:
12215 case GIMPLE_OMP_MASTER
:
12216 case GIMPLE_OMP_TASKGROUP
:
12217 case GIMPLE_OMP_ORDERED
:
12218 case GIMPLE_OMP_CRITICAL
:
12219 case GIMPLE_OMP_SECTION
:
12220 cur_region
= new_omp_region (bb
, code
, cur_region
);
12224 case GIMPLE_OMP_TARGET
:
12225 cur_region
= new_omp_region (bb
, code
, cur_region
);
12227 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
12228 || gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_OACC_UPDATE
)
12229 cur_region
= cur_region
->outer
;
12232 case GIMPLE_OMP_SECTIONS
:
12233 cur_region
= new_omp_region (bb
, code
, cur_region
);
12237 case GIMPLE_OMP_SECTIONS_SWITCH
:
12241 case GIMPLE_OMP_ATOMIC_LOAD
:
12242 case GIMPLE_OMP_ATOMIC_STORE
:
12246 case GIMPLE_OMP_RETURN
:
12247 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12248 somewhere other than the next block. This will be
12250 cur_region
->exit
= bb
;
12251 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
12252 cur_region
= cur_region
->outer
;
12255 case GIMPLE_OMP_CONTINUE
:
12256 cur_region
->cont
= bb
;
12257 switch (cur_region
->type
)
12259 case GIMPLE_OMP_FOR
:
12260 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12261 succs edges as abnormal to prevent splitting
12263 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
12264 /* Make the loopback edge. */
12265 make_edge (bb
, single_succ (cur_region
->entry
),
12268 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12269 corresponds to the case that the body of the loop
12270 is not executed at all. */
12271 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
12272 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
12276 case GIMPLE_OMP_SECTIONS
:
12277 /* Wire up the edges into and out of the nested sections. */
12279 basic_block switch_bb
= single_succ (cur_region
->entry
);
12281 struct omp_region
*i
;
12282 for (i
= cur_region
->inner
; i
; i
= i
->next
)
12284 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
12285 make_edge (switch_bb
, i
->entry
, 0);
12286 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
12289 /* Make the loopback edge to the block with
12290 GIMPLE_OMP_SECTIONS_SWITCH. */
12291 make_edge (bb
, switch_bb
, 0);
12293 /* Make the edge from the switch to exit. */
12294 make_edge (switch_bb
, bb
->next_bb
, 0);
12300 gcc_unreachable ();
12305 gcc_unreachable ();
12308 if (*region
!= cur_region
)
12310 *region
= cur_region
;
12312 *region_idx
= cur_region
->entry
->index
;
12320 static unsigned int
12321 diagnose_omp_structured_block_errors (void)
12323 struct walk_stmt_info wi
;
12324 gimple_seq body
= gimple_body (current_function_decl
);
12326 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
12328 memset (&wi
, 0, sizeof (wi
));
12329 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
12331 memset (&wi
, 0, sizeof (wi
));
12332 wi
.want_locations
= true;
12333 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
12335 gimple_set_body (current_function_decl
, body
);
12337 splay_tree_delete (all_labels
);
12345 const pass_data pass_data_diagnose_omp_blocks
=
12347 GIMPLE_PASS
, /* type */
12348 "*diagnose_omp_blocks", /* name */
12349 OPTGROUP_NONE
, /* optinfo_flags */
12350 TV_NONE
, /* tv_id */
12351 PROP_gimple_any
, /* properties_required */
12352 0, /* properties_provided */
12353 0, /* properties_destroyed */
12354 0, /* todo_flags_start */
12355 0, /* todo_flags_finish */
12358 class pass_diagnose_omp_blocks
: public gimple_opt_pass
12361 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12362 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
12365 /* opt_pass methods: */
12366 virtual bool gate (function
*)
12368 return flag_openacc
|| flag_openmp
|| flag_cilkplus
;
12370 virtual unsigned int execute (function
*)
12372 return diagnose_omp_structured_block_errors ();
12375 }; // class pass_diagnose_omp_blocks
12377 } // anon namespace
12380 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12382 return new pass_diagnose_omp_blocks (ctxt
);
12385 /* SIMD clone supporting code. */
12387 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12388 of arguments to reserve space for. */
12390 static struct cgraph_simd_clone
*
12391 simd_clone_struct_alloc (int nargs
)
12393 struct cgraph_simd_clone
*clone_info
;
12394 size_t len
= (sizeof (struct cgraph_simd_clone
)
12395 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
12396 clone_info
= (struct cgraph_simd_clone
*)
12397 ggc_internal_cleared_alloc (len
);
12401 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12404 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
12405 struct cgraph_simd_clone
*from
)
12407 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
12408 + ((from
->nargs
- from
->inbranch
)
12409 * sizeof (struct cgraph_simd_clone_arg
))));
12412 /* Return vector of parameter types of function FNDECL. This uses
12413 TYPE_ARG_TYPES if available, otherwise falls back to types of
12414 DECL_ARGUMENTS types. */
12417 simd_clone_vector_of_formal_parm_types (tree fndecl
)
12419 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
12420 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
12421 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
12424 FOR_EACH_VEC_ELT (args
, i
, arg
)
12425 args
[i
] = TREE_TYPE (args
[i
]);
12429 /* Given a simd function in NODE, extract the simd specific
12430 information from the OMP clauses passed in CLAUSES, and return
12431 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12432 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12433 otherwise set to FALSE. */
12435 static struct cgraph_simd_clone
*
12436 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
12437 bool *inbranch_specified
)
12439 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12442 *inbranch_specified
= false;
12444 n
= args
.length ();
12445 if (n
> 0 && args
.last () == void_type_node
)
12448 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12449 be cloned have a distinctive artificial label in addition to "omp
12453 && lookup_attribute ("cilk simd function",
12454 DECL_ATTRIBUTES (node
->decl
)));
12456 /* Allocate one more than needed just in case this is an in-branch
12457 clone which will require a mask argument. */
12458 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
12459 clone_info
->nargs
= n
;
12460 clone_info
->cilk_elemental
= cilk_clone
;
12467 clauses
= TREE_VALUE (clauses
);
12468 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
12471 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
12473 switch (OMP_CLAUSE_CODE (t
))
12475 case OMP_CLAUSE_INBRANCH
:
12476 clone_info
->inbranch
= 1;
12477 *inbranch_specified
= true;
12479 case OMP_CLAUSE_NOTINBRANCH
:
12480 clone_info
->inbranch
= 0;
12481 *inbranch_specified
= true;
12483 case OMP_CLAUSE_SIMDLEN
:
12484 clone_info
->simdlen
12485 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
12487 case OMP_CLAUSE_LINEAR
:
12489 tree decl
= OMP_CLAUSE_DECL (t
);
12490 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
12491 int argno
= TREE_INT_CST_LOW (decl
);
12492 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
12494 clone_info
->args
[argno
].arg_type
12495 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
12496 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12497 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
12498 && clone_info
->args
[argno
].linear_step
< n
);
12502 if (POINTER_TYPE_P (args
[argno
]))
12503 step
= fold_convert (ssizetype
, step
);
12504 if (!tree_fits_shwi_p (step
))
12506 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12507 "ignoring large linear step");
12511 else if (integer_zerop (step
))
12513 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12514 "ignoring zero linear step");
12520 clone_info
->args
[argno
].arg_type
12521 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
12522 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12527 case OMP_CLAUSE_UNIFORM
:
12529 tree decl
= OMP_CLAUSE_DECL (t
);
12530 int argno
= tree_to_uhwi (decl
);
12531 clone_info
->args
[argno
].arg_type
12532 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
12535 case OMP_CLAUSE_ALIGNED
:
12537 tree decl
= OMP_CLAUSE_DECL (t
);
12538 int argno
= tree_to_uhwi (decl
);
12539 clone_info
->args
[argno
].alignment
12540 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
12551 /* Given a SIMD clone in NODE, calculate the characteristic data
12552 type and return the coresponding type. The characteristic data
12553 type is computed as described in the Intel Vector ABI. */
12556 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
12557 struct cgraph_simd_clone
*clone_info
)
12559 tree type
= integer_type_node
;
12560 tree fndecl
= node
->decl
;
12562 /* a) For non-void function, the characteristic data type is the
12564 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
12565 type
= TREE_TYPE (TREE_TYPE (fndecl
));
12567 /* b) If the function has any non-uniform, non-linear parameters,
12568 then the characteristic data type is the type of the first
12572 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
12573 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
12574 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
12582 /* c) If the characteristic data type determined by a) or b) above
12583 is struct, union, or class type which is pass-by-value (except
12584 for the type that maps to the built-in complex data type), the
12585 characteristic data type is int. */
12586 if (RECORD_OR_UNION_TYPE_P (type
)
12587 && !aggregate_value_p (type
, NULL
)
12588 && TREE_CODE (type
) != COMPLEX_TYPE
)
12589 return integer_type_node
;
12591 /* d) If none of the above three classes is applicable, the
12592 characteristic data type is int. */
12596 /* e) For Intel Xeon Phi native and offload compilation, if the
12597 resulting characteristic data type is 8-bit or 16-bit integer
12598 data type, the characteristic data type is int. */
12599 /* Well, we don't handle Xeon Phi yet. */
12603 simd_clone_mangle (struct cgraph_node
*node
,
12604 struct cgraph_simd_clone
*clone_info
)
12606 char vecsize_mangle
= clone_info
->vecsize_mangle
;
12607 char mask
= clone_info
->inbranch
? 'M' : 'N';
12608 unsigned int simdlen
= clone_info
->simdlen
;
12612 gcc_assert (vecsize_mangle
&& simdlen
);
12614 pp_string (&pp
, "_ZGV");
12615 pp_character (&pp
, vecsize_mangle
);
12616 pp_character (&pp
, mask
);
12617 pp_decimal_int (&pp
, simdlen
);
12619 for (n
= 0; n
< clone_info
->nargs
; ++n
)
12621 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
12623 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
12624 pp_character (&pp
, 'u');
12625 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12627 gcc_assert (arg
.linear_step
!= 0);
12628 pp_character (&pp
, 'l');
12629 if (arg
.linear_step
> 1)
12630 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12631 else if (arg
.linear_step
< 0)
12633 pp_character (&pp
, 'n');
12634 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
12638 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
12640 pp_character (&pp
, 's');
12641 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12644 pp_character (&pp
, 'v');
12647 pp_character (&pp
, 'a');
12648 pp_decimal_int (&pp
, arg
.alignment
);
12652 pp_underscore (&pp
);
12654 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
12655 const char *str
= pp_formatted_text (&pp
);
12657 /* If there already is a SIMD clone with the same mangled name, don't
12658 add another one. This can happen e.g. for
12659 #pragma omp declare simd
12660 #pragma omp declare simd simdlen(8)
12661 int foo (int, int);
12662 if the simdlen is assumed to be 8 for the first one, etc. */
12663 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
12664 clone
= clone
->simdclone
->next_clone
)
12665 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
12669 return get_identifier (str
);
12672 /* Create a simd clone of OLD_NODE and return it. */
12674 static struct cgraph_node
*
12675 simd_clone_create (struct cgraph_node
*old_node
)
12677 struct cgraph_node
*new_node
;
12678 if (old_node
->definition
)
12680 if (!old_node
->has_gimple_body_p ())
12682 old_node
->get_body ();
12683 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
12689 tree old_decl
= old_node
->decl
;
12690 tree new_decl
= copy_node (old_node
->decl
);
12691 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
12692 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
12693 SET_DECL_RTL (new_decl
, NULL
);
12694 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
12695 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
12696 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
12697 symtab
->call_cgraph_insertion_hooks (new_node
);
12699 if (new_node
== NULL
)
12702 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
12704 /* The function cgraph_function_versioning () will force the new
12705 symbol local. Undo this, and inherit external visability from
12707 new_node
->local
.local
= old_node
->local
.local
;
12708 new_node
->externally_visible
= old_node
->externally_visible
;
12713 /* Adjust the return type of the given function to its appropriate
12714 vector counterpart. Returns a simd array to be used throughout the
12715 function as a return value. */
12718 simd_clone_adjust_return_type (struct cgraph_node
*node
)
12720 tree fndecl
= node
->decl
;
12721 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
12722 unsigned int veclen
;
12725 /* Adjust the function return type. */
12726 if (orig_rettype
== void_type_node
)
12728 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
12729 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
)))
12730 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
))))
12731 veclen
= node
->simdclone
->vecsize_int
;
12733 veclen
= node
->simdclone
->vecsize_float
;
12734 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl
))));
12735 if (veclen
> node
->simdclone
->simdlen
)
12736 veclen
= node
->simdclone
->simdlen
;
12737 if (veclen
== node
->simdclone
->simdlen
)
12738 TREE_TYPE (TREE_TYPE (fndecl
))
12739 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)),
12740 node
->simdclone
->simdlen
);
12743 t
= build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)), veclen
);
12744 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
12745 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
12747 if (!node
->definition
)
12750 t
= DECL_RESULT (fndecl
);
12751 /* Adjust the DECL_RESULT. */
12752 gcc_assert (TREE_TYPE (t
) != void_type_node
);
12753 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
12756 tree atype
= build_array_type_nelts (orig_rettype
,
12757 node
->simdclone
->simdlen
);
12758 if (veclen
!= node
->simdclone
->simdlen
)
12759 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
12761 /* Set up a SIMD array to use as the return value. */
12762 tree retval
= create_tmp_var_raw (atype
, "retval");
12763 gimple_add_tmp_var (retval
);
12767 /* Each vector argument has a corresponding array to be used locally
12768 as part of the eventual loop. Create such temporary array and
12771 PREFIX is the prefix to be used for the temporary.
12773 TYPE is the inner element type.
12775 SIMDLEN is the number of elements. */
12778 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
12780 tree atype
= build_array_type_nelts (type
, simdlen
);
12781 tree avar
= create_tmp_var_raw (atype
, prefix
);
12782 gimple_add_tmp_var (avar
);
12786 /* Modify the function argument types to their corresponding vector
12787 counterparts if appropriate. Also, create one array for each simd
12788 argument to be used locally when using the function arguments as
12791 NODE is the function whose arguments are to be adjusted.
12793 Returns an adjustment vector that will be filled describing how the
12794 argument types will be adjusted. */
12796 static ipa_parm_adjustment_vec
12797 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
12800 ipa_parm_adjustment_vec adjustments
;
12802 if (node
->definition
)
12803 args
= ipa_get_vector_of_formal_parms (node
->decl
);
12805 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12806 adjustments
.create (args
.length ());
12807 unsigned i
, j
, veclen
;
12808 struct ipa_parm_adjustment adj
;
12809 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
12811 memset (&adj
, 0, sizeof (adj
));
12812 tree parm
= args
[i
];
12813 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
12814 adj
.base_index
= i
;
12817 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
12818 node
->simdclone
->args
[i
].orig_type
= parm_type
;
12820 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
12822 /* No adjustment necessary for scalar arguments. */
12823 adj
.op
= IPA_PARM_OP_COPY
;
12827 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
12828 veclen
= node
->simdclone
->vecsize_int
;
12830 veclen
= node
->simdclone
->vecsize_float
;
12831 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
12832 if (veclen
> node
->simdclone
->simdlen
)
12833 veclen
= node
->simdclone
->simdlen
;
12834 adj
.arg_prefix
= "simd";
12835 adj
.type
= build_vector_type (parm_type
, veclen
);
12836 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
12837 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12839 adjustments
.safe_push (adj
);
12842 memset (&adj
, 0, sizeof (adj
));
12843 adj
.op
= IPA_PARM_OP_NEW
;
12844 adj
.arg_prefix
= "simd";
12845 adj
.base_index
= i
;
12846 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
12850 if (node
->definition
)
12851 node
->simdclone
->args
[i
].simd_array
12852 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
12853 parm_type
, node
->simdclone
->simdlen
);
12855 adjustments
.safe_push (adj
);
12858 if (node
->simdclone
->inbranch
)
12861 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
12864 memset (&adj
, 0, sizeof (adj
));
12865 adj
.op
= IPA_PARM_OP_NEW
;
12866 adj
.arg_prefix
= "mask";
12868 adj
.base_index
= i
;
12869 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
12870 veclen
= node
->simdclone
->vecsize_int
;
12872 veclen
= node
->simdclone
->vecsize_float
;
12873 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
12874 if (veclen
> node
->simdclone
->simdlen
)
12875 veclen
= node
->simdclone
->simdlen
;
12876 adj
.type
= build_vector_type (base_type
, veclen
);
12877 adjustments
.safe_push (adj
);
12879 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12880 adjustments
.safe_push (adj
);
12882 /* We have previously allocated one extra entry for the mask. Use
12884 struct cgraph_simd_clone
*sc
= node
->simdclone
;
12886 if (node
->definition
)
12888 sc
->args
[i
].orig_arg
12889 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
12890 sc
->args
[i
].simd_array
12891 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
12893 sc
->args
[i
].orig_type
= base_type
;
12894 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
12897 if (node
->definition
)
12898 ipa_modify_formal_parameters (node
->decl
, adjustments
);
12901 tree new_arg_types
= NULL_TREE
, new_reversed
;
12902 bool last_parm_void
= false;
12903 if (args
.length () > 0 && args
.last () == void_type_node
)
12904 last_parm_void
= true;
12906 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
12907 j
= adjustments
.length ();
12908 for (i
= 0; i
< j
; i
++)
12910 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
12912 if (adj
->op
== IPA_PARM_OP_COPY
)
12913 ptype
= args
[adj
->base_index
];
12916 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
12918 new_reversed
= nreverse (new_arg_types
);
12919 if (last_parm_void
)
12922 TREE_CHAIN (new_arg_types
) = void_list_node
;
12924 new_reversed
= void_list_node
;
12927 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
12928 TYPE_ARG_TYPES (new_type
) = new_reversed
;
12929 TREE_TYPE (node
->decl
) = new_type
;
12931 adjustments
.release ();
12934 return adjustments
;
12937 /* Initialize and copy the function arguments in NODE to their
12938 corresponding local simd arrays. Returns a fresh gimple_seq with
12939 the instruction sequence generated. */
12942 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
12943 ipa_parm_adjustment_vec adjustments
)
12945 gimple_seq seq
= NULL
;
12946 unsigned i
= 0, j
= 0, k
;
12948 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
12950 arg
= DECL_CHAIN (arg
), i
++, j
++)
12952 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
12955 node
->simdclone
->args
[i
].vector_arg
= arg
;
12957 tree array
= node
->simdclone
->args
[i
].simd_array
;
12958 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
12960 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12961 tree ptr
= build_fold_addr_expr (array
);
12962 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
12963 build_int_cst (ptype
, 0));
12964 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
12965 gimplify_and_add (t
, &seq
);
12969 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
12970 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12971 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
12973 tree ptr
= build_fold_addr_expr (array
);
12977 arg
= DECL_CHAIN (arg
);
12981 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
12982 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
12983 build_int_cst (ptype
, k
* elemsize
));
12984 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
12985 gimplify_and_add (t
, &seq
);
12992 /* Callback info for ipa_simd_modify_stmt_ops below. */
12994 struct modify_stmt_info
{
12995 ipa_parm_adjustment_vec adjustments
;
12997 /* True if the parent statement was modified by
12998 ipa_simd_modify_stmt_ops. */
13002 /* Callback for walk_gimple_op.
13004 Adjust operands from a given statement as specified in the
13005 adjustments vector in the callback data. */
13008 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
13010 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13011 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
13012 tree
*orig_tp
= tp
;
13013 if (TREE_CODE (*tp
) == ADDR_EXPR
)
13014 tp
= &TREE_OPERAND (*tp
, 0);
13015 struct ipa_parm_adjustment
*cand
= NULL
;
13016 if (TREE_CODE (*tp
) == PARM_DECL
)
13017 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
13021 *walk_subtrees
= 0;
13024 tree repl
= NULL_TREE
;
13026 repl
= unshare_expr (cand
->new_decl
);
13031 *walk_subtrees
= 0;
13032 bool modified
= info
->modified
;
13033 info
->modified
= false;
13034 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
13035 if (!info
->modified
)
13037 info
->modified
= modified
;
13040 info
->modified
= modified
;
13049 repl
= build_fold_addr_expr (repl
);
13051 if (is_gimple_debug (info
->stmt
))
13053 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
13054 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
13055 DECL_ARTIFICIAL (vexpr
) = 1;
13056 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
13057 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
13062 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
),
13064 repl
= gimple_assign_lhs (stmt
);
13066 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
13067 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13070 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
13072 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
13078 info
->modified
= true;
13082 /* Traverse the function body and perform all modifications as
13083 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13084 modified such that the replacement/reduction value will now be an
13085 offset into the corresponding simd_array.
13087 This function will replace all function argument uses with their
13088 corresponding simd array elements, and ajust the return values
13092 ipa_simd_modify_function_body (struct cgraph_node
*node
,
13093 ipa_parm_adjustment_vec adjustments
,
13094 tree retval_array
, tree iter
)
13097 unsigned int i
, j
, l
;
13099 /* Re-use the adjustments array, but this time use it to replace
13100 every function argument use to an offset into the corresponding
13102 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
13104 if (!node
->simdclone
->args
[i
].vector_arg
)
13107 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
13108 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
13109 adjustments
[j
].new_decl
13110 = build4 (ARRAY_REF
,
13112 node
->simdclone
->args
[i
].simd_array
,
13114 NULL_TREE
, NULL_TREE
);
13115 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
13116 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
13117 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
13120 l
= adjustments
.length ();
13121 for (i
= 1; i
< num_ssa_names
; i
++)
13123 tree name
= ssa_name (i
);
13125 && SSA_NAME_VAR (name
)
13126 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
13128 for (j
= 0; j
< l
; j
++)
13129 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
13130 && adjustments
[j
].new_decl
)
13133 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
13136 = copy_var_decl (adjustments
[j
].base
,
13137 DECL_NAME (adjustments
[j
].base
),
13138 TREE_TYPE (adjustments
[j
].base
));
13139 adjustments
[j
].new_ssa_base
= base_var
;
13142 base_var
= adjustments
[j
].new_ssa_base
;
13143 if (SSA_NAME_IS_DEFAULT_DEF (name
))
13145 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13146 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
13147 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
13148 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
13149 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13150 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
13151 gimple stmt
= gimple_build_assign (name
, new_decl
);
13152 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13155 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13160 struct modify_stmt_info info
;
13161 info
.adjustments
= adjustments
;
13163 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
13165 gimple_stmt_iterator gsi
;
13167 gsi
= gsi_start_bb (bb
);
13168 while (!gsi_end_p (gsi
))
13170 gimple stmt
= gsi_stmt (gsi
);
13172 struct walk_stmt_info wi
;
13174 memset (&wi
, 0, sizeof (wi
));
13175 info
.modified
= false;
13177 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
13179 if (gimple_code (stmt
) == GIMPLE_RETURN
)
13181 tree retval
= gimple_return_retval (stmt
);
13184 gsi_remove (&gsi
, true);
13188 /* Replace `return foo' with `retval_array[iter] = foo'. */
13189 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
13190 retval_array
, iter
, NULL
, NULL
);
13191 stmt
= gimple_build_assign (ref
, retval
);
13192 gsi_replace (&gsi
, stmt
, true);
13193 info
.modified
= true;
13198 update_stmt (stmt
);
13199 if (maybe_clean_eh_stmt (stmt
))
13200 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
13207 /* Adjust the argument types in NODE to their appropriate vector
13211 simd_clone_adjust (struct cgraph_node
*node
)
13213 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
13215 targetm
.simd_clone
.adjust (node
);
13217 tree retval
= simd_clone_adjust_return_type (node
);
13218 ipa_parm_adjustment_vec adjustments
13219 = simd_clone_adjust_argument_types (node
);
13221 push_gimplify_context ();
13223 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
13225 /* Adjust all uses of vector arguments accordingly. Adjust all
13226 return values accordingly. */
13227 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
13228 tree iter1
= make_ssa_name (iter
, NULL
);
13229 tree iter2
= make_ssa_name (iter
, NULL
);
13230 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
13232 /* Initialize the iteration variable. */
13233 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13234 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
13235 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
13236 /* Insert the SIMD array and iv initialization at function
13238 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
13240 pop_gimplify_context (NULL
);
13242 /* Create a new BB right before the original exit BB, to hold the
13243 iteration increment and the condition/branch. */
13244 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
13245 basic_block incr_bb
= create_empty_bb (orig_exit
);
13246 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
13247 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13248 flag. Set it now to be a FALLTHRU_EDGE. */
13249 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
13250 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
13251 for (unsigned i
= 0;
13252 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
13254 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
13255 redirect_edge_succ (e
, incr_bb
);
13257 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
13258 e
->probability
= REG_BR_PROB_BASE
;
13259 gsi
= gsi_last_bb (incr_bb
);
13260 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
13261 build_int_cst (unsigned_type_node
,
13263 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13265 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13266 struct loop
*loop
= alloc_loop ();
13267 cfun
->has_force_vectorize_loops
= true;
13268 loop
->safelen
= node
->simdclone
->simdlen
;
13269 loop
->force_vectorize
= true;
13270 loop
->header
= body_bb
;
13272 /* Branch around the body if the mask applies. */
13273 if (node
->simdclone
->inbranch
)
13275 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
13277 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
13278 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
13279 tree aref
= build4 (ARRAY_REF
,
13280 TREE_TYPE (TREE_TYPE (mask_array
)),
13283 g
= gimple_build_assign (mask
, aref
);
13284 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13285 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
13286 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
13288 aref
= build1 (VIEW_CONVERT_EXPR
,
13289 build_nonstandard_integer_type (bitsize
, 0), mask
);
13290 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
13291 g
= gimple_build_assign (mask
, aref
);
13292 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13295 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
13297 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13298 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
13299 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
13302 /* Generate the condition. */
13303 g
= gimple_build_cond (LT_EXPR
,
13305 build_int_cst (unsigned_type_node
,
13306 node
->simdclone
->simdlen
),
13308 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13309 e
= split_block (incr_bb
, gsi_stmt (gsi
));
13310 basic_block latch_bb
= e
->dest
;
13311 basic_block new_exit_bb
;
13312 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
13313 loop
->latch
= latch_bb
;
13315 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
13317 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
13318 /* The successor of incr_bb is already pointing to latch_bb; just
13320 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13321 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
13323 gimple phi
= create_phi_node (iter1
, body_bb
);
13324 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
13325 edge latch_edge
= single_succ_edge (latch_bb
);
13326 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
13328 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13330 /* Generate the new return. */
13331 gsi
= gsi_last_bb (new_exit_bb
);
13333 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
13334 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
13335 retval
= TREE_OPERAND (retval
, 0);
13338 retval
= build1 (VIEW_CONVERT_EXPR
,
13339 TREE_TYPE (TREE_TYPE (node
->decl
)),
13341 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
13342 false, GSI_CONTINUE_LINKING
);
13344 g
= gimple_build_return (retval
);
13345 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13347 /* Handle aligned clauses by replacing default defs of the aligned
13348 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13349 lhs. Handle linear by adding PHIs. */
13350 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
13351 if (node
->simdclone
->args
[i
].alignment
13352 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13353 && (node
->simdclone
->args
[i
].alignment
13354 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
13355 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13358 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
13359 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13360 tree def
= ssa_default_def (cfun
, orig_arg
);
13361 if (def
&& !has_zero_uses (def
))
13363 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
13364 gimple_seq seq
= NULL
;
13365 bool need_cvt
= false;
13367 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
13369 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
13372 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
13373 gimple_call_set_lhs (g
, t
);
13374 gimple_seq_add_stmt_without_update (&seq
, g
);
13377 t
= make_ssa_name (orig_arg
, NULL
);
13378 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
13379 gimple_call_lhs (g
),
13381 gimple_seq_add_stmt_without_update (&seq
, g
);
13383 gsi_insert_seq_on_edge_immediate
13384 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
13386 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13387 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
13389 node
->create_edge (cgraph_node::get_create (fn
),
13390 call
, entry_bb
->count
, freq
);
13392 imm_use_iterator iter
;
13393 use_operand_p use_p
;
13395 tree repl
= gimple_get_lhs (g
);
13396 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13397 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
13400 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13401 SET_USE (use_p
, repl
);
13404 else if (node
->simdclone
->args
[i
].arg_type
13405 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
13407 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13408 tree def
= ssa_default_def (cfun
, orig_arg
);
13409 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13410 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
13411 if (def
&& !has_zero_uses (def
))
13413 iter1
= make_ssa_name (orig_arg
, NULL
);
13414 iter2
= make_ssa_name (orig_arg
, NULL
);
13415 phi
= create_phi_node (iter1
, body_bb
);
13416 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
13417 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13418 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13419 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
13420 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13421 ? TREE_TYPE (orig_arg
) : sizetype
;
13423 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
13424 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
13425 gsi
= gsi_last_bb (incr_bb
);
13426 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13428 imm_use_iterator iter
;
13429 use_operand_p use_p
;
13431 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13432 if (use_stmt
== phi
)
13435 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13436 SET_USE (use_p
, iter1
);
13440 calculate_dominance_info (CDI_DOMINATORS
);
13441 add_loop (loop
, loop
->header
->loop_father
);
13442 update_ssa (TODO_update_ssa
);
13447 /* If the function in NODE is tagged as an elemental SIMD function,
13448 create the appropriate SIMD clones. */
13451 expand_simd_clones (struct cgraph_node
*node
)
13453 tree attr
= lookup_attribute ("omp declare simd",
13454 DECL_ATTRIBUTES (node
->decl
));
13455 if (attr
== NULL_TREE
13456 || node
->global
.inlined_to
13457 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
13461 #pragma omp declare simd
13463 in C, there we don't know the argument types at all. */
13464 if (!node
->definition
13465 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
13470 /* Start with parsing the "omp declare simd" attribute(s). */
13471 bool inbranch_clause_specified
;
13472 struct cgraph_simd_clone
*clone_info
13473 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
13474 &inbranch_clause_specified
);
13475 if (clone_info
== NULL
)
13478 int orig_simdlen
= clone_info
->simdlen
;
13479 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
13480 /* The target can return 0 (no simd clones should be created),
13481 1 (just one ISA of simd clones should be created) or higher
13482 count of ISA variants. In that case, clone_info is initialized
13483 for the first ISA variant. */
13485 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
13490 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13491 also create one inbranch and one !inbranch clone of it. */
13492 for (int i
= 0; i
< count
* 2; i
++)
13494 struct cgraph_simd_clone
*clone
= clone_info
;
13495 if (inbranch_clause_specified
&& (i
& 1) != 0)
13500 clone
= simd_clone_struct_alloc (clone_info
->nargs
13502 simd_clone_struct_copy (clone
, clone_info
);
13503 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13504 and simd_clone_adjust_argument_types did to the first
13506 clone
->nargs
-= clone_info
->inbranch
;
13507 clone
->simdlen
= orig_simdlen
;
13508 /* And call the target hook again to get the right ISA. */
13509 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
13513 clone
->inbranch
= 1;
13516 /* simd_clone_mangle might fail if such a clone has been created
13518 tree id
= simd_clone_mangle (node
, clone
);
13519 if (id
== NULL_TREE
)
13522 /* Only when we are sure we want to create the clone actually
13523 clone the function (or definitions) or create another
13524 extern FUNCTION_DECL (for prototypes without definitions). */
13525 struct cgraph_node
*n
= simd_clone_create (node
);
13529 n
->simdclone
= clone
;
13530 clone
->origin
= node
;
13531 clone
->next_clone
= NULL
;
13532 if (node
->simd_clones
== NULL
)
13534 clone
->prev_clone
= n
;
13535 node
->simd_clones
= n
;
13539 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
13540 clone
->prev_clone
->simdclone
->next_clone
= n
;
13541 node
->simd_clones
->simdclone
->prev_clone
= n
;
13543 symtab
->change_decl_assembler_name (n
->decl
, id
);
13544 /* And finally adjust the return type, parameters and for
13545 definitions also function body. */
13546 if (node
->definition
)
13547 simd_clone_adjust (n
);
13550 simd_clone_adjust_return_type (n
);
13551 simd_clone_adjust_argument_types (n
);
13555 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
13558 /* Entry point for IPA simd clone creation pass. */
13560 static unsigned int
13561 ipa_omp_simd_clone (void)
13563 struct cgraph_node
*node
;
13564 FOR_EACH_FUNCTION (node
)
13565 expand_simd_clones (node
);
13571 const pass_data pass_data_omp_simd_clone
=
13573 SIMPLE_IPA_PASS
, /* type */
13574 "simdclone", /* name */
13575 OPTGROUP_NONE
, /* optinfo_flags */
13576 TV_NONE
, /* tv_id */
13577 ( PROP_ssa
| PROP_cfg
), /* properties_required */
13578 0, /* properties_provided */
13579 0, /* properties_destroyed */
13580 0, /* todo_flags_start */
13581 0, /* todo_flags_finish */
13584 class pass_omp_simd_clone
: public simple_ipa_opt_pass
13587 pass_omp_simd_clone(gcc::context
*ctxt
)
13588 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
13591 /* opt_pass methods: */
13592 virtual bool gate (function
*);
13593 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
13597 pass_omp_simd_clone::gate (function
*)
13599 return ((flag_openmp
|| flag_openmp_simd
13601 || (in_lto_p
&& !flag_wpa
))
13602 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
13605 } // anon namespace
13607 simple_ipa_opt_pass
*
13608 make_pass_omp_simd_clone (gcc::context
*ctxt
)
13610 return new pass_omp_simd_clone (ctxt
);
13613 /* Helper function for omp_finish_file routine.
13614 Takes decls from V_DECLS and adds their addresses and sizes to
13615 constructor-vector V_CTOR. It will be later used as DECL_INIT for decl
13616 representing a global symbol for OpenMP descriptor. */
13618 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
13619 vec
<constructor_elt
, va_gc
> *v_ctor
)
13621 unsigned len
= vec_safe_length (v_decls
);
13622 for (unsigned i
= 0; i
< len
; i
++)
13624 tree it
= (*v_decls
)[i
];
13625 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
13627 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
13629 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
13630 fold_convert (const_ptr_type_node
,
13635 /* Create new symbol containing (address, size) pairs for omp-marked
13636 functions and global variables. */
13638 omp_finish_file (void)
13640 const char *funcs_section_name
= OFFLOAD_FUNC_TABLE_SECTION_NAME
;
13641 const char *vars_section_name
= OFFLOAD_VAR_TABLE_SECTION_NAME
;
13643 unsigned num_funcs
= vec_safe_length (offload_funcs
);
13644 unsigned num_vars
= vec_safe_length (offload_vars
);
13646 if (num_funcs
== 0 && num_vars
== 0)
13649 if (targetm_common
.have_named_sections
)
13651 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
13652 vec_alloc (v_f
, num_funcs
);
13653 vec_alloc (v_v
, num_vars
* 2);
13655 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
13656 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
13658 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13660 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13662 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13663 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13664 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
13665 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
13666 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
13667 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
13668 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13669 get_identifier (".omp_func_table"),
13671 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13672 get_identifier (".omp_var_table"),
13674 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
13675 DECL_INITIAL (funcs_decl
) = ctor_f
;
13676 DECL_INITIAL (vars_decl
) = ctor_v
;
13677 set_decl_section_name (funcs_decl
, funcs_section_name
);
13678 set_decl_section_name (vars_decl
, vars_section_name
);
13680 varpool_node::finalize_decl (funcs_decl
);
13681 varpool_node::finalize_decl (vars_decl
);
13685 for (unsigned i
= 0; i
< num_funcs
; i
++)
13687 tree it
= (*offload_funcs
)[i
];
13688 targetm
.record_offload_symbol (it
);
13690 for (unsigned i
= 0; i
< num_vars
; i
++)
13692 tree it
= (*offload_vars
)[i
];
13693 targetm
.record_offload_symbol (it
);
13698 #include "gt-omp-low.h"