1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2014 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
30 #include "stringpool.h"
31 #include "stor-layout.h"
38 #include "hard-reg-set.h"
41 #include "dominance.h"
44 #include "basic-block.h"
45 #include "tree-ssa-alias.h"
46 #include "internal-fn.h"
47 #include "gimple-fold.h"
48 #include "gimple-expr.h"
52 #include "gimple-iterator.h"
53 #include "gimplify-me.h"
54 #include "gimple-walk.h"
55 #include "tree-iterator.h"
56 #include "tree-inline.h"
57 #include "langhooks.h"
58 #include "diagnostic-core.h"
59 #include "gimple-ssa.h"
61 #include "plugin-api.h"
65 #include "tree-phinodes.h"
66 #include "ssa-iterators.h"
67 #include "tree-ssanames.h"
68 #include "tree-into-ssa.h"
74 #include "tree-pass.h"
76 #include "splay-tree.h"
77 #include "insn-codes.h"
81 #include "common/common-target.h"
83 #include "gimple-low.h"
84 #include "tree-cfgcleanup.h"
85 #include "pretty-print.h"
86 #include "alloc-pool.h"
87 #include "symbol-summary.h"
89 #include "tree-nested.h"
93 #include "lto-section-names.h"
94 #include "gomp-constants.h"
97 /* Lowering of OMP parallel and workshare constructs proceeds in two
98 phases. The first phase scans the function looking for OMP statements
99 and then for variables that must be replaced to satisfy data sharing
100 clauses. The second phase expands code for the constructs, as well as
101 re-gimplifying things when variables have been replaced with complex
104 Final code generation is done by pass_expand_omp. The flowgraph is
105 scanned for regions which are then moved to a new
106 function, to be invoked by the thread library, or offloaded. */
108 /* OMP region information. Every parallel and workshare
109 directive is enclosed between two markers, the OMP_* directive
110 and a corresponding OMP_RETURN statement. */
114 /* The enclosing region. */
115 struct omp_region
*outer
;
117 /* First child region. */
118 struct omp_region
*inner
;
120 /* Next peer region. */
121 struct omp_region
*next
;
123 /* Block containing the omp directive as its last stmt. */
126 /* Block containing the OMP_RETURN as its last stmt. */
129 /* Block containing the OMP_CONTINUE as its last stmt. */
132 /* If this is a combined parallel+workshare region, this is a list
133 of additional arguments needed by the combined parallel+workshare
135 vec
<tree
, va_gc
> *ws_args
;
137 /* The code for the omp directive of this region. */
138 enum gimple_code type
;
140 /* Schedule kind, only used for OMP_FOR type regions. */
141 enum omp_clause_schedule_kind sched_kind
;
143 /* True if this is a combined parallel+workshare region. */
144 bool is_combined_parallel
;
147 /* Levels of parallelism as defined by OpenACC. Increasing numbers
148 correspond to deeper loop nesting levels. */
150 #define MASK_WORKER 2
151 #define MASK_VECTOR 4
153 /* Context structure. Used to store information about each parallel
154 directive in the code. */
156 typedef struct omp_context
158 /* This field must be at the beginning, as we do "inheritance": Some
159 callback functions for tree-inline.c (e.g., omp_copy_decl)
160 receive a copy_body_data pointer that is up-casted to an
161 omp_context pointer. */
164 /* The tree of contexts corresponding to the encountered constructs. */
165 struct omp_context
*outer
;
168 /* Map variables to fields in a structure that allows communication
169 between sending and receiving threads. */
170 splay_tree field_map
;
175 /* These are used just by task contexts, if task firstprivate fn is
176 needed. srecord_type is used to communicate from the thread
177 that encountered the task construct to task firstprivate fn,
178 record_type is allocated by GOMP_task, initialized by task firstprivate
179 fn and passed to the task body fn. */
180 splay_tree sfield_map
;
183 /* A chain of variables to add to the top-level block surrounding the
184 construct. In the case of a parallel, this is in the child function. */
187 /* A map of reduction pointer variables. For accelerators, each
188 reduction variable is replaced with an array. Each thread, in turn,
189 is assigned to a slot on that array. */
190 splay_tree reduction_map
;
192 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
193 barriers should jump to during omplower pass. */
196 /* What to do with variables with implicitly determined sharing
198 enum omp_clause_default_kind default_kind
;
200 /* Nesting depth of this context. Used to beautify error messages re
201 invalid gotos. The outermost ctx is depth 1, with depth 0 being
202 reserved for the main body of the function. */
205 /* True if this parallel directive is nested within another. */
208 /* True if this construct can be cancelled. */
211 /* For OpenACC loops, a mask of gang, worker and vector used at
212 levels below this one. */
214 /* For OpenACC loops, a mask of gang, worker and vector used at
215 this level and above. For parallel and kernels clauses, a mask
216 indicating which of num_gangs/num_workers/num_vectors was used. */
220 /* A structure holding the elements of:
221 for (V = N1; V cond N2; V += STEP) [...] */
223 struct omp_for_data_loop
225 tree v
, n1
, n2
, step
;
226 enum tree_code cond_code
;
229 /* A structure describing the main elements of a parallel loop. */
233 struct omp_for_data_loop loop
;
238 bool have_nowait
, have_ordered
;
239 enum omp_clause_schedule_kind sched_kind
;
240 struct omp_for_data_loop
*loops
;
244 static splay_tree all_contexts
;
245 static int taskreg_nesting_level
;
246 static int target_nesting_level
;
247 static struct omp_region
*root_omp_region
;
248 static bitmap task_shared_vars
;
249 static vec
<omp_context
*> taskreg_contexts
;
251 static void scan_omp (gimple_seq
*, omp_context
*);
252 static tree
scan_omp_1_op (tree
*, int *, void *);
254 #define WALK_SUBSTMTS \
258 case GIMPLE_EH_FILTER: \
259 case GIMPLE_TRANSACTION: \
260 /* The sub-statements for these should be walked. */ \
261 *handled_ops_p = false; \
264 /* Helper function to get the name of the array containing the partial
265 reductions for OpenACC reductions. */
267 oacc_get_reduction_array_id (tree node
)
269 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
270 int len
= strlen ("OACC") + strlen (id
);
271 char *temp_name
= XALLOCAVEC (char, len
+ 1);
272 snprintf (temp_name
, len
+ 1, "OACC%s", id
);
273 return IDENTIFIER_POINTER (get_identifier (temp_name
));
276 /* Determine the number of threads OpenACC threads used to determine the
277 size of the array of partial reductions. Currently, this is num_gangs
278 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
279 because it is independed of the device used. */
282 oacc_max_threads (omp_context
*ctx
)
284 tree nthreads
, vector_length
, gangs
, clauses
;
286 gangs
= fold_convert (sizetype
, integer_one_node
);
287 vector_length
= gangs
;
289 /* The reduction clause may be nested inside a loop directive.
290 Scan for the innermost vector_length clause. */
291 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
293 if (gimple_code (oc
->stmt
) != GIMPLE_OMP_TARGET
294 || (gimple_omp_target_kind (oc
->stmt
)
295 != GF_OMP_TARGET_KIND_OACC_PARALLEL
))
298 clauses
= gimple_omp_target_clauses (oc
->stmt
);
300 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
302 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
304 OMP_CLAUSE_VECTOR_LENGTH_EXPR
307 vector_length
= fold_convert (sizetype
, integer_one_node
);
309 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
311 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
312 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
314 gangs
= fold_convert (sizetype
, integer_one_node
);
319 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
324 /* Holds offload tables with decls. */
325 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
327 /* Holds a decl for __OFFLOAD_TABLE__. */
328 static GTY(()) tree offload_symbol_decl
;
330 /* Get the __OFFLOAD_TABLE__ symbol. */
332 get_offload_symbol_decl (void)
334 if (!offload_symbol_decl
)
336 tree decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
337 get_identifier ("__OFFLOAD_TABLE__"),
339 TREE_ADDRESSABLE (decl
) = 1;
340 TREE_PUBLIC (decl
) = 1;
341 DECL_EXTERNAL (decl
) = 1;
342 DECL_WEAK (decl
) = 1;
343 DECL_ATTRIBUTES (decl
)
344 = tree_cons (get_identifier ("weak"),
345 NULL_TREE
, DECL_ATTRIBUTES (decl
));
346 offload_symbol_decl
= decl
;
348 return offload_symbol_decl
;
351 /* Convenience function for calling scan_omp_1_op on tree operands. */
354 scan_omp_op (tree
*tp
, omp_context
*ctx
)
356 struct walk_stmt_info wi
;
358 memset (&wi
, 0, sizeof (wi
));
360 wi
.want_locations
= true;
362 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
365 static void lower_omp (gimple_seq
*, omp_context
*);
366 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
367 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
369 /* Find an OMP clause of type KIND within CLAUSES. */
372 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
374 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
375 if (OMP_CLAUSE_CODE (clauses
) == kind
)
381 /* Return true if CTX is for an omp parallel. */
384 is_parallel_ctx (omp_context
*ctx
)
386 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
390 /* Return true if CTX is for an omp task. */
393 is_task_ctx (omp_context
*ctx
)
395 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
399 /* Return true if CTX is for an omp parallel or omp task. */
402 is_taskreg_ctx (omp_context
*ctx
)
404 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
405 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
409 /* Return true if REGION is a combined parallel+workshare region. */
412 is_combined_parallel (struct omp_region
*region
)
414 return region
->is_combined_parallel
;
418 /* Extract the header elements of parallel loop FOR_STMT and store
422 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
423 struct omp_for_data_loop
*loops
)
425 tree t
, var
, *collapse_iter
, *collapse_count
;
426 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
427 struct omp_for_data_loop
*loop
;
429 struct omp_for_data_loop dummy_loop
;
430 location_t loc
= gimple_location (for_stmt
);
431 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
432 bool distribute
= gimple_omp_for_kind (for_stmt
)
433 == GF_OMP_FOR_KIND_DISTRIBUTE
;
435 fd
->for_stmt
= for_stmt
;
437 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
438 if (fd
->collapse
> 1)
441 fd
->loops
= &fd
->loop
;
443 fd
->have_nowait
= distribute
|| simd
;
444 fd
->have_ordered
= false;
445 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
446 fd
->chunk_size
= NULL_TREE
;
447 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
448 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
449 collapse_iter
= NULL
;
450 collapse_count
= NULL
;
452 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
453 switch (OMP_CLAUSE_CODE (t
))
455 case OMP_CLAUSE_NOWAIT
:
456 fd
->have_nowait
= true;
458 case OMP_CLAUSE_ORDERED
:
459 fd
->have_ordered
= true;
461 case OMP_CLAUSE_SCHEDULE
:
462 gcc_assert (!distribute
);
463 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
464 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
466 case OMP_CLAUSE_DIST_SCHEDULE
:
467 gcc_assert (distribute
);
468 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
470 case OMP_CLAUSE_COLLAPSE
:
471 if (fd
->collapse
> 1)
473 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
474 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
481 /* FIXME: for now map schedule(auto) to schedule(static).
482 There should be analysis to determine whether all iterations
483 are approximately the same amount of work (then schedule(static)
484 is best) or if it varies (then schedule(dynamic,N) is better). */
485 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
487 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
488 gcc_assert (fd
->chunk_size
== NULL
);
490 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
491 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
492 gcc_assert (fd
->chunk_size
== NULL
);
493 else if (fd
->chunk_size
== NULL
)
495 /* We only need to compute a default chunk size for ordered
496 static loops and dynamic loops. */
497 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
499 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
500 ? integer_zero_node
: integer_one_node
;
503 for (i
= 0; i
< fd
->collapse
; i
++)
505 if (fd
->collapse
== 1)
507 else if (loops
!= NULL
)
512 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
513 gcc_assert (SSA_VAR_P (loop
->v
));
514 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
515 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
516 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
517 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
519 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
520 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
521 switch (loop
->cond_code
)
527 gcc_assert (gimple_omp_for_kind (for_stmt
)
528 == GF_OMP_FOR_KIND_CILKSIMD
529 || (gimple_omp_for_kind (for_stmt
)
530 == GF_OMP_FOR_KIND_CILKFOR
));
533 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
534 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
536 loop
->n2
= fold_build2_loc (loc
,
537 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
538 build_int_cst (TREE_TYPE (loop
->n2
), 1));
539 loop
->cond_code
= LT_EXPR
;
542 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
543 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
545 loop
->n2
= fold_build2_loc (loc
,
546 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
547 build_int_cst (TREE_TYPE (loop
->n2
), 1));
548 loop
->cond_code
= GT_EXPR
;
554 t
= gimple_omp_for_incr (for_stmt
, i
);
555 gcc_assert (TREE_OPERAND (t
, 0) == var
);
556 switch (TREE_CODE (t
))
559 loop
->step
= TREE_OPERAND (t
, 1);
561 case POINTER_PLUS_EXPR
:
562 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
565 loop
->step
= TREE_OPERAND (t
, 1);
566 loop
->step
= fold_build1_loc (loc
,
567 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
575 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
576 && !fd
->have_ordered
))
578 if (fd
->collapse
== 1)
579 iter_type
= TREE_TYPE (loop
->v
);
581 || TYPE_PRECISION (iter_type
)
582 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
584 = build_nonstandard_integer_type
585 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
587 else if (iter_type
!= long_long_unsigned_type_node
)
589 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
590 iter_type
= long_long_unsigned_type_node
;
591 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
592 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
593 >= TYPE_PRECISION (iter_type
))
597 if (loop
->cond_code
== LT_EXPR
)
598 n
= fold_build2_loc (loc
,
599 PLUS_EXPR
, TREE_TYPE (loop
->v
),
600 loop
->n2
, loop
->step
);
603 if (TREE_CODE (n
) != INTEGER_CST
604 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
605 iter_type
= long_long_unsigned_type_node
;
607 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
608 > TYPE_PRECISION (iter_type
))
612 if (loop
->cond_code
== LT_EXPR
)
615 n2
= fold_build2_loc (loc
,
616 PLUS_EXPR
, TREE_TYPE (loop
->v
),
617 loop
->n2
, loop
->step
);
621 n1
= fold_build2_loc (loc
,
622 MINUS_EXPR
, TREE_TYPE (loop
->v
),
623 loop
->n2
, loop
->step
);
626 if (TREE_CODE (n1
) != INTEGER_CST
627 || TREE_CODE (n2
) != INTEGER_CST
628 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
629 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
630 iter_type
= long_long_unsigned_type_node
;
634 if (collapse_count
&& *collapse_count
== NULL
)
636 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
637 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
638 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
639 if (t
&& integer_zerop (t
))
640 count
= build_zero_cst (long_long_unsigned_type_node
);
641 else if ((i
== 0 || count
!= NULL_TREE
)
642 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
643 && TREE_CONSTANT (loop
->n1
)
644 && TREE_CONSTANT (loop
->n2
)
645 && TREE_CODE (loop
->step
) == INTEGER_CST
)
647 tree itype
= TREE_TYPE (loop
->v
);
649 if (POINTER_TYPE_P (itype
))
650 itype
= signed_type_for (itype
);
651 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
652 t
= fold_build2_loc (loc
,
654 fold_convert_loc (loc
, itype
, loop
->step
), t
);
655 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
656 fold_convert_loc (loc
, itype
, loop
->n2
));
657 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
658 fold_convert_loc (loc
, itype
, loop
->n1
));
659 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
660 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
661 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
662 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
663 fold_convert_loc (loc
, itype
,
666 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
667 fold_convert_loc (loc
, itype
, loop
->step
));
668 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
669 if (count
!= NULL_TREE
)
670 count
= fold_build2_loc (loc
,
671 MULT_EXPR
, long_long_unsigned_type_node
,
675 if (TREE_CODE (count
) != INTEGER_CST
)
678 else if (count
&& !integer_zerop (count
))
685 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
686 || fd
->have_ordered
))
688 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
689 iter_type
= long_long_unsigned_type_node
;
691 iter_type
= long_integer_type_node
;
693 else if (collapse_iter
&& *collapse_iter
!= NULL
)
694 iter_type
= TREE_TYPE (*collapse_iter
);
695 fd
->iter_type
= iter_type
;
696 if (collapse_iter
&& *collapse_iter
== NULL
)
697 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
698 if (collapse_count
&& *collapse_count
== NULL
)
701 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
703 *collapse_count
= create_tmp_var (iter_type
, ".count");
706 if (fd
->collapse
> 1)
708 fd
->loop
.v
= *collapse_iter
;
709 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
710 fd
->loop
.n2
= *collapse_count
;
711 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
712 fd
->loop
.cond_code
= LT_EXPR
;
715 /* For OpenACC loops, force a chunk size of one, as this avoids the default
716 scheduling where several subsequent iterations are being executed by the
718 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
720 gcc_assert (fd
->chunk_size
== NULL_TREE
);
721 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
726 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
727 is the immediate dominator of PAR_ENTRY_BB, return true if there
728 are no data dependencies that would prevent expanding the parallel
729 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
731 When expanding a combined parallel+workshare region, the call to
732 the child function may need additional arguments in the case of
733 GIMPLE_OMP_FOR regions. In some cases, these arguments are
734 computed out of variables passed in from the parent to the child
735 via 'struct .omp_data_s'. For instance:
737 #pragma omp parallel for schedule (guided, i * 4)
742 # BLOCK 2 (PAR_ENTRY_BB)
744 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
746 # BLOCK 3 (WS_ENTRY_BB)
747 .omp_data_i = &.omp_data_o;
748 D.1667 = .omp_data_i->i;
750 #pragma omp for schedule (guided, D.1598)
752 When we outline the parallel region, the call to the child function
753 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
754 that value is computed *after* the call site. So, in principle we
755 cannot do the transformation.
757 To see whether the code in WS_ENTRY_BB blocks the combined
758 parallel+workshare call, we collect all the variables used in the
759 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
760 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
763 FIXME. If we had the SSA form built at this point, we could merely
764 hoist the code in block 3 into block 2 and be done with it. But at
765 this point we don't have dataflow information and though we could
766 hack something up here, it is really not worth the aggravation. */
769 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
771 struct omp_for_data fd
;
772 gimple ws_stmt
= last_stmt (ws_entry_bb
);
774 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
777 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
779 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
781 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
783 if (fd
.iter_type
!= long_integer_type_node
)
786 /* FIXME. We give up too easily here. If any of these arguments
787 are not constants, they will likely involve variables that have
788 been mapped into fields of .omp_data_s for sharing with the child
789 function. With appropriate data flow, it would be possible to
791 if (!is_gimple_min_invariant (fd
.loop
.n1
)
792 || !is_gimple_min_invariant (fd
.loop
.n2
)
793 || !is_gimple_min_invariant (fd
.loop
.step
)
794 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
801 /* Collect additional arguments needed to emit a combined
802 parallel+workshare call. WS_STMT is the workshare directive being
805 static vec
<tree
, va_gc
> *
806 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
809 location_t loc
= gimple_location (ws_stmt
);
810 vec
<tree
, va_gc
> *ws_args
;
812 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
814 struct omp_for_data fd
;
817 extract_omp_for_data (for_stmt
, &fd
, NULL
);
821 if (gimple_omp_for_combined_into_p (for_stmt
))
824 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
825 OMP_CLAUSE__LOOPTEMP_
);
827 n1
= OMP_CLAUSE_DECL (innerc
);
828 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
829 OMP_CLAUSE__LOOPTEMP_
);
831 n2
= OMP_CLAUSE_DECL (innerc
);
834 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
836 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
837 ws_args
->quick_push (t
);
839 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
840 ws_args
->quick_push (t
);
842 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
843 ws_args
->quick_push (t
);
847 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
848 ws_args
->quick_push (t
);
853 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
855 /* Number of sections is equal to the number of edges from the
856 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
857 the exit of the sections region. */
858 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
859 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
860 vec_alloc (ws_args
, 1);
861 ws_args
->quick_push (t
);
869 /* Discover whether REGION is a combined parallel+workshare region. */
872 determine_parallel_type (struct omp_region
*region
)
874 basic_block par_entry_bb
, par_exit_bb
;
875 basic_block ws_entry_bb
, ws_exit_bb
;
877 if (region
== NULL
|| region
->inner
== NULL
878 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
879 || region
->inner
->cont
== NULL
)
882 /* We only support parallel+for and parallel+sections. */
883 if (region
->type
!= GIMPLE_OMP_PARALLEL
884 || (region
->inner
->type
!= GIMPLE_OMP_FOR
885 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
888 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
889 WS_EXIT_BB -> PAR_EXIT_BB. */
890 par_entry_bb
= region
->entry
;
891 par_exit_bb
= region
->exit
;
892 ws_entry_bb
= region
->inner
->entry
;
893 ws_exit_bb
= region
->inner
->exit
;
895 if (single_succ (par_entry_bb
) == ws_entry_bb
896 && single_succ (ws_exit_bb
) == par_exit_bb
897 && workshare_safe_to_combine_p (ws_entry_bb
)
898 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
899 || (last_and_only_stmt (ws_entry_bb
)
900 && last_and_only_stmt (par_exit_bb
))))
902 gimple par_stmt
= last_stmt (par_entry_bb
);
903 gimple ws_stmt
= last_stmt (ws_entry_bb
);
905 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
907 /* If this is a combined parallel loop, we need to determine
908 whether or not to use the combined library calls. There
909 are two cases where we do not apply the transformation:
910 static loops and any kind of ordered loop. In the first
911 case, we already open code the loop so there is no need
912 to do anything else. In the latter case, the combined
913 parallel loop call would still need extra synchronization
914 to implement ordered semantics, so there would not be any
915 gain in using the combined call. */
916 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
917 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
919 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
920 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
922 region
->is_combined_parallel
= false;
923 region
->inner
->is_combined_parallel
= false;
928 region
->is_combined_parallel
= true;
929 region
->inner
->is_combined_parallel
= true;
930 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
935 /* Return true if EXPR is variable sized. */
938 is_variable_sized (const_tree expr
)
940 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
943 /* Return true if DECL is a reference type. */
946 is_reference (tree decl
)
948 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
951 /* Return the type of a decl. If the decl is reference type,
952 return its base type. */
954 get_base_type (tree decl
)
956 tree type
= TREE_TYPE (decl
);
957 if (is_reference (decl
))
958 type
= TREE_TYPE (type
);
962 /* Lookup variables. The "maybe" form
963 allows for the variable form to not have been entered, otherwise we
964 assert that the variable must have been entered. */
967 lookup_decl (tree var
, omp_context
*ctx
)
969 tree
*n
= ctx
->cb
.decl_map
->get (var
);
974 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
976 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
977 return n
? *n
: NULL_TREE
;
981 lookup_field (tree var
, omp_context
*ctx
)
984 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
985 return (tree
) n
->value
;
989 lookup_sfield (tree var
, omp_context
*ctx
)
992 n
= splay_tree_lookup (ctx
->sfield_map
993 ? ctx
->sfield_map
: ctx
->field_map
,
994 (splay_tree_key
) var
);
995 return (tree
) n
->value
;
999 maybe_lookup_field (tree var
, omp_context
*ctx
)
1002 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1003 return n
? (tree
) n
->value
: NULL_TREE
;
1007 lookup_oacc_reduction (const char *id
, omp_context
*ctx
)
1010 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
1011 return (tree
) n
->value
;
1015 maybe_lookup_oacc_reduction (tree var
, omp_context
*ctx
)
1017 splay_tree_node n
= NULL
;
1018 if (ctx
->reduction_map
)
1019 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
1020 return n
? (tree
) n
->value
: NULL_TREE
;
1023 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1024 the parallel context if DECL is to be shared. */
1027 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1029 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1032 /* We can only use copy-in/copy-out semantics for shared variables
1033 when we know the value is not accessible from an outer scope. */
1036 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1038 /* ??? Trivially accessible from anywhere. But why would we even
1039 be passing an address in this case? Should we simply assert
1040 this to be false, or should we have a cleanup pass that removes
1041 these from the list of mappings? */
1042 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1045 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1046 without analyzing the expression whether or not its location
1047 is accessible to anyone else. In the case of nested parallel
1048 regions it certainly may be. */
1049 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1052 /* Do not use copy-in/copy-out for variables that have their
1054 if (TREE_ADDRESSABLE (decl
))
1057 /* lower_send_shared_vars only uses copy-in, but not copy-out
1059 if (TREE_READONLY (decl
)
1060 || ((TREE_CODE (decl
) == RESULT_DECL
1061 || TREE_CODE (decl
) == PARM_DECL
)
1062 && DECL_BY_REFERENCE (decl
)))
1065 /* Disallow copy-in/out in nested parallel if
1066 decl is shared in outer parallel, otherwise
1067 each thread could store the shared variable
1068 in its own copy-in location, making the
1069 variable no longer really shared. */
1070 if (shared_ctx
->is_nested
)
1074 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1075 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1082 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1083 c
; c
= OMP_CLAUSE_CHAIN (c
))
1084 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1085 && OMP_CLAUSE_DECL (c
) == decl
)
1089 goto maybe_mark_addressable_and_ret
;
1093 /* For tasks avoid using copy-in/out. As tasks can be
1094 deferred or executed in different thread, when GOMP_task
1095 returns, the task hasn't necessarily terminated. */
1096 if (is_task_ctx (shared_ctx
))
1099 maybe_mark_addressable_and_ret
:
1100 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1101 if (is_gimple_reg (outer
))
1103 /* Taking address of OUTER in lower_send_shared_vars
1104 might need regimplification of everything that uses the
1106 if (!task_shared_vars
)
1107 task_shared_vars
= BITMAP_ALLOC (NULL
);
1108 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1109 TREE_ADDRESSABLE (outer
) = 1;
1118 /* Construct a new automatic decl similar to VAR. */
1121 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1123 tree copy
= copy_var_decl (var
, name
, type
);
1125 DECL_CONTEXT (copy
) = current_function_decl
;
1126 DECL_CHAIN (copy
) = ctx
->block_vars
;
1127 ctx
->block_vars
= copy
;
1133 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1135 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1138 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1141 omp_build_component_ref (tree obj
, tree field
)
1143 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1144 if (TREE_THIS_VOLATILE (field
))
1145 TREE_THIS_VOLATILE (ret
) |= 1;
1146 if (TREE_READONLY (field
))
1147 TREE_READONLY (ret
) |= 1;
1151 /* Build tree nodes to access the field for VAR on the receiver side. */
1154 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1156 tree x
, field
= lookup_field (var
, ctx
);
1158 /* If the receiver record type was remapped in the child function,
1159 remap the field into the new record type. */
1160 x
= maybe_lookup_field (field
, ctx
);
1164 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1165 x
= omp_build_component_ref (x
, field
);
1167 x
= build_simple_mem_ref (x
);
1172 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1173 of a parallel, this is a component reference; for workshare constructs
1174 this is some variable. */
1177 build_outer_var_ref (tree var
, omp_context
*ctx
)
1181 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1183 else if (is_variable_sized (var
))
1185 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1186 x
= build_outer_var_ref (x
, ctx
);
1187 x
= build_simple_mem_ref (x
);
1189 else if (is_taskreg_ctx (ctx
))
1191 bool by_ref
= use_pointer_for_field (var
, NULL
);
1192 x
= build_receiver_ref (var
, by_ref
, ctx
);
1194 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1195 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1197 /* #pragma omp simd isn't a worksharing construct, and can reference even
1198 private vars in its linear etc. clauses. */
1200 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1201 x
= lookup_decl (var
, ctx
->outer
);
1202 else if (ctx
->outer
)
1203 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1207 else if (ctx
->outer
)
1208 x
= lookup_decl (var
, ctx
->outer
);
1209 else if (is_reference (var
))
1210 /* This can happen with orphaned constructs. If var is reference, it is
1211 possible it is shared and as such valid. */
1216 if (is_reference (var
))
1217 x
= build_simple_mem_ref (x
);
1222 /* Build tree nodes to access the field for VAR on the sender side. */
1225 build_sender_ref (tree var
, omp_context
*ctx
)
1227 tree field
= lookup_sfield (var
, ctx
);
1228 return omp_build_component_ref (ctx
->sender_decl
, field
);
1231 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1234 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1236 tree field
, type
, sfield
= NULL_TREE
;
1238 gcc_assert ((mask
& 1) == 0
1239 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1240 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1241 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1242 gcc_assert ((mask
& 3) == 3
1243 || !is_gimple_omp_oacc (ctx
->stmt
));
1245 type
= TREE_TYPE (var
);
1248 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1249 type
= build_pointer_type (build_pointer_type (type
));
1252 type
= build_pointer_type (type
);
1253 else if ((mask
& 3) == 1 && is_reference (var
))
1254 type
= TREE_TYPE (type
);
1256 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1257 FIELD_DECL
, DECL_NAME (var
), type
);
1259 /* Remember what variable this field was created for. This does have a
1260 side effect of making dwarf2out ignore this member, so for helpful
1261 debugging we clear it later in delete_omp_context. */
1262 DECL_ABSTRACT_ORIGIN (field
) = var
;
1263 if (type
== TREE_TYPE (var
))
1265 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1266 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1267 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1270 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1272 if ((mask
& 3) == 3)
1274 insert_field_into_struct (ctx
->record_type
, field
);
1275 if (ctx
->srecord_type
)
1277 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1278 FIELD_DECL
, DECL_NAME (var
), type
);
1279 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1280 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1281 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1282 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1283 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1288 if (ctx
->srecord_type
== NULL_TREE
)
1292 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1293 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1294 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1296 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1297 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1298 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1299 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1300 splay_tree_insert (ctx
->sfield_map
,
1301 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1302 (splay_tree_value
) sfield
);
1306 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1307 : ctx
->srecord_type
, field
);
1311 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1312 (splay_tree_value
) field
);
1313 if ((mask
& 2) && ctx
->sfield_map
)
1314 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1315 (splay_tree_value
) sfield
);
1319 install_var_local (tree var
, omp_context
*ctx
)
1321 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1322 insert_decl_map (&ctx
->cb
, var
, new_var
);
1326 /* Adjust the replacement for DECL in CTX for the new context. This means
1327 copying the DECL_VALUE_EXPR, and fixing up the type. */
1330 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1332 tree new_decl
, size
;
1334 new_decl
= lookup_decl (decl
, ctx
);
1336 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1338 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1339 && DECL_HAS_VALUE_EXPR_P (decl
))
1341 tree ve
= DECL_VALUE_EXPR (decl
);
1342 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1343 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1344 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1347 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1349 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1350 if (size
== error_mark_node
)
1351 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1352 DECL_SIZE (new_decl
) = size
;
1354 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1355 if (size
== error_mark_node
)
1356 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1357 DECL_SIZE_UNIT (new_decl
) = size
;
1361 /* The callback for remap_decl. Search all containing contexts for a
1362 mapping of the variable; this avoids having to duplicate the splay
1363 tree ahead of time. We know a mapping doesn't already exist in the
1364 given context. Create new mappings to implement default semantics. */
1367 omp_copy_decl (tree var
, copy_body_data
*cb
)
1369 omp_context
*ctx
= (omp_context
*) cb
;
1372 if (TREE_CODE (var
) == LABEL_DECL
)
1374 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1375 DECL_CONTEXT (new_var
) = current_function_decl
;
1376 insert_decl_map (&ctx
->cb
, var
, new_var
);
1380 while (!is_taskreg_ctx (ctx
))
1385 new_var
= maybe_lookup_decl (var
, ctx
);
1390 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1393 return error_mark_node
;
1397 /* Debugging dumps for parallel regions. */
1398 void dump_omp_region (FILE *, struct omp_region
*, int);
1399 void debug_omp_region (struct omp_region
*);
1400 void debug_all_omp_regions (void);
1402 /* Dump the parallel region tree rooted at REGION. */
1405 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1407 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1408 gimple_code_name
[region
->type
]);
1411 dump_omp_region (file
, region
->inner
, indent
+ 4);
1415 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1416 region
->cont
->index
);
1420 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1421 region
->exit
->index
);
1423 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1426 dump_omp_region (file
, region
->next
, indent
);
1430 debug_omp_region (struct omp_region
*region
)
1432 dump_omp_region (stderr
, region
, 0);
1436 debug_all_omp_regions (void)
1438 dump_omp_region (stderr
, root_omp_region
, 0);
1442 /* Create a new parallel region starting at STMT inside region PARENT. */
1444 static struct omp_region
*
1445 new_omp_region (basic_block bb
, enum gimple_code type
,
1446 struct omp_region
*parent
)
1448 struct omp_region
*region
= XCNEW (struct omp_region
);
1450 region
->outer
= parent
;
1452 region
->type
= type
;
1456 /* This is a nested region. Add it to the list of inner
1457 regions in PARENT. */
1458 region
->next
= parent
->inner
;
1459 parent
->inner
= region
;
1463 /* This is a toplevel region. Add it to the list of toplevel
1464 regions in ROOT_OMP_REGION. */
1465 region
->next
= root_omp_region
;
1466 root_omp_region
= region
;
1472 /* Release the memory associated with the region tree rooted at REGION. */
1475 free_omp_region_1 (struct omp_region
*region
)
1477 struct omp_region
*i
, *n
;
1479 for (i
= region
->inner
; i
; i
= n
)
1482 free_omp_region_1 (i
);
1488 /* Release the memory for the entire omp region tree. */
1491 free_omp_regions (void)
1493 struct omp_region
*r
, *n
;
1494 for (r
= root_omp_region
; r
; r
= n
)
1497 free_omp_region_1 (r
);
1499 root_omp_region
= NULL
;
1503 /* Create a new context, with OUTER_CTX being the surrounding context. */
1505 static omp_context
*
1506 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1508 omp_context
*ctx
= XCNEW (omp_context
);
1510 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1511 (splay_tree_value
) ctx
);
1516 ctx
->outer
= outer_ctx
;
1517 ctx
->cb
= outer_ctx
->cb
;
1518 ctx
->cb
.block
= NULL
;
1519 ctx
->depth
= outer_ctx
->depth
+ 1;
1520 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1524 ctx
->cb
.src_fn
= current_function_decl
;
1525 ctx
->cb
.dst_fn
= current_function_decl
;
1526 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1527 gcc_checking_assert (ctx
->cb
.src_node
);
1528 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1529 ctx
->cb
.src_cfun
= cfun
;
1530 ctx
->cb
.copy_decl
= omp_copy_decl
;
1531 ctx
->cb
.eh_lp_nr
= 0;
1532 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1536 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1541 static gimple_seq
maybe_catch_exception (gimple_seq
);
1543 /* Finalize task copyfn. */
1546 finalize_task_copyfn (gomp_task
*task_stmt
)
1548 struct function
*child_cfun
;
1550 gimple_seq seq
= NULL
, new_seq
;
1553 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1554 if (child_fn
== NULL_TREE
)
1557 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1558 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1560 push_cfun (child_cfun
);
1561 bind
= gimplify_body (child_fn
, false);
1562 gimple_seq_add_stmt (&seq
, bind
);
1563 new_seq
= maybe_catch_exception (seq
);
1566 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1568 gimple_seq_add_stmt (&seq
, bind
);
1570 gimple_set_body (child_fn
, seq
);
1573 /* Inform the callgraph about the new function. */
1574 cgraph_node::add_new_function (child_fn
, false);
1577 /* Destroy a omp_context data structures. Called through the splay tree
1578 value delete callback. */
1581 delete_omp_context (splay_tree_value value
)
1583 omp_context
*ctx
= (omp_context
*) value
;
1585 delete ctx
->cb
.decl_map
;
1588 splay_tree_delete (ctx
->field_map
);
1589 if (ctx
->sfield_map
)
1590 splay_tree_delete (ctx
->sfield_map
);
1591 if (ctx
->reduction_map
1592 /* Shared over several omp_contexts. */
1593 && (ctx
->outer
== NULL
1594 || ctx
->reduction_map
!= ctx
->outer
->reduction_map
))
1595 splay_tree_delete (ctx
->reduction_map
);
1597 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1598 it produces corrupt debug information. */
1599 if (ctx
->record_type
)
1602 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1603 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1605 if (ctx
->srecord_type
)
1608 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1609 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1612 if (is_task_ctx (ctx
))
1613 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1618 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1622 fixup_child_record_type (omp_context
*ctx
)
1624 tree f
, type
= ctx
->record_type
;
1626 /* ??? It isn't sufficient to just call remap_type here, because
1627 variably_modified_type_p doesn't work the way we expect for
1628 record types. Testing each field for whether it needs remapping
1629 and creating a new record by hand works, however. */
1630 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1631 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1635 tree name
, new_fields
= NULL
;
1637 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1638 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1639 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1640 TYPE_DECL
, name
, type
);
1641 TYPE_NAME (type
) = name
;
1643 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1645 tree new_f
= copy_node (f
);
1646 DECL_CONTEXT (new_f
) = type
;
1647 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1648 DECL_CHAIN (new_f
) = new_fields
;
1649 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1650 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1652 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1656 /* Arrange to be able to look up the receiver field
1657 given the sender field. */
1658 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1659 (splay_tree_value
) new_f
);
1661 TYPE_FIELDS (type
) = nreverse (new_fields
);
1665 TREE_TYPE (ctx
->receiver_decl
)
1666 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1669 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1670 specified by CLAUSES. */
1673 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1676 bool scan_array_reductions
= false;
1678 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1682 switch (OMP_CLAUSE_CODE (c
))
1684 case OMP_CLAUSE_PRIVATE
:
1685 decl
= OMP_CLAUSE_DECL (c
);
1686 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1688 else if (!is_variable_sized (decl
))
1689 install_var_local (decl
, ctx
);
1692 case OMP_CLAUSE_SHARED
:
1693 decl
= OMP_CLAUSE_DECL (c
);
1694 /* Ignore shared directives in teams construct. */
1695 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1697 /* Global variables don't need to be copied,
1698 the receiver side will use them directly. */
1699 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1700 if (is_global_var (odecl
))
1702 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1705 gcc_assert (is_taskreg_ctx (ctx
));
1706 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1707 || !is_variable_sized (decl
));
1708 /* Global variables don't need to be copied,
1709 the receiver side will use them directly. */
1710 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1712 by_ref
= use_pointer_for_field (decl
, ctx
);
1713 if (! TREE_READONLY (decl
)
1714 || TREE_ADDRESSABLE (decl
)
1716 || is_reference (decl
))
1718 install_var_field (decl
, by_ref
, 3, ctx
);
1719 install_var_local (decl
, ctx
);
1722 /* We don't need to copy const scalar vars back. */
1723 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1726 case OMP_CLAUSE_LASTPRIVATE
:
1727 /* Let the corresponding firstprivate clause create
1729 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1733 case OMP_CLAUSE_FIRSTPRIVATE
:
1734 if (is_gimple_omp_oacc (ctx
->stmt
))
1736 sorry ("clause not supported yet");
1740 case OMP_CLAUSE_REDUCTION
:
1741 case OMP_CLAUSE_LINEAR
:
1742 decl
= OMP_CLAUSE_DECL (c
);
1744 if (is_variable_sized (decl
))
1746 if (is_task_ctx (ctx
))
1747 install_var_field (decl
, false, 1, ctx
);
1750 else if (is_taskreg_ctx (ctx
))
1753 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1754 by_ref
= use_pointer_for_field (decl
, NULL
);
1756 if (is_task_ctx (ctx
)
1757 && (global
|| by_ref
|| is_reference (decl
)))
1759 install_var_field (decl
, false, 1, ctx
);
1761 install_var_field (decl
, by_ref
, 2, ctx
);
1764 install_var_field (decl
, by_ref
, 3, ctx
);
1766 install_var_local (decl
, ctx
);
1767 if (is_gimple_omp_oacc (ctx
->stmt
)
1768 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
1770 /* Create a decl for the reduction array. */
1771 tree var
= OMP_CLAUSE_DECL (c
);
1772 tree type
= get_base_type (var
);
1773 tree ptype
= build_pointer_type (type
);
1774 tree array
= create_tmp_var (ptype
,
1775 oacc_get_reduction_array_id (var
));
1776 omp_context
*c
= (ctx
->field_map
? ctx
: ctx
->outer
);
1777 install_var_field (array
, true, 3, c
);
1778 install_var_local (array
, c
);
1780 /* Insert it into the current context. */
1781 splay_tree_insert (ctx
->reduction_map
, (splay_tree_key
)
1782 oacc_get_reduction_array_id (var
),
1783 (splay_tree_value
) array
);
1784 splay_tree_insert (ctx
->reduction_map
,
1785 (splay_tree_key
) array
,
1786 (splay_tree_value
) array
);
1790 case OMP_CLAUSE__LOOPTEMP_
:
1791 gcc_assert (is_parallel_ctx (ctx
));
1792 decl
= OMP_CLAUSE_DECL (c
);
1793 install_var_field (decl
, false, 3, ctx
);
1794 install_var_local (decl
, ctx
);
1797 case OMP_CLAUSE_COPYPRIVATE
:
1798 case OMP_CLAUSE_COPYIN
:
1799 decl
= OMP_CLAUSE_DECL (c
);
1800 by_ref
= use_pointer_for_field (decl
, NULL
);
1801 install_var_field (decl
, by_ref
, 3, ctx
);
1804 case OMP_CLAUSE_DEFAULT
:
1805 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1808 case OMP_CLAUSE_FINAL
:
1810 case OMP_CLAUSE_NUM_THREADS
:
1811 case OMP_CLAUSE_NUM_TEAMS
:
1812 case OMP_CLAUSE_THREAD_LIMIT
:
1813 case OMP_CLAUSE_DEVICE
:
1814 case OMP_CLAUSE_SCHEDULE
:
1815 case OMP_CLAUSE_DIST_SCHEDULE
:
1816 case OMP_CLAUSE_DEPEND
:
1817 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1818 case OMP_CLAUSE_NUM_GANGS
:
1819 case OMP_CLAUSE_NUM_WORKERS
:
1820 case OMP_CLAUSE_VECTOR_LENGTH
:
1822 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1826 case OMP_CLAUSE_FROM
:
1827 case OMP_CLAUSE_MAP
:
1829 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1830 decl
= OMP_CLAUSE_DECL (c
);
1831 /* Global variables with "omp declare target" attribute
1832 don't need to be copied, the receiver side will use them
1834 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1836 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1837 && varpool_node::get_create (decl
)->offloadable
)
1839 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1840 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1842 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1843 regions that are not offloaded; there is nothing to map for
1845 if (!is_gimple_omp_offloaded (ctx
->stmt
)
1846 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1851 if (DECL_SIZE (decl
)
1852 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1854 tree decl2
= DECL_VALUE_EXPR (decl
);
1855 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1856 decl2
= TREE_OPERAND (decl2
, 0);
1857 gcc_assert (DECL_P (decl2
));
1858 install_var_field (decl2
, true, 3, ctx
);
1859 install_var_local (decl2
, ctx
);
1860 install_var_local (decl
, ctx
);
1864 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1865 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1866 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1867 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1868 install_var_field (decl
, true, 7, ctx
);
1870 install_var_field (decl
, true, 3, ctx
);
1871 if (is_gimple_omp_offloaded (ctx
->stmt
))
1872 install_var_local (decl
, ctx
);
1877 tree base
= get_base_address (decl
);
1878 tree nc
= OMP_CLAUSE_CHAIN (c
);
1881 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1882 && OMP_CLAUSE_DECL (nc
) == base
1883 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1884 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1886 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1887 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1893 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1894 decl
= OMP_CLAUSE_DECL (c
);
1896 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1897 (splay_tree_key
) decl
));
1899 = build_decl (OMP_CLAUSE_LOCATION (c
),
1900 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1901 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1902 insert_field_into_struct (ctx
->record_type
, field
);
1903 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1904 (splay_tree_value
) field
);
1909 case OMP_CLAUSE_NOWAIT
:
1910 case OMP_CLAUSE_ORDERED
:
1911 case OMP_CLAUSE_COLLAPSE
:
1912 case OMP_CLAUSE_UNTIED
:
1913 case OMP_CLAUSE_MERGEABLE
:
1914 case OMP_CLAUSE_PROC_BIND
:
1915 case OMP_CLAUSE_SAFELEN
:
1916 case OMP_CLAUSE_ASYNC
:
1917 case OMP_CLAUSE_WAIT
:
1918 case OMP_CLAUSE_GANG
:
1919 case OMP_CLAUSE_WORKER
:
1920 case OMP_CLAUSE_VECTOR
:
1923 case OMP_CLAUSE_ALIGNED
:
1924 decl
= OMP_CLAUSE_DECL (c
);
1925 if (is_global_var (decl
)
1926 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1927 install_var_local (decl
, ctx
);
1930 case OMP_CLAUSE_DEVICE_RESIDENT
:
1931 case OMP_CLAUSE_USE_DEVICE
:
1932 case OMP_CLAUSE__CACHE_
:
1933 case OMP_CLAUSE_INDEPENDENT
:
1934 case OMP_CLAUSE_AUTO
:
1935 case OMP_CLAUSE_SEQ
:
1936 sorry ("Clause not supported yet");
1944 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1946 switch (OMP_CLAUSE_CODE (c
))
1948 case OMP_CLAUSE_LASTPRIVATE
:
1949 /* Let the corresponding firstprivate clause create
1951 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1952 scan_array_reductions
= true;
1953 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1957 case OMP_CLAUSE_FIRSTPRIVATE
:
1958 if (is_gimple_omp_oacc (ctx
->stmt
))
1960 sorry ("clause not supported yet");
1964 case OMP_CLAUSE_PRIVATE
:
1965 case OMP_CLAUSE_REDUCTION
:
1966 case OMP_CLAUSE_LINEAR
:
1967 decl
= OMP_CLAUSE_DECL (c
);
1968 if (is_variable_sized (decl
))
1969 install_var_local (decl
, ctx
);
1970 fixup_remapped_decl (decl
, ctx
,
1971 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1972 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1973 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1974 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1975 scan_array_reductions
= true;
1976 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1977 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1978 scan_array_reductions
= true;
1981 case OMP_CLAUSE_SHARED
:
1982 /* Ignore shared directives in teams construct. */
1983 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1985 decl
= OMP_CLAUSE_DECL (c
);
1986 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1987 fixup_remapped_decl (decl
, ctx
, false);
1990 case OMP_CLAUSE_MAP
:
1991 if (!is_gimple_omp_offloaded (ctx
->stmt
))
1993 decl
= OMP_CLAUSE_DECL (c
);
1995 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1996 && varpool_node::get_create (decl
)->offloadable
)
2000 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
2001 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2002 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2004 tree new_decl
= lookup_decl (decl
, ctx
);
2005 TREE_TYPE (new_decl
)
2006 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2008 else if (DECL_SIZE (decl
)
2009 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2011 tree decl2
= DECL_VALUE_EXPR (decl
);
2012 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2013 decl2
= TREE_OPERAND (decl2
, 0);
2014 gcc_assert (DECL_P (decl2
));
2015 fixup_remapped_decl (decl2
, ctx
, false);
2016 fixup_remapped_decl (decl
, ctx
, true);
2019 fixup_remapped_decl (decl
, ctx
, false);
2023 case OMP_CLAUSE_COPYPRIVATE
:
2024 case OMP_CLAUSE_COPYIN
:
2025 case OMP_CLAUSE_DEFAULT
:
2027 case OMP_CLAUSE_NUM_THREADS
:
2028 case OMP_CLAUSE_NUM_TEAMS
:
2029 case OMP_CLAUSE_THREAD_LIMIT
:
2030 case OMP_CLAUSE_DEVICE
:
2031 case OMP_CLAUSE_SCHEDULE
:
2032 case OMP_CLAUSE_DIST_SCHEDULE
:
2033 case OMP_CLAUSE_NOWAIT
:
2034 case OMP_CLAUSE_ORDERED
:
2035 case OMP_CLAUSE_COLLAPSE
:
2036 case OMP_CLAUSE_UNTIED
:
2037 case OMP_CLAUSE_FINAL
:
2038 case OMP_CLAUSE_MERGEABLE
:
2039 case OMP_CLAUSE_PROC_BIND
:
2040 case OMP_CLAUSE_SAFELEN
:
2041 case OMP_CLAUSE_ALIGNED
:
2042 case OMP_CLAUSE_DEPEND
:
2043 case OMP_CLAUSE__LOOPTEMP_
:
2045 case OMP_CLAUSE_FROM
:
2046 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2047 case OMP_CLAUSE_ASYNC
:
2048 case OMP_CLAUSE_WAIT
:
2049 case OMP_CLAUSE_NUM_GANGS
:
2050 case OMP_CLAUSE_NUM_WORKERS
:
2051 case OMP_CLAUSE_VECTOR_LENGTH
:
2052 case OMP_CLAUSE_GANG
:
2053 case OMP_CLAUSE_WORKER
:
2054 case OMP_CLAUSE_VECTOR
:
2057 case OMP_CLAUSE_DEVICE_RESIDENT
:
2058 case OMP_CLAUSE_USE_DEVICE
:
2059 case OMP_CLAUSE__CACHE_
:
2060 case OMP_CLAUSE_INDEPENDENT
:
2061 case OMP_CLAUSE_AUTO
:
2062 case OMP_CLAUSE_SEQ
:
2063 sorry ("Clause not supported yet");
2071 gcc_checking_assert (!scan_array_reductions
2072 || !is_gimple_omp_oacc (ctx
->stmt
));
2073 if (scan_array_reductions
)
2074 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2075 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2076 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2078 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2079 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2081 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2082 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2083 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2084 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2085 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2086 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2089 /* Create a new name for omp child function. Returns an identifier. If
2090 IS_CILK_FOR is true then the suffix for the child function is
2094 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2097 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2098 return clone_function_name (current_function_decl
,
2099 task_copy
? "_omp_cpyfn" : "_omp_fn");
2102 /* Returns the type of the induction variable for the child function for
2103 _Cilk_for and the types for _high and _low variables based on TYPE. */
2106 cilk_for_check_loop_diff_type (tree type
)
2108 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2110 if (TYPE_UNSIGNED (type
))
2111 return uint32_type_node
;
2113 return integer_type_node
;
2117 if (TYPE_UNSIGNED (type
))
2118 return uint64_type_node
;
2120 return long_long_integer_type_node
;
2124 /* Build a decl for the omp child function. It'll not contain a body
2125 yet, just the bare decl. */
2128 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2130 tree decl
, type
, name
, t
;
2133 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2134 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2135 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2136 tree cilk_var_type
= NULL_TREE
;
2138 name
= create_omp_child_function_name (task_copy
,
2139 cilk_for_count
!= NULL_TREE
);
2141 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2142 ptr_type_node
, NULL_TREE
);
2143 else if (cilk_for_count
)
2145 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2146 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2147 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2148 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2151 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2153 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2155 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2158 ctx
->cb
.dst_fn
= decl
;
2160 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2162 TREE_STATIC (decl
) = 1;
2163 TREE_USED (decl
) = 1;
2164 DECL_ARTIFICIAL (decl
) = 1;
2165 DECL_IGNORED_P (decl
) = 0;
2166 TREE_PUBLIC (decl
) = 0;
2167 DECL_UNINLINABLE (decl
) = 1;
2168 DECL_EXTERNAL (decl
) = 0;
2169 DECL_CONTEXT (decl
) = NULL_TREE
;
2170 DECL_INITIAL (decl
) = make_node (BLOCK
);
2171 if (cgraph_node::get (current_function_decl
)->offloadable
)
2172 cgraph_node::get_create (decl
)->offloadable
= 1;
2176 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2177 if (is_gimple_omp_offloaded (octx
->stmt
))
2179 cgraph_node::get_create (decl
)->offloadable
= 1;
2180 #ifdef ENABLE_OFFLOADING
2181 g
->have_offload
= true;
2187 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2188 RESULT_DECL
, NULL_TREE
, void_type_node
);
2189 DECL_ARTIFICIAL (t
) = 1;
2190 DECL_IGNORED_P (t
) = 1;
2191 DECL_CONTEXT (t
) = decl
;
2192 DECL_RESULT (decl
) = t
;
2194 /* _Cilk_for's child function requires two extra parameters called
2195 __low and __high that are set the by Cilk runtime when it calls this
2199 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2200 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2201 DECL_ARTIFICIAL (t
) = 1;
2202 DECL_NAMELESS (t
) = 1;
2203 DECL_ARG_TYPE (t
) = ptr_type_node
;
2204 DECL_CONTEXT (t
) = current_function_decl
;
2206 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2207 DECL_ARGUMENTS (decl
) = t
;
2209 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2210 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2211 DECL_ARTIFICIAL (t
) = 1;
2212 DECL_NAMELESS (t
) = 1;
2213 DECL_ARG_TYPE (t
) = ptr_type_node
;
2214 DECL_CONTEXT (t
) = current_function_decl
;
2216 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2217 DECL_ARGUMENTS (decl
) = t
;
2220 tree data_name
= get_identifier (".omp_data_i");
2221 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2223 DECL_ARTIFICIAL (t
) = 1;
2224 DECL_NAMELESS (t
) = 1;
2225 DECL_ARG_TYPE (t
) = ptr_type_node
;
2226 DECL_CONTEXT (t
) = current_function_decl
;
2229 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2230 DECL_ARGUMENTS (decl
) = t
;
2232 ctx
->receiver_decl
= t
;
2235 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2236 PARM_DECL
, get_identifier (".omp_data_o"),
2238 DECL_ARTIFICIAL (t
) = 1;
2239 DECL_NAMELESS (t
) = 1;
2240 DECL_ARG_TYPE (t
) = ptr_type_node
;
2241 DECL_CONTEXT (t
) = current_function_decl
;
2243 TREE_ADDRESSABLE (t
) = 1;
2244 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2245 DECL_ARGUMENTS (decl
) = t
;
2248 /* Allocate memory for the function structure. The call to
2249 allocate_struct_function clobbers CFUN, so we need to restore
2251 push_struct_function (decl
);
2252 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2256 /* Callback for walk_gimple_seq. Check if combined parallel
2257 contains gimple_omp_for_combined_into_p OMP_FOR. */
2260 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2261 bool *handled_ops_p
,
2262 struct walk_stmt_info
*wi
)
2264 gimple stmt
= gsi_stmt (*gsi_p
);
2266 *handled_ops_p
= true;
2267 switch (gimple_code (stmt
))
2271 case GIMPLE_OMP_FOR
:
2272 if (gimple_omp_for_combined_into_p (stmt
)
2273 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2276 return integer_zero_node
;
2285 /* Scan an OpenMP parallel directive. */
2288 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2292 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2294 /* Ignore parallel directives with empty bodies, unless there
2295 are copyin clauses. */
2297 && empty_body_p (gimple_omp_body (stmt
))
2298 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2299 OMP_CLAUSE_COPYIN
) == NULL
)
2301 gsi_replace (gsi
, gimple_build_nop (), false);
2305 if (gimple_omp_parallel_combined_p (stmt
))
2307 struct walk_stmt_info wi
;
2309 memset (&wi
, 0, sizeof (wi
));
2311 walk_gimple_seq (gimple_omp_body (stmt
),
2312 find_combined_for
, NULL
, &wi
);
2315 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
) wi
.info
);
2316 struct omp_for_data fd
;
2317 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2318 /* We need two temporaries with fd.loop.v type (istart/iend)
2319 and then (fd.collapse - 1) temporaries with the same
2320 type for count2 ... countN-1 vars if not constant. */
2321 size_t count
= 2, i
;
2322 tree type
= fd
.iter_type
;
2324 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2325 count
+= fd
.collapse
- 1;
2326 for (i
= 0; i
< count
; i
++)
2328 tree temp
= create_tmp_var (type
);
2329 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2330 OMP_CLAUSE__LOOPTEMP_
);
2331 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2332 OMP_CLAUSE_DECL (c
) = temp
;
2333 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2334 gimple_omp_parallel_set_clauses (stmt
, c
);
2339 ctx
= new_omp_context (stmt
, outer_ctx
);
2340 taskreg_contexts
.safe_push (ctx
);
2341 if (taskreg_nesting_level
> 1)
2342 ctx
->is_nested
= true;
2343 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2344 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2345 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2346 name
= create_tmp_var_name (".omp_data_s");
2347 name
= build_decl (gimple_location (stmt
),
2348 TYPE_DECL
, name
, ctx
->record_type
);
2349 DECL_ARTIFICIAL (name
) = 1;
2350 DECL_NAMELESS (name
) = 1;
2351 TYPE_NAME (ctx
->record_type
) = name
;
2352 create_omp_child_function (ctx
, false);
2353 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2355 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2356 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2358 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2359 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2362 /* Scan an OpenMP task directive. */
2365 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2369 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2371 /* Ignore task directives with empty bodies. */
2373 && empty_body_p (gimple_omp_body (stmt
)))
2375 gsi_replace (gsi
, gimple_build_nop (), false);
2379 ctx
= new_omp_context (stmt
, outer_ctx
);
2380 taskreg_contexts
.safe_push (ctx
);
2381 if (taskreg_nesting_level
> 1)
2382 ctx
->is_nested
= true;
2383 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2384 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2385 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2386 name
= create_tmp_var_name (".omp_data_s");
2387 name
= build_decl (gimple_location (stmt
),
2388 TYPE_DECL
, name
, ctx
->record_type
);
2389 DECL_ARTIFICIAL (name
) = 1;
2390 DECL_NAMELESS (name
) = 1;
2391 TYPE_NAME (ctx
->record_type
) = name
;
2392 create_omp_child_function (ctx
, false);
2393 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2395 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2397 if (ctx
->srecord_type
)
2399 name
= create_tmp_var_name (".omp_data_a");
2400 name
= build_decl (gimple_location (stmt
),
2401 TYPE_DECL
, name
, ctx
->srecord_type
);
2402 DECL_ARTIFICIAL (name
) = 1;
2403 DECL_NAMELESS (name
) = 1;
2404 TYPE_NAME (ctx
->srecord_type
) = name
;
2405 create_omp_child_function (ctx
, true);
2408 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2410 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2412 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2413 t
= build_int_cst (long_integer_type_node
, 0);
2414 gimple_omp_task_set_arg_size (stmt
, t
);
2415 t
= build_int_cst (long_integer_type_node
, 1);
2416 gimple_omp_task_set_arg_align (stmt
, t
);
2421 /* If any decls have been made addressable during scan_omp,
2422 adjust their fields if needed, and layout record types
2423 of parallel/task constructs. */
2426 finish_taskreg_scan (omp_context
*ctx
)
2428 if (ctx
->record_type
== NULL_TREE
)
2431 /* If any task_shared_vars were needed, verify all
2432 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2433 statements if use_pointer_for_field hasn't changed
2434 because of that. If it did, update field types now. */
2435 if (task_shared_vars
)
2439 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2440 c
; c
= OMP_CLAUSE_CHAIN (c
))
2441 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2443 tree decl
= OMP_CLAUSE_DECL (c
);
2445 /* Global variables don't need to be copied,
2446 the receiver side will use them directly. */
2447 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2449 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2450 || !use_pointer_for_field (decl
, ctx
))
2452 tree field
= lookup_field (decl
, ctx
);
2453 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2454 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2456 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2457 TREE_THIS_VOLATILE (field
) = 0;
2458 DECL_USER_ALIGN (field
) = 0;
2459 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2460 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2461 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2462 if (ctx
->srecord_type
)
2464 tree sfield
= lookup_sfield (decl
, ctx
);
2465 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2466 TREE_THIS_VOLATILE (sfield
) = 0;
2467 DECL_USER_ALIGN (sfield
) = 0;
2468 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2469 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2470 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2475 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2477 layout_type (ctx
->record_type
);
2478 fixup_child_record_type (ctx
);
2482 location_t loc
= gimple_location (ctx
->stmt
);
2483 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2484 /* Move VLA fields to the end. */
2485 p
= &TYPE_FIELDS (ctx
->record_type
);
2487 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2488 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2491 *p
= TREE_CHAIN (*p
);
2492 TREE_CHAIN (*q
) = NULL_TREE
;
2493 q
= &TREE_CHAIN (*q
);
2496 p
= &DECL_CHAIN (*p
);
2498 layout_type (ctx
->record_type
);
2499 fixup_child_record_type (ctx
);
2500 if (ctx
->srecord_type
)
2501 layout_type (ctx
->srecord_type
);
2502 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2503 TYPE_SIZE_UNIT (ctx
->record_type
));
2504 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2505 t
= build_int_cst (long_integer_type_node
,
2506 TYPE_ALIGN_UNIT (ctx
->record_type
));
2507 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2512 static omp_context
*
2513 enclosing_target_ctx (omp_context
*ctx
)
2516 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2518 gcc_assert (ctx
!= NULL
);
2523 oacc_loop_or_target_p (gimple stmt
)
2525 enum gimple_code outer_type
= gimple_code (stmt
);
2526 return ((outer_type
== GIMPLE_OMP_TARGET
2527 && ((gimple_omp_target_kind (stmt
)
2528 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2529 || (gimple_omp_target_kind (stmt
)
2530 == GF_OMP_TARGET_KIND_OACC_KERNELS
)))
2531 || (outer_type
== GIMPLE_OMP_FOR
2532 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
));
2535 /* Scan a GIMPLE_OMP_FOR. */
2538 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2540 enum gimple_code outer_type
= GIMPLE_ERROR_MARK
;
2543 tree clauses
= gimple_omp_for_clauses (stmt
);
2546 outer_type
= gimple_code (outer_ctx
->stmt
);
2548 ctx
= new_omp_context (stmt
, outer_ctx
);
2550 if (is_gimple_omp_oacc (stmt
))
2552 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2553 ctx
->gwv_this
= outer_ctx
->gwv_this
;
2554 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2557 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_GANG
)
2559 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WORKER
)
2561 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR
)
2565 ctx
->gwv_this
|= val
;
2568 /* Skip; not nested inside a region. */
2571 if (!oacc_loop_or_target_p (outer_ctx
->stmt
))
2573 /* Skip; not nested inside an OpenACC region. */
2576 if (outer_type
== GIMPLE_OMP_FOR
)
2577 outer_ctx
->gwv_below
|= val
;
2578 if (OMP_CLAUSE_OPERAND (c
, 0) != NULL_TREE
)
2580 omp_context
*enclosing
= enclosing_target_ctx (outer_ctx
);
2581 if (gimple_omp_target_kind (enclosing
->stmt
)
2582 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2583 error_at (gimple_location (stmt
),
2584 "no arguments allowed to gang, worker and vector clauses inside parallel");
2589 scan_sharing_clauses (clauses
, ctx
);
2591 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2592 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2594 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2595 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2596 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2597 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2599 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2601 if (is_gimple_omp_oacc (stmt
))
2603 if (ctx
->gwv_this
& ctx
->gwv_below
)
2604 error_at (gimple_location (stmt
),
2605 "gang, worker and vector may occur only once in a loop nest");
2606 else if (ctx
->gwv_below
!= 0
2607 && ctx
->gwv_this
> ctx
->gwv_below
)
2608 error_at (gimple_location (stmt
),
2609 "gang, worker and vector must occur in this order in a loop nest");
2610 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2611 outer_ctx
->gwv_below
|= ctx
->gwv_below
;
2615 /* Scan an OpenMP sections directive. */
2618 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
2622 ctx
= new_omp_context (stmt
, outer_ctx
);
2623 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2624 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2627 /* Scan an OpenMP single directive. */
2630 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
2635 ctx
= new_omp_context (stmt
, outer_ctx
);
2636 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2637 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2638 name
= create_tmp_var_name (".omp_copy_s");
2639 name
= build_decl (gimple_location (stmt
),
2640 TYPE_DECL
, name
, ctx
->record_type
);
2641 TYPE_NAME (ctx
->record_type
) = name
;
2643 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2644 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2646 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2647 ctx
->record_type
= NULL
;
2649 layout_type (ctx
->record_type
);
2652 /* Scan a GIMPLE_OMP_TARGET. */
2655 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
2659 bool offloaded
= is_gimple_omp_offloaded (stmt
);
2660 tree clauses
= gimple_omp_target_clauses (stmt
);
2662 ctx
= new_omp_context (stmt
, outer_ctx
);
2663 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2664 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2665 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2666 name
= create_tmp_var_name (".omp_data_t");
2667 name
= build_decl (gimple_location (stmt
),
2668 TYPE_DECL
, name
, ctx
->record_type
);
2669 DECL_ARTIFICIAL (name
) = 1;
2670 DECL_NAMELESS (name
) = 1;
2671 TYPE_NAME (ctx
->record_type
) = name
;
2674 if (is_gimple_omp_oacc (stmt
))
2675 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
2678 create_omp_child_function (ctx
, false);
2679 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2682 if (is_gimple_omp_oacc (stmt
))
2684 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2686 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_GANGS
)
2687 ctx
->gwv_this
|= MASK_GANG
;
2688 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_WORKERS
)
2689 ctx
->gwv_this
|= MASK_WORKER
;
2690 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR_LENGTH
)
2691 ctx
->gwv_this
|= MASK_VECTOR
;
2695 scan_sharing_clauses (clauses
, ctx
);
2696 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2698 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2699 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2702 TYPE_FIELDS (ctx
->record_type
)
2703 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2704 #ifdef ENABLE_CHECKING
2706 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2707 for (field
= TYPE_FIELDS (ctx
->record_type
);
2709 field
= DECL_CHAIN (field
))
2710 gcc_assert (DECL_ALIGN (field
) == align
);
2712 layout_type (ctx
->record_type
);
2714 fixup_child_record_type (ctx
);
2718 /* Scan an OpenMP teams directive. */
2721 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
2723 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2724 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2725 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2728 /* Check nesting restrictions. */
2730 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2732 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2733 inside an OpenACC CTX. */
2734 if (!(is_gimple_omp (stmt
)
2735 && is_gimple_omp_oacc (stmt
)))
2737 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
2738 if (is_gimple_omp (ctx_
->stmt
)
2739 && is_gimple_omp_oacc (ctx_
->stmt
))
2741 error_at (gimple_location (stmt
),
2742 "non-OpenACC construct inside of OpenACC region");
2749 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2750 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2752 error_at (gimple_location (stmt
),
2753 "OpenMP constructs may not be nested inside simd region");
2756 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2758 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2759 || (gimple_omp_for_kind (stmt
)
2760 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2761 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2763 error_at (gimple_location (stmt
),
2764 "only distribute or parallel constructs are allowed to "
2765 "be closely nested inside teams construct");
2770 switch (gimple_code (stmt
))
2772 case GIMPLE_OMP_FOR
:
2773 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2775 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2777 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2779 error_at (gimple_location (stmt
),
2780 "distribute construct must be closely nested inside "
2788 if (is_gimple_call (stmt
)
2789 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2790 == BUILT_IN_GOMP_CANCEL
2791 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2792 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2794 const char *bad
= NULL
;
2795 const char *kind
= NULL
;
2798 error_at (gimple_location (stmt
), "orphaned %qs construct",
2799 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2800 == BUILT_IN_GOMP_CANCEL
2801 ? "#pragma omp cancel"
2802 : "#pragma omp cancellation point");
2805 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2806 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2810 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2811 bad
= "#pragma omp parallel";
2812 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2813 == BUILT_IN_GOMP_CANCEL
2814 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2815 ctx
->cancellable
= true;
2819 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2820 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2821 bad
= "#pragma omp for";
2822 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2823 == BUILT_IN_GOMP_CANCEL
2824 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2826 ctx
->cancellable
= true;
2827 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2829 warning_at (gimple_location (stmt
), 0,
2830 "%<#pragma omp cancel for%> inside "
2831 "%<nowait%> for construct");
2832 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2833 OMP_CLAUSE_ORDERED
))
2834 warning_at (gimple_location (stmt
), 0,
2835 "%<#pragma omp cancel for%> inside "
2836 "%<ordered%> for construct");
2841 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2842 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2843 bad
= "#pragma omp sections";
2844 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2845 == BUILT_IN_GOMP_CANCEL
2846 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2848 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2850 ctx
->cancellable
= true;
2851 if (find_omp_clause (gimple_omp_sections_clauses
2854 warning_at (gimple_location (stmt
), 0,
2855 "%<#pragma omp cancel sections%> inside "
2856 "%<nowait%> sections construct");
2860 gcc_assert (ctx
->outer
2861 && gimple_code (ctx
->outer
->stmt
)
2862 == GIMPLE_OMP_SECTIONS
);
2863 ctx
->outer
->cancellable
= true;
2864 if (find_omp_clause (gimple_omp_sections_clauses
2867 warning_at (gimple_location (stmt
), 0,
2868 "%<#pragma omp cancel sections%> inside "
2869 "%<nowait%> sections construct");
2875 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2876 bad
= "#pragma omp task";
2878 ctx
->cancellable
= true;
2882 error_at (gimple_location (stmt
), "invalid arguments");
2887 error_at (gimple_location (stmt
),
2888 "%<%s %s%> construct not closely nested inside of %qs",
2889 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2890 == BUILT_IN_GOMP_CANCEL
2891 ? "#pragma omp cancel"
2892 : "#pragma omp cancellation point", kind
, bad
);
2897 case GIMPLE_OMP_SECTIONS
:
2898 case GIMPLE_OMP_SINGLE
:
2899 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2900 switch (gimple_code (ctx
->stmt
))
2902 case GIMPLE_OMP_FOR
:
2903 case GIMPLE_OMP_SECTIONS
:
2904 case GIMPLE_OMP_SINGLE
:
2905 case GIMPLE_OMP_ORDERED
:
2906 case GIMPLE_OMP_MASTER
:
2907 case GIMPLE_OMP_TASK
:
2908 case GIMPLE_OMP_CRITICAL
:
2909 if (is_gimple_call (stmt
))
2911 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2912 != BUILT_IN_GOMP_BARRIER
)
2914 error_at (gimple_location (stmt
),
2915 "barrier region may not be closely nested inside "
2916 "of work-sharing, critical, ordered, master or "
2917 "explicit task region");
2920 error_at (gimple_location (stmt
),
2921 "work-sharing region may not be closely nested inside "
2922 "of work-sharing, critical, ordered, master or explicit "
2925 case GIMPLE_OMP_PARALLEL
:
2931 case GIMPLE_OMP_MASTER
:
2932 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2933 switch (gimple_code (ctx
->stmt
))
2935 case GIMPLE_OMP_FOR
:
2936 case GIMPLE_OMP_SECTIONS
:
2937 case GIMPLE_OMP_SINGLE
:
2938 case GIMPLE_OMP_TASK
:
2939 error_at (gimple_location (stmt
),
2940 "master region may not be closely nested inside "
2941 "of work-sharing or explicit task region");
2943 case GIMPLE_OMP_PARALLEL
:
2949 case GIMPLE_OMP_ORDERED
:
2950 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2951 switch (gimple_code (ctx
->stmt
))
2953 case GIMPLE_OMP_CRITICAL
:
2954 case GIMPLE_OMP_TASK
:
2955 error_at (gimple_location (stmt
),
2956 "ordered region may not be closely nested inside "
2957 "of critical or explicit task region");
2959 case GIMPLE_OMP_FOR
:
2960 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2961 OMP_CLAUSE_ORDERED
) == NULL
)
2963 error_at (gimple_location (stmt
),
2964 "ordered region must be closely nested inside "
2965 "a loop region with an ordered clause");
2969 case GIMPLE_OMP_PARALLEL
:
2970 error_at (gimple_location (stmt
),
2971 "ordered region must be closely nested inside "
2972 "a loop region with an ordered clause");
2978 case GIMPLE_OMP_CRITICAL
:
2981 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
2982 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2983 if (gomp_critical
*other_crit
2984 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
2985 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
2987 error_at (gimple_location (stmt
),
2988 "critical region may not be nested inside a critical "
2989 "region with the same name");
2994 case GIMPLE_OMP_TEAMS
:
2996 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2997 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2999 error_at (gimple_location (stmt
),
3000 "teams construct not closely nested inside of target "
3005 case GIMPLE_OMP_TARGET
:
3006 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3008 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3010 if (is_gimple_omp (stmt
)
3011 && is_gimple_omp_oacc (stmt
)
3012 && is_gimple_omp (ctx
->stmt
))
3014 error_at (gimple_location (stmt
),
3015 "OpenACC construct inside of non-OpenACC region");
3021 const char *stmt_name
, *ctx_stmt_name
;
3022 switch (gimple_omp_target_kind (stmt
))
3024 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3025 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3026 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3027 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3028 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3029 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3030 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3031 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
: stmt_name
= "enter/exit data"; break;
3032 default: gcc_unreachable ();
3034 switch (gimple_omp_target_kind (ctx
->stmt
))
3036 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3037 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3038 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: ctx_stmt_name
= "parallel"; break;
3039 case GF_OMP_TARGET_KIND_OACC_KERNELS
: ctx_stmt_name
= "kernels"; break;
3040 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3041 default: gcc_unreachable ();
3044 /* OpenACC/OpenMP mismatch? */
3045 if (is_gimple_omp_oacc (stmt
)
3046 != is_gimple_omp_oacc (ctx
->stmt
))
3048 error_at (gimple_location (stmt
),
3049 "%s %s construct inside of %s %s region",
3050 (is_gimple_omp_oacc (stmt
)
3051 ? "OpenACC" : "OpenMP"), stmt_name
,
3052 (is_gimple_omp_oacc (ctx
->stmt
)
3053 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3056 if (is_gimple_omp_offloaded (ctx
->stmt
))
3058 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3059 if (is_gimple_omp_oacc (ctx
->stmt
))
3061 error_at (gimple_location (stmt
),
3062 "%s construct inside of %s region",
3063 stmt_name
, ctx_stmt_name
);
3068 gcc_checking_assert (!is_gimple_omp_oacc (stmt
));
3069 warning_at (gimple_location (stmt
), 0,
3070 "%s construct inside of %s region",
3071 stmt_name
, ctx_stmt_name
);
3083 /* Helper function scan_omp.
3085 Callback for walk_tree or operators in walk_gimple_stmt used to
3086 scan for OMP directives in TP. */
3089 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3091 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3092 omp_context
*ctx
= (omp_context
*) wi
->info
;
3095 switch (TREE_CODE (t
))
3102 *tp
= remap_decl (t
, &ctx
->cb
);
3106 if (ctx
&& TYPE_P (t
))
3107 *tp
= remap_type (t
, &ctx
->cb
);
3108 else if (!DECL_P (t
))
3113 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3114 if (tem
!= TREE_TYPE (t
))
3116 if (TREE_CODE (t
) == INTEGER_CST
)
3117 *tp
= wide_int_to_tree (tem
, t
);
3119 TREE_TYPE (t
) = tem
;
3129 /* Return true if FNDECL is a setjmp or a longjmp. */
3132 setjmp_or_longjmp_p (const_tree fndecl
)
3134 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3135 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3136 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3139 tree declname
= DECL_NAME (fndecl
);
3142 const char *name
= IDENTIFIER_POINTER (declname
);
3143 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3147 /* Helper function for scan_omp.
3149 Callback for walk_gimple_stmt used to scan for OMP directives in
3150 the current statement in GSI. */
3153 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3154 struct walk_stmt_info
*wi
)
3156 gimple stmt
= gsi_stmt (*gsi
);
3157 omp_context
*ctx
= (omp_context
*) wi
->info
;
3159 if (gimple_has_location (stmt
))
3160 input_location
= gimple_location (stmt
);
3162 /* Check the nesting restrictions. */
3163 bool remove
= false;
3164 if (is_gimple_omp (stmt
))
3165 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3166 else if (is_gimple_call (stmt
))
3168 tree fndecl
= gimple_call_fndecl (stmt
);
3171 if (setjmp_or_longjmp_p (fndecl
)
3173 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3174 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3177 error_at (gimple_location (stmt
),
3178 "setjmp/longjmp inside simd construct");
3180 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3181 switch (DECL_FUNCTION_CODE (fndecl
))
3183 case BUILT_IN_GOMP_BARRIER
:
3184 case BUILT_IN_GOMP_CANCEL
:
3185 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3186 case BUILT_IN_GOMP_TASKYIELD
:
3187 case BUILT_IN_GOMP_TASKWAIT
:
3188 case BUILT_IN_GOMP_TASKGROUP_START
:
3189 case BUILT_IN_GOMP_TASKGROUP_END
:
3190 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3199 stmt
= gimple_build_nop ();
3200 gsi_replace (gsi
, stmt
, false);
3203 *handled_ops_p
= true;
3205 switch (gimple_code (stmt
))
3207 case GIMPLE_OMP_PARALLEL
:
3208 taskreg_nesting_level
++;
3209 scan_omp_parallel (gsi
, ctx
);
3210 taskreg_nesting_level
--;
3213 case GIMPLE_OMP_TASK
:
3214 taskreg_nesting_level
++;
3215 scan_omp_task (gsi
, ctx
);
3216 taskreg_nesting_level
--;
3219 case GIMPLE_OMP_FOR
:
3220 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3223 case GIMPLE_OMP_SECTIONS
:
3224 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3227 case GIMPLE_OMP_SINGLE
:
3228 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3231 case GIMPLE_OMP_SECTION
:
3232 case GIMPLE_OMP_MASTER
:
3233 case GIMPLE_OMP_TASKGROUP
:
3234 case GIMPLE_OMP_ORDERED
:
3235 case GIMPLE_OMP_CRITICAL
:
3236 ctx
= new_omp_context (stmt
, ctx
);
3237 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3240 case GIMPLE_OMP_TARGET
:
3241 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3244 case GIMPLE_OMP_TEAMS
:
3245 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3252 *handled_ops_p
= false;
3254 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3256 var
= DECL_CHAIN (var
))
3257 insert_decl_map (&ctx
->cb
, var
, var
);
3261 *handled_ops_p
= false;
3269 /* Scan all the statements starting at the current statement. CTX
3270 contains context information about the OMP directives and
3271 clauses found during the scan. */
3274 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3276 location_t saved_location
;
3277 struct walk_stmt_info wi
;
3279 memset (&wi
, 0, sizeof (wi
));
3281 wi
.want_locations
= true;
3283 saved_location
= input_location
;
3284 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3285 input_location
= saved_location
;
3288 /* Re-gimplification and code generation routines. */
3290 /* Build a call to GOMP_barrier. */
3293 build_omp_barrier (tree lhs
)
3295 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3296 : BUILT_IN_GOMP_BARRIER
);
3297 gcall
*g
= gimple_build_call (fndecl
, 0);
3299 gimple_call_set_lhs (g
, lhs
);
3303 /* If a context was created for STMT when it was scanned, return it. */
3305 static omp_context
*
3306 maybe_lookup_ctx (gimple stmt
)
3309 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3310 return n
? (omp_context
*) n
->value
: NULL
;
3314 /* Find the mapping for DECL in CTX or the immediately enclosing
3315 context that has a mapping for DECL.
3317 If CTX is a nested parallel directive, we may have to use the decl
3318 mappings created in CTX's parent context. Suppose that we have the
3319 following parallel nesting (variable UIDs showed for clarity):
3322 #omp parallel shared(iD.1562) -> outer parallel
3323 iD.1562 = iD.1562 + 1;
3325 #omp parallel shared (iD.1562) -> inner parallel
3326 iD.1562 = iD.1562 - 1;
3328 Each parallel structure will create a distinct .omp_data_s structure
3329 for copying iD.1562 in/out of the directive:
3331 outer parallel .omp_data_s.1.i -> iD.1562
3332 inner parallel .omp_data_s.2.i -> iD.1562
3334 A shared variable mapping will produce a copy-out operation before
3335 the parallel directive and a copy-in operation after it. So, in
3336 this case we would have:
3339 .omp_data_o.1.i = iD.1562;
3340 #omp parallel shared(iD.1562) -> outer parallel
3341 .omp_data_i.1 = &.omp_data_o.1
3342 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3344 .omp_data_o.2.i = iD.1562; -> **
3345 #omp parallel shared(iD.1562) -> inner parallel
3346 .omp_data_i.2 = &.omp_data_o.2
3347 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3350 ** This is a problem. The symbol iD.1562 cannot be referenced
3351 inside the body of the outer parallel region. But since we are
3352 emitting this copy operation while expanding the inner parallel
3353 directive, we need to access the CTX structure of the outer
3354 parallel directive to get the correct mapping:
3356 .omp_data_o.2.i = .omp_data_i.1->i
3358 Since there may be other workshare or parallel directives enclosing
3359 the parallel directive, it may be necessary to walk up the context
3360 parent chain. This is not a problem in general because nested
3361 parallelism happens only rarely. */
3364 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3369 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3370 t
= maybe_lookup_decl (decl
, up
);
3372 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3374 return t
? t
: decl
;
3378 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3379 in outer contexts. */
3382 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3387 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3388 t
= maybe_lookup_decl (decl
, up
);
3390 return t
? t
: decl
;
3394 /* Construct the initialization value for reduction CLAUSE. */
3397 omp_reduction_init (tree clause
, tree type
)
3399 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3400 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3407 case TRUTH_ORIF_EXPR
:
3408 case TRUTH_XOR_EXPR
:
3410 return build_zero_cst (type
);
3413 case TRUTH_AND_EXPR
:
3414 case TRUTH_ANDIF_EXPR
:
3416 return fold_convert_loc (loc
, type
, integer_one_node
);
3419 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3422 if (SCALAR_FLOAT_TYPE_P (type
))
3424 REAL_VALUE_TYPE max
, min
;
3425 if (HONOR_INFINITIES (type
))
3428 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3431 real_maxval (&min
, 1, TYPE_MODE (type
));
3432 return build_real (type
, min
);
3436 gcc_assert (INTEGRAL_TYPE_P (type
));
3437 return TYPE_MIN_VALUE (type
);
3441 if (SCALAR_FLOAT_TYPE_P (type
))
3443 REAL_VALUE_TYPE max
;
3444 if (HONOR_INFINITIES (type
))
3447 real_maxval (&max
, 0, TYPE_MODE (type
));
3448 return build_real (type
, max
);
3452 gcc_assert (INTEGRAL_TYPE_P (type
));
3453 return TYPE_MAX_VALUE (type
);
3461 /* Return alignment to be assumed for var in CLAUSE, which should be
3462 OMP_CLAUSE_ALIGNED. */
3465 omp_clause_aligned_alignment (tree clause
)
3467 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3468 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3470 /* Otherwise return implementation defined alignment. */
3471 unsigned int al
= 1;
3472 machine_mode mode
, vmode
;
3473 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3475 vs
= 1 << floor_log2 (vs
);
3476 static enum mode_class classes
[]
3477 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3478 for (int i
= 0; i
< 4; i
+= 2)
3479 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3481 mode
= GET_MODE_WIDER_MODE (mode
))
3483 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3484 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3487 && GET_MODE_SIZE (vmode
) < vs
3488 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3489 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3491 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3492 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3494 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3495 / GET_MODE_SIZE (mode
));
3496 if (TYPE_MODE (type
) != vmode
)
3498 if (TYPE_ALIGN_UNIT (type
) > al
)
3499 al
= TYPE_ALIGN_UNIT (type
);
3501 return build_int_cst (integer_type_node
, al
);
3504 /* Return maximum possible vectorization factor for the target. */
3511 || !flag_tree_loop_optimize
3512 || (!flag_tree_loop_vectorize
3513 && (global_options_set
.x_flag_tree_loop_vectorize
3514 || global_options_set
.x_flag_tree_vectorize
)))
3517 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3520 vs
= 1 << floor_log2 (vs
);
3523 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3524 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3525 return GET_MODE_NUNITS (vqimode
);
3529 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3533 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3534 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3538 max_vf
= omp_max_vf ();
3541 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3542 OMP_CLAUSE_SAFELEN
);
3543 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3545 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3547 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3551 idx
= create_tmp_var (unsigned_type_node
);
3552 lane
= create_tmp_var (unsigned_type_node
);
3558 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3559 tree avar
= create_tmp_var_raw (atype
);
3560 if (TREE_ADDRESSABLE (new_var
))
3561 TREE_ADDRESSABLE (avar
) = 1;
3562 DECL_ATTRIBUTES (avar
)
3563 = tree_cons (get_identifier ("omp simd array"), NULL
,
3564 DECL_ATTRIBUTES (avar
));
3565 gimple_add_tmp_var (avar
);
3566 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3567 NULL_TREE
, NULL_TREE
);
3568 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3569 NULL_TREE
, NULL_TREE
);
3570 if (DECL_P (new_var
))
3572 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3573 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3578 /* Helper function of lower_rec_input_clauses. For a reference
3579 in simd reduction, add an underlying variable it will reference. */
3582 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3584 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3585 if (TREE_CONSTANT (z
))
3587 const char *name
= NULL
;
3588 if (DECL_NAME (new_vard
))
3589 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3591 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3592 gimple_add_tmp_var (z
);
3593 TREE_ADDRESSABLE (z
) = 1;
3594 z
= build_fold_addr_expr_loc (loc
, z
);
3595 gimplify_assign (new_vard
, z
, ilist
);
3599 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3600 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3601 private variables. Initialization statements go in ILIST, while calls
3602 to destructors go in DLIST. */
3605 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3606 omp_context
*ctx
, struct omp_for_data
*fd
)
3608 tree c
, dtor
, copyin_seq
, x
, ptr
;
3609 bool copyin_by_ref
= false;
3610 bool lastprivate_firstprivate
= false;
3611 bool reduction_omp_orig_ref
= false;
3613 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3614 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3616 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3617 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3618 gimple_seq llist
[2] = { NULL
, NULL
};
3622 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3623 with data sharing clauses referencing variable sized vars. That
3624 is unnecessarily hard to support and very unlikely to result in
3625 vectorized code anyway. */
3627 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3628 switch (OMP_CLAUSE_CODE (c
))
3630 case OMP_CLAUSE_LINEAR
:
3631 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3634 case OMP_CLAUSE_REDUCTION
:
3635 case OMP_CLAUSE_PRIVATE
:
3636 case OMP_CLAUSE_FIRSTPRIVATE
:
3637 case OMP_CLAUSE_LASTPRIVATE
:
3638 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3645 /* Do all the fixed sized types in the first pass, and the variable sized
3646 types in the second pass. This makes sure that the scalar arguments to
3647 the variable sized types are processed before we use them in the
3648 variable sized operations. */
3649 for (pass
= 0; pass
< 2; ++pass
)
3651 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3653 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3656 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3660 case OMP_CLAUSE_PRIVATE
:
3661 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3664 case OMP_CLAUSE_SHARED
:
3665 /* Ignore shared directives in teams construct. */
3666 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3668 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3670 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3673 case OMP_CLAUSE_FIRSTPRIVATE
:
3674 case OMP_CLAUSE_COPYIN
:
3675 case OMP_CLAUSE_LINEAR
:
3677 case OMP_CLAUSE_REDUCTION
:
3678 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3679 reduction_omp_orig_ref
= true;
3681 case OMP_CLAUSE__LOOPTEMP_
:
3682 /* Handle _looptemp_ clauses only on parallel. */
3686 case OMP_CLAUSE_LASTPRIVATE
:
3687 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3689 lastprivate_firstprivate
= true;
3693 /* Even without corresponding firstprivate, if
3694 decl is Fortran allocatable, it needs outer var
3697 && lang_hooks
.decls
.omp_private_outer_ref
3698 (OMP_CLAUSE_DECL (c
)))
3699 lastprivate_firstprivate
= true;
3701 case OMP_CLAUSE_ALIGNED
:
3704 var
= OMP_CLAUSE_DECL (c
);
3705 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3706 && !is_global_var (var
))
3708 new_var
= maybe_lookup_decl (var
, ctx
);
3709 if (new_var
== NULL_TREE
)
3710 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3711 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3712 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3713 omp_clause_aligned_alignment (c
));
3714 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3715 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3716 gimplify_and_add (x
, ilist
);
3718 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3719 && is_global_var (var
))
3721 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3722 new_var
= lookup_decl (var
, ctx
);
3723 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3724 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3725 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3726 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3727 omp_clause_aligned_alignment (c
));
3728 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3729 x
= create_tmp_var (ptype
);
3730 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3731 gimplify_and_add (t
, ilist
);
3732 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3733 SET_DECL_VALUE_EXPR (new_var
, t
);
3734 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3741 new_var
= var
= OMP_CLAUSE_DECL (c
);
3742 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3743 new_var
= lookup_decl (var
, ctx
);
3745 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3750 else if (is_variable_sized (var
))
3752 /* For variable sized types, we need to allocate the
3753 actual storage here. Call alloca and store the
3754 result in the pointer decl that we created elsewhere. */
3758 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3763 ptr
= DECL_VALUE_EXPR (new_var
);
3764 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3765 ptr
= TREE_OPERAND (ptr
, 0);
3766 gcc_assert (DECL_P (ptr
));
3767 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3769 /* void *tmp = __builtin_alloca */
3770 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3771 stmt
= gimple_build_call (atmp
, 1, x
);
3772 tmp
= create_tmp_var_raw (ptr_type_node
);
3773 gimple_add_tmp_var (tmp
);
3774 gimple_call_set_lhs (stmt
, tmp
);
3776 gimple_seq_add_stmt (ilist
, stmt
);
3778 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3779 gimplify_assign (ptr
, x
, ilist
);
3782 else if (is_reference (var
))
3784 /* For references that are being privatized for Fortran,
3785 allocate new backing storage for the new pointer
3786 variable. This allows us to avoid changing all the
3787 code that expects a pointer to something that expects
3788 a direct variable. */
3792 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3793 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3795 x
= build_receiver_ref (var
, false, ctx
);
3796 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3798 else if (TREE_CONSTANT (x
))
3800 /* For reduction in SIMD loop, defer adding the
3801 initialization of the reference, because if we decide
3802 to use SIMD array for it, the initilization could cause
3804 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3808 const char *name
= NULL
;
3809 if (DECL_NAME (var
))
3810 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3812 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3814 gimple_add_tmp_var (x
);
3815 TREE_ADDRESSABLE (x
) = 1;
3816 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3821 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3822 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3827 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3828 gimplify_assign (new_var
, x
, ilist
);
3831 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3833 else if (c_kind
== OMP_CLAUSE_REDUCTION
3834 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3842 switch (OMP_CLAUSE_CODE (c
))
3844 case OMP_CLAUSE_SHARED
:
3845 /* Ignore shared directives in teams construct. */
3846 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3848 /* Shared global vars are just accessed directly. */
3849 if (is_global_var (new_var
))
3851 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3852 needs to be delayed until after fixup_child_record_type so
3853 that we get the correct type during the dereference. */
3854 by_ref
= use_pointer_for_field (var
, ctx
);
3855 x
= build_receiver_ref (var
, by_ref
, ctx
);
3856 SET_DECL_VALUE_EXPR (new_var
, x
);
3857 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3859 /* ??? If VAR is not passed by reference, and the variable
3860 hasn't been initialized yet, then we'll get a warning for
3861 the store into the omp_data_s structure. Ideally, we'd be
3862 able to notice this and not store anything at all, but
3863 we're generating code too early. Suppress the warning. */
3865 TREE_NO_WARNING (var
) = 1;
3868 case OMP_CLAUSE_LASTPRIVATE
:
3869 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3873 case OMP_CLAUSE_PRIVATE
:
3874 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3875 x
= build_outer_var_ref (var
, ctx
);
3876 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3878 if (is_task_ctx (ctx
))
3879 x
= build_receiver_ref (var
, false, ctx
);
3881 x
= build_outer_var_ref (var
, ctx
);
3887 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3890 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3891 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3892 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3893 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3894 idx
, lane
, ivar
, lvar
))
3897 x
= lang_hooks
.decls
.omp_clause_default_ctor
3898 (c
, unshare_expr (ivar
), x
);
3900 gimplify_and_add (x
, &llist
[0]);
3903 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3906 gimple_seq tseq
= NULL
;
3909 gimplify_stmt (&dtor
, &tseq
);
3910 gimple_seq_add_seq (&llist
[1], tseq
);
3917 gimplify_and_add (nx
, ilist
);
3921 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3924 gimple_seq tseq
= NULL
;
3927 gimplify_stmt (&dtor
, &tseq
);
3928 gimple_seq_add_seq (dlist
, tseq
);
3932 case OMP_CLAUSE_LINEAR
:
3933 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3934 goto do_firstprivate
;
3935 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3938 x
= build_outer_var_ref (var
, ctx
);
3941 case OMP_CLAUSE_FIRSTPRIVATE
:
3942 if (is_task_ctx (ctx
))
3944 if (is_reference (var
) || is_variable_sized (var
))
3946 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3948 || use_pointer_for_field (var
, NULL
))
3950 x
= build_receiver_ref (var
, false, ctx
);
3951 SET_DECL_VALUE_EXPR (new_var
, x
);
3952 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3957 x
= build_outer_var_ref (var
, ctx
);
3960 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3961 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3963 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3964 tree stept
= TREE_TYPE (t
);
3965 tree ct
= find_omp_clause (clauses
,
3966 OMP_CLAUSE__LOOPTEMP_
);
3968 tree l
= OMP_CLAUSE_DECL (ct
);
3969 tree n1
= fd
->loop
.n1
;
3970 tree step
= fd
->loop
.step
;
3971 tree itype
= TREE_TYPE (l
);
3972 if (POINTER_TYPE_P (itype
))
3973 itype
= signed_type_for (itype
);
3974 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3975 if (TYPE_UNSIGNED (itype
)
3976 && fd
->loop
.cond_code
== GT_EXPR
)
3977 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3978 fold_build1 (NEGATE_EXPR
, itype
, l
),
3979 fold_build1 (NEGATE_EXPR
,
3982 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3983 t
= fold_build2 (MULT_EXPR
, stept
,
3984 fold_convert (stept
, l
), t
);
3986 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3988 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3990 gimplify_and_add (x
, ilist
);
3994 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3995 x
= fold_build2 (POINTER_PLUS_EXPR
,
3996 TREE_TYPE (x
), x
, t
);
3998 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
4001 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
4002 || TREE_ADDRESSABLE (new_var
))
4003 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4004 idx
, lane
, ivar
, lvar
))
4006 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
4008 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
4009 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
4010 gimplify_and_add (x
, ilist
);
4011 gimple_stmt_iterator gsi
4012 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4014 = gimple_build_assign (unshare_expr (lvar
), iv
);
4015 gsi_insert_before_without_update (&gsi
, g
,
4017 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4018 enum tree_code code
= PLUS_EXPR
;
4019 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4020 code
= POINTER_PLUS_EXPR
;
4021 g
= gimple_build_assign (iv
, code
, iv
, t
);
4022 gsi_insert_before_without_update (&gsi
, g
,
4026 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4027 (c
, unshare_expr (ivar
), x
);
4028 gimplify_and_add (x
, &llist
[0]);
4029 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4032 gimple_seq tseq
= NULL
;
4035 gimplify_stmt (&dtor
, &tseq
);
4036 gimple_seq_add_seq (&llist
[1], tseq
);
4041 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
4042 gimplify_and_add (x
, ilist
);
4045 case OMP_CLAUSE__LOOPTEMP_
:
4046 gcc_assert (is_parallel_ctx (ctx
));
4047 x
= build_outer_var_ref (var
, ctx
);
4048 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4049 gimplify_and_add (x
, ilist
);
4052 case OMP_CLAUSE_COPYIN
:
4053 by_ref
= use_pointer_for_field (var
, NULL
);
4054 x
= build_receiver_ref (var
, by_ref
, ctx
);
4055 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4056 append_to_statement_list (x
, ©in_seq
);
4057 copyin_by_ref
|= by_ref
;
4060 case OMP_CLAUSE_REDUCTION
:
4061 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4063 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4065 x
= build_outer_var_ref (var
, ctx
);
4067 if (is_reference (var
)
4068 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4070 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4071 SET_DECL_VALUE_EXPR (placeholder
, x
);
4072 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4073 tree new_vard
= new_var
;
4074 if (is_reference (var
))
4076 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4077 new_vard
= TREE_OPERAND (new_var
, 0);
4078 gcc_assert (DECL_P (new_vard
));
4081 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4082 idx
, lane
, ivar
, lvar
))
4084 if (new_vard
== new_var
)
4086 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4087 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4091 SET_DECL_VALUE_EXPR (new_vard
,
4092 build_fold_addr_expr (ivar
));
4093 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4095 x
= lang_hooks
.decls
.omp_clause_default_ctor
4096 (c
, unshare_expr (ivar
),
4097 build_outer_var_ref (var
, ctx
));
4099 gimplify_and_add (x
, &llist
[0]);
4100 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4102 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4103 lower_omp (&tseq
, ctx
);
4104 gimple_seq_add_seq (&llist
[0], tseq
);
4106 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4107 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4108 lower_omp (&tseq
, ctx
);
4109 gimple_seq_add_seq (&llist
[1], tseq
);
4110 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4111 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4112 if (new_vard
== new_var
)
4113 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4115 SET_DECL_VALUE_EXPR (new_vard
,
4116 build_fold_addr_expr (lvar
));
4117 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4122 gimplify_stmt (&dtor
, &tseq
);
4123 gimple_seq_add_seq (&llist
[1], tseq
);
4127 /* If this is a reference to constant size reduction var
4128 with placeholder, we haven't emitted the initializer
4129 for it because it is undesirable if SIMD arrays are used.
4130 But if they aren't used, we need to emit the deferred
4131 initialization now. */
4132 else if (is_reference (var
) && is_simd
)
4133 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4134 x
= lang_hooks
.decls
.omp_clause_default_ctor
4135 (c
, unshare_expr (new_var
),
4136 build_outer_var_ref (var
, ctx
));
4138 gimplify_and_add (x
, ilist
);
4139 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4141 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4142 lower_omp (&tseq
, ctx
);
4143 gimple_seq_add_seq (ilist
, tseq
);
4145 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4148 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4149 lower_omp (&tseq
, ctx
);
4150 gimple_seq_add_seq (dlist
, tseq
);
4151 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4153 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4158 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4159 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4160 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4162 /* reduction(-:var) sums up the partial results, so it
4163 acts identically to reduction(+:var). */
4164 if (code
== MINUS_EXPR
)
4167 tree new_vard
= new_var
;
4168 if (is_simd
&& is_reference (var
))
4170 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4171 new_vard
= TREE_OPERAND (new_var
, 0);
4172 gcc_assert (DECL_P (new_vard
));
4175 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4176 idx
, lane
, ivar
, lvar
))
4178 tree ref
= build_outer_var_ref (var
, ctx
);
4180 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
4182 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
4183 ref
= build_outer_var_ref (var
, ctx
);
4184 gimplify_assign (ref
, x
, &llist
[1]);
4186 if (new_vard
!= new_var
)
4188 SET_DECL_VALUE_EXPR (new_vard
,
4189 build_fold_addr_expr (lvar
));
4190 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4195 if (is_reference (var
) && is_simd
)
4196 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4197 gimplify_assign (new_var
, x
, ilist
);
4200 tree ref
= build_outer_var_ref (var
, ctx
);
4202 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4203 ref
= build_outer_var_ref (var
, ctx
);
4204 gimplify_assign (ref
, x
, dlist
);
4218 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
4219 /* Don't want uninit warnings on simduid, it is always uninitialized,
4220 but we use it not for the value, but for the DECL_UID only. */
4221 TREE_NO_WARNING (uid
) = 1;
4223 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
4224 gimple_call_set_lhs (g
, lane
);
4225 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4226 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
4227 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
4228 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
4229 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4230 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4231 g
= gimple_build_assign (lane
, INTEGER_CST
,
4232 build_int_cst (unsigned_type_node
, 0));
4233 gimple_seq_add_stmt (ilist
, g
);
4234 for (int i
= 0; i
< 2; i
++)
4237 tree vf
= create_tmp_var (unsigned_type_node
);
4238 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
4239 gimple_call_set_lhs (g
, vf
);
4240 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
4241 gimple_seq_add_stmt (seq
, g
);
4242 tree t
= build_int_cst (unsigned_type_node
, 0);
4243 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
4244 gimple_seq_add_stmt (seq
, g
);
4245 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4246 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
4247 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4248 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
4249 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
4250 gimple_seq_add_seq (seq
, llist
[i
]);
4251 t
= build_int_cst (unsigned_type_node
, 1);
4252 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
4253 gimple_seq_add_stmt (seq
, g
);
4254 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
4255 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
4256 gimple_seq_add_stmt (seq
, g
);
4257 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
4261 /* The copyin sequence is not to be executed by the main thread, since
4262 that would result in self-copies. Perhaps not visible to scalars,
4263 but it certainly is to C++ operator=. */
4266 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
4268 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
4269 build_int_cst (TREE_TYPE (x
), 0));
4270 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
4271 gimplify_and_add (x
, ilist
);
4274 /* If any copyin variable is passed by reference, we must ensure the
4275 master thread doesn't modify it before it is copied over in all
4276 threads. Similarly for variables in both firstprivate and
4277 lastprivate clauses we need to ensure the lastprivate copying
4278 happens after firstprivate copying in all threads. And similarly
4279 for UDRs if initializer expression refers to omp_orig. */
4280 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
4282 /* Don't add any barrier for #pragma omp simd or
4283 #pragma omp distribute. */
4284 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
4285 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
4286 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
4289 /* If max_vf is non-zero, then we can use only a vectorization factor
4290 up to the max_vf we chose. So stick it into the safelen clause. */
4293 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4294 OMP_CLAUSE_SAFELEN
);
4296 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
4297 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4300 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
4301 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
4303 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4304 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4310 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4311 both parallel and workshare constructs. PREDICATE may be NULL if it's
4315 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
4318 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
4319 bool par_clauses
= false;
4320 tree simduid
= NULL
, lastlane
= NULL
;
4322 /* Early exit if there are no lastprivate or linear clauses. */
4323 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
4324 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
4325 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
4326 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
4328 if (clauses
== NULL
)
4330 /* If this was a workshare clause, see if it had been combined
4331 with its parallel. In that case, look for the clauses on the
4332 parallel statement itself. */
4333 if (is_parallel_ctx (ctx
))
4337 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4340 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4341 OMP_CLAUSE_LASTPRIVATE
);
4342 if (clauses
== NULL
)
4350 tree label_true
, arm1
, arm2
;
4352 label
= create_artificial_label (UNKNOWN_LOCATION
);
4353 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
4354 arm1
= TREE_OPERAND (predicate
, 0);
4355 arm2
= TREE_OPERAND (predicate
, 1);
4356 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4357 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4358 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
4360 gimple_seq_add_stmt (stmt_list
, stmt
);
4361 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
4364 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4365 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4367 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
4369 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
4372 for (c
= clauses
; c
;)
4375 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4377 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4378 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4379 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
4381 var
= OMP_CLAUSE_DECL (c
);
4382 new_var
= lookup_decl (var
, ctx
);
4384 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
4386 tree val
= DECL_VALUE_EXPR (new_var
);
4387 if (TREE_CODE (val
) == ARRAY_REF
4388 && VAR_P (TREE_OPERAND (val
, 0))
4389 && lookup_attribute ("omp simd array",
4390 DECL_ATTRIBUTES (TREE_OPERAND (val
,
4393 if (lastlane
== NULL
)
4395 lastlane
= create_tmp_var (unsigned_type_node
);
4397 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4399 TREE_OPERAND (val
, 1));
4400 gimple_call_set_lhs (g
, lastlane
);
4401 gimple_seq_add_stmt (stmt_list
, g
);
4403 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4404 TREE_OPERAND (val
, 0), lastlane
,
4405 NULL_TREE
, NULL_TREE
);
4409 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4410 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4412 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4413 gimple_seq_add_seq (stmt_list
,
4414 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4415 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4417 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4418 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4420 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4421 gimple_seq_add_seq (stmt_list
,
4422 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4423 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4426 x
= build_outer_var_ref (var
, ctx
);
4427 if (is_reference (var
))
4428 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4429 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4430 gimplify_and_add (x
, stmt_list
);
4432 c
= OMP_CLAUSE_CHAIN (c
);
4433 if (c
== NULL
&& !par_clauses
)
4435 /* If this was a workshare clause, see if it had been combined
4436 with its parallel. In that case, continue looking for the
4437 clauses also on the parallel statement itself. */
4438 if (is_parallel_ctx (ctx
))
4442 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4445 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4446 OMP_CLAUSE_LASTPRIVATE
);
4452 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4456 oacc_lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
,
4457 tree tid
, tree var
, tree new_var
)
4459 /* The atomic add at the end of the sum creates unnecessary
4460 write contention on accelerators. To work around this,
4461 create an array to store the partial reductions. Later, in
4462 lower_omp_for (for openacc), the values of array will be
4465 tree t
= NULL_TREE
, array
, x
;
4466 tree type
= get_base_type (var
);
4469 /* Now insert the partial reductions into the array. */
4471 /* Find the reduction array. */
4473 tree ptype
= build_pointer_type (type
);
4475 t
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
), ctx
);
4476 t
= build_receiver_ref (t
, false, ctx
->outer
);
4478 array
= create_tmp_var (ptype
);
4479 gimplify_assign (array
, t
, stmt_seqp
);
4481 tree ptr
= create_tmp_var (TREE_TYPE (array
));
4483 /* Find the reduction array. */
4485 /* testing a unary conversion. */
4486 tree offset
= create_tmp_var (sizetype
);
4487 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
4489 t
= create_tmp_var (sizetype
);
4490 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
4492 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, t
);
4493 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4495 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4496 of adding sizeof(var) to the array? */
4497 ptr
= create_tmp_var (ptype
);
4498 stmt
= gimple_build_assign (unshare_expr (ptr
), POINTER_PLUS_EXPR
, array
,
4500 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4502 /* Move the local sum to gfc$sum[i]. */
4503 x
= unshare_expr (build_simple_mem_ref (ptr
));
4504 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
4507 /* Generate code to implement the REDUCTION clauses. */
4510 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4512 gimple_seq sub_seq
= NULL
;
4514 tree x
, c
, tid
= NULL_TREE
;
4517 /* SIMD reductions are handled in lower_rec_input_clauses. */
4518 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4519 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4522 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4523 update in that case, otherwise use a lock. */
4524 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4525 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4527 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4529 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4539 /* Initialize thread info for OpenACC. */
4540 if (is_gimple_omp_oacc (ctx
->stmt
))
4542 /* Get the current thread id. */
4543 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
4544 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)));
4545 gimple stmt
= gimple_build_call (call
, 0);
4546 gimple_call_set_lhs (stmt
, tid
);
4547 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4550 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4552 tree var
, ref
, new_var
;
4553 enum tree_code code
;
4554 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4556 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4559 var
= OMP_CLAUSE_DECL (c
);
4560 new_var
= lookup_decl (var
, ctx
);
4561 if (is_reference (var
))
4562 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4563 ref
= build_outer_var_ref (var
, ctx
);
4564 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4566 /* reduction(-:var) sums up the partial results, so it acts
4567 identically to reduction(+:var). */
4568 if (code
== MINUS_EXPR
)
4571 if (is_gimple_omp_oacc (ctx
->stmt
))
4573 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
));
4575 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
4577 else if (count
== 1)
4579 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4581 addr
= save_expr (addr
);
4582 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4583 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4584 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4585 gimplify_and_add (x
, stmt_seqp
);
4588 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4590 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4592 if (is_reference (var
)
4593 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4595 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4596 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4597 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4598 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4599 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4600 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4601 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4605 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4606 ref
= build_outer_var_ref (var
, ctx
);
4607 gimplify_assign (ref
, x
, &sub_seq
);
4611 if (is_gimple_omp_oacc (ctx
->stmt
))
4614 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4616 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4618 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4620 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4622 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4626 /* Generate code to implement the COPYPRIVATE clauses. */
4629 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4634 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4636 tree var
, new_var
, ref
, x
;
4638 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4640 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4643 var
= OMP_CLAUSE_DECL (c
);
4644 by_ref
= use_pointer_for_field (var
, NULL
);
4646 ref
= build_sender_ref (var
, ctx
);
4647 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4650 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4651 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4653 gimplify_assign (ref
, x
, slist
);
4655 ref
= build_receiver_ref (var
, false, ctx
);
4658 ref
= fold_convert_loc (clause_loc
,
4659 build_pointer_type (TREE_TYPE (new_var
)),
4661 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4663 if (is_reference (var
))
4665 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4666 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4667 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4669 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4670 gimplify_and_add (x
, rlist
);
4675 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4676 and REDUCTION from the sender (aka parent) side. */
4679 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4684 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4686 tree val
, ref
, x
, var
;
4687 bool by_ref
, do_in
= false, do_out
= false;
4688 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4690 switch (OMP_CLAUSE_CODE (c
))
4692 case OMP_CLAUSE_PRIVATE
:
4693 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4696 case OMP_CLAUSE_FIRSTPRIVATE
:
4697 case OMP_CLAUSE_COPYIN
:
4698 case OMP_CLAUSE_LASTPRIVATE
:
4699 case OMP_CLAUSE_REDUCTION
:
4700 case OMP_CLAUSE__LOOPTEMP_
:
4706 val
= OMP_CLAUSE_DECL (c
);
4707 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4709 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4710 && is_global_var (var
))
4712 if (is_variable_sized (val
))
4714 by_ref
= use_pointer_for_field (val
, NULL
);
4716 switch (OMP_CLAUSE_CODE (c
))
4718 case OMP_CLAUSE_PRIVATE
:
4719 case OMP_CLAUSE_FIRSTPRIVATE
:
4720 case OMP_CLAUSE_COPYIN
:
4721 case OMP_CLAUSE__LOOPTEMP_
:
4725 case OMP_CLAUSE_LASTPRIVATE
:
4726 if (by_ref
|| is_reference (val
))
4728 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4735 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4740 case OMP_CLAUSE_REDUCTION
:
4742 do_out
= !(by_ref
|| is_reference (val
));
4751 ref
= build_sender_ref (val
, ctx
);
4752 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4753 gimplify_assign (ref
, x
, ilist
);
4754 if (is_task_ctx (ctx
))
4755 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4760 ref
= build_sender_ref (val
, ctx
);
4761 gimplify_assign (var
, ref
, olist
);
4766 /* Generate code to implement SHARED from the sender (aka parent)
4767 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4768 list things that got automatically shared. */
4771 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4773 tree var
, ovar
, nvar
, f
, x
, record_type
;
4775 if (ctx
->record_type
== NULL
)
4778 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4779 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4781 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4782 nvar
= maybe_lookup_decl (ovar
, ctx
);
4783 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4786 /* If CTX is a nested parallel directive. Find the immediately
4787 enclosing parallel or workshare construct that contains a
4788 mapping for OVAR. */
4789 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4791 if (use_pointer_for_field (ovar
, ctx
))
4793 x
= build_sender_ref (ovar
, ctx
);
4794 var
= build_fold_addr_expr (var
);
4795 gimplify_assign (x
, var
, ilist
);
4799 x
= build_sender_ref (ovar
, ctx
);
4800 gimplify_assign (x
, var
, ilist
);
4802 if (!TREE_READONLY (var
)
4803 /* We don't need to receive a new reference to a result
4804 or parm decl. In fact we may not store to it as we will
4805 invalidate any pending RSO and generate wrong gimple
4807 && !((TREE_CODE (var
) == RESULT_DECL
4808 || TREE_CODE (var
) == PARM_DECL
)
4809 && DECL_BY_REFERENCE (var
)))
4811 x
= build_sender_ref (ovar
, ctx
);
4812 gimplify_assign (var
, x
, olist
);
4819 /* A convenience function to build an empty GIMPLE_COND with just the
4823 gimple_build_cond_empty (tree cond
)
4825 enum tree_code pred_code
;
4828 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4829 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4833 /* Build the function calls to GOMP_parallel_start etc to actually
4834 generate the parallel operation. REGION is the parallel region
4835 being expanded. BB is the block where to insert the code. WS_ARGS
4836 will be set if this is a call to a combined parallel+workshare
4837 construct, it contains the list of additional arguments needed by
4838 the workshare construct. */
4841 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4842 gomp_parallel
*entry_stmt
,
4843 vec
<tree
, va_gc
> *ws_args
)
4845 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4846 gimple_stmt_iterator gsi
;
4848 enum built_in_function start_ix
;
4850 location_t clause_loc
;
4851 vec
<tree
, va_gc
> *args
;
4853 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4855 /* Determine what flavor of GOMP_parallel we will be
4857 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4858 if (is_combined_parallel (region
))
4860 switch (region
->inner
->type
)
4862 case GIMPLE_OMP_FOR
:
4863 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4864 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4865 + (region
->inner
->sched_kind
4866 == OMP_CLAUSE_SCHEDULE_RUNTIME
4867 ? 3 : region
->inner
->sched_kind
));
4868 start_ix
= (enum built_in_function
)start_ix2
;
4870 case GIMPLE_OMP_SECTIONS
:
4871 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4878 /* By default, the value of NUM_THREADS is zero (selected at run time)
4879 and there is no conditional. */
4881 val
= build_int_cst (unsigned_type_node
, 0);
4882 flags
= build_int_cst (unsigned_type_node
, 0);
4884 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4886 cond
= OMP_CLAUSE_IF_EXPR (c
);
4888 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4891 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4892 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4895 clause_loc
= gimple_location (entry_stmt
);
4897 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4899 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4901 /* Ensure 'val' is of the correct type. */
4902 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4904 /* If we found the clause 'if (cond)', build either
4905 (cond != 0) or (cond ? val : 1u). */
4908 cond
= gimple_boolify (cond
);
4910 if (integer_zerop (val
))
4911 val
= fold_build2_loc (clause_loc
,
4912 EQ_EXPR
, unsigned_type_node
, cond
,
4913 build_int_cst (TREE_TYPE (cond
), 0));
4916 basic_block cond_bb
, then_bb
, else_bb
;
4917 edge e
, e_then
, e_else
;
4918 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4920 tmp_var
= create_tmp_var (TREE_TYPE (val
));
4921 if (gimple_in_ssa_p (cfun
))
4923 tmp_then
= make_ssa_name (tmp_var
);
4924 tmp_else
= make_ssa_name (tmp_var
);
4925 tmp_join
= make_ssa_name (tmp_var
);
4934 e
= split_block (bb
, NULL
);
4939 then_bb
= create_empty_bb (cond_bb
);
4940 else_bb
= create_empty_bb (then_bb
);
4941 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4942 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4944 stmt
= gimple_build_cond_empty (cond
);
4945 gsi
= gsi_start_bb (cond_bb
);
4946 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4948 gsi
= gsi_start_bb (then_bb
);
4949 stmt
= gimple_build_assign (tmp_then
, val
);
4950 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4952 gsi
= gsi_start_bb (else_bb
);
4953 stmt
= gimple_build_assign
4954 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4955 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4957 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4958 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4959 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4960 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4961 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4962 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4964 if (gimple_in_ssa_p (cfun
))
4966 gphi
*phi
= create_phi_node (tmp_join
, bb
);
4967 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4968 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4974 gsi
= gsi_start_bb (bb
);
4975 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4976 false, GSI_CONTINUE_LINKING
);
4979 gsi
= gsi_last_bb (bb
);
4980 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4982 t1
= null_pointer_node
;
4984 t1
= build_fold_addr_expr (t
);
4985 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4987 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4988 args
->quick_push (t2
);
4989 args
->quick_push (t1
);
4990 args
->quick_push (val
);
4992 args
->splice (*ws_args
);
4993 args
->quick_push (flags
);
4995 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4996 builtin_decl_explicit (start_ix
), args
);
4998 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4999 false, GSI_CONTINUE_LINKING
);
5002 /* Insert a function call whose name is FUNC_NAME with the information from
5003 ENTRY_STMT into the basic_block BB. */
5006 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
5007 vec
<tree
, va_gc
> *ws_args
)
5010 gimple_stmt_iterator gsi
;
5011 vec
<tree
, va_gc
> *args
;
5013 gcc_assert (vec_safe_length (ws_args
) == 2);
5014 tree func_name
= (*ws_args
)[0];
5015 tree grain
= (*ws_args
)[1];
5017 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
5018 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
5019 gcc_assert (count
!= NULL_TREE
);
5020 count
= OMP_CLAUSE_OPERAND (count
, 0);
5022 gsi
= gsi_last_bb (bb
);
5023 t
= gimple_omp_parallel_data_arg (entry_stmt
);
5025 t1
= null_pointer_node
;
5027 t1
= build_fold_addr_expr (t
);
5028 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5030 vec_alloc (args
, 4);
5031 args
->quick_push (t2
);
5032 args
->quick_push (t1
);
5033 args
->quick_push (count
);
5034 args
->quick_push (grain
);
5035 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
5037 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
5038 GSI_CONTINUE_LINKING
);
5041 /* Build the function call to GOMP_task to actually
5042 generate the task operation. BB is the block where to insert the code. */
5045 expand_task_call (basic_block bb
, gomp_task
*entry_stmt
)
5047 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
5048 gimple_stmt_iterator gsi
;
5049 location_t loc
= gimple_location (entry_stmt
);
5051 clauses
= gimple_omp_task_clauses (entry_stmt
);
5053 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
5055 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
5057 cond
= boolean_true_node
;
5059 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
5060 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
5061 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
5062 flags
= build_int_cst (unsigned_type_node
,
5063 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
5065 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
5068 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
5069 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
5070 build_int_cst (unsigned_type_node
, 2),
5071 build_int_cst (unsigned_type_node
, 0));
5072 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
5075 depend
= OMP_CLAUSE_DECL (depend
);
5077 depend
= build_int_cst (ptr_type_node
, 0);
5079 gsi
= gsi_last_bb (bb
);
5080 t
= gimple_omp_task_data_arg (entry_stmt
);
5082 t2
= null_pointer_node
;
5084 t2
= build_fold_addr_expr_loc (loc
, t
);
5085 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
5086 t
= gimple_omp_task_copy_fn (entry_stmt
);
5088 t3
= null_pointer_node
;
5090 t3
= build_fold_addr_expr_loc (loc
, t
);
5092 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
5094 gimple_omp_task_arg_size (entry_stmt
),
5095 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
5098 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5099 false, GSI_CONTINUE_LINKING
);
5103 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5104 catch handler and return it. This prevents programs from violating the
5105 structured block semantics with throws. */
5108 maybe_catch_exception (gimple_seq body
)
5113 if (!flag_exceptions
)
5116 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
5117 decl
= lang_hooks
.eh_protect_cleanup_actions ();
5119 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
5121 g
= gimple_build_eh_must_not_throw (decl
);
5122 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
5125 return gimple_seq_alloc_with_stmt (g
);
5128 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5131 vec2chain (vec
<tree
, va_gc
> *v
)
5133 tree chain
= NULL_TREE
, t
;
5136 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
5138 DECL_CHAIN (t
) = chain
;
5146 /* Remove barriers in REGION->EXIT's block. Note that this is only
5147 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5148 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5149 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5153 remove_exit_barrier (struct omp_region
*region
)
5155 gimple_stmt_iterator gsi
;
5156 basic_block exit_bb
;
5160 int any_addressable_vars
= -1;
5162 exit_bb
= region
->exit
;
5164 /* If the parallel region doesn't return, we don't have REGION->EXIT
5169 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5170 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5171 statements that can appear in between are extremely limited -- no
5172 memory operations at all. Here, we allow nothing at all, so the
5173 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5174 gsi
= gsi_last_bb (exit_bb
);
5175 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5177 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
5180 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
5182 gsi
= gsi_last_bb (e
->src
);
5183 if (gsi_end_p (gsi
))
5185 stmt
= gsi_stmt (gsi
);
5186 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
5187 && !gimple_omp_return_nowait_p (stmt
))
5189 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5190 in many cases. If there could be tasks queued, the barrier
5191 might be needed to let the tasks run before some local
5192 variable of the parallel that the task uses as shared
5193 runs out of scope. The task can be spawned either
5194 from within current function (this would be easy to check)
5195 or from some function it calls and gets passed an address
5196 of such a variable. */
5197 if (any_addressable_vars
< 0)
5199 gomp_parallel
*parallel_stmt
5200 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
5201 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
5202 tree local_decls
, block
, decl
;
5205 any_addressable_vars
= 0;
5206 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
5207 if (TREE_ADDRESSABLE (decl
))
5209 any_addressable_vars
= 1;
5212 for (block
= gimple_block (stmt
);
5213 !any_addressable_vars
5215 && TREE_CODE (block
) == BLOCK
;
5216 block
= BLOCK_SUPERCONTEXT (block
))
5218 for (local_decls
= BLOCK_VARS (block
);
5220 local_decls
= DECL_CHAIN (local_decls
))
5221 if (TREE_ADDRESSABLE (local_decls
))
5223 any_addressable_vars
= 1;
5226 if (block
== gimple_block (parallel_stmt
))
5230 if (!any_addressable_vars
)
5231 gimple_omp_return_set_nowait (stmt
);
5237 remove_exit_barriers (struct omp_region
*region
)
5239 if (region
->type
== GIMPLE_OMP_PARALLEL
)
5240 remove_exit_barrier (region
);
5244 region
= region
->inner
;
5245 remove_exit_barriers (region
);
5246 while (region
->next
)
5248 region
= region
->next
;
5249 remove_exit_barriers (region
);
5254 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5255 calls. These can't be declared as const functions, but
5256 within one parallel body they are constant, so they can be
5257 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5258 which are declared const. Similarly for task body, except
5259 that in untied task omp_get_thread_num () can change at any task
5260 scheduling point. */
5263 optimize_omp_library_calls (gimple entry_stmt
)
5266 gimple_stmt_iterator gsi
;
5267 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5268 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
5269 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5270 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
5271 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
5272 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
5273 OMP_CLAUSE_UNTIED
) != NULL
);
5275 FOR_EACH_BB_FN (bb
, cfun
)
5276 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5278 gimple call
= gsi_stmt (gsi
);
5281 if (is_gimple_call (call
)
5282 && (decl
= gimple_call_fndecl (call
))
5283 && DECL_EXTERNAL (decl
)
5284 && TREE_PUBLIC (decl
)
5285 && DECL_INITIAL (decl
) == NULL
)
5289 if (DECL_NAME (decl
) == thr_num_id
)
5291 /* In #pragma omp task untied omp_get_thread_num () can change
5292 during the execution of the task region. */
5295 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5297 else if (DECL_NAME (decl
) == num_thr_id
)
5298 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5302 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
5303 || gimple_call_num_args (call
) != 0)
5306 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
5309 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
5310 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
5311 TREE_TYPE (TREE_TYPE (built_in
))))
5314 gimple_call_set_fndecl (call
, built_in
);
5319 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5323 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
5327 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5328 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
5331 if (TREE_CODE (t
) == ADDR_EXPR
)
5332 recompute_tree_invariant_for_addr_expr (t
);
5334 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
5338 /* Prepend TO = FROM assignment before *GSI_P. */
5341 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
5343 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
5344 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
5345 true, GSI_SAME_STMT
);
5346 gimple stmt
= gimple_build_assign (to
, from
);
5347 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
5348 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
5349 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
5351 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
5352 gimple_regimplify_operands (stmt
, &gsi
);
5356 /* Expand the OpenMP parallel or task directive starting at REGION. */
5359 expand_omp_taskreg (struct omp_region
*region
)
5361 basic_block entry_bb
, exit_bb
, new_bb
;
5362 struct function
*child_cfun
;
5363 tree child_fn
, block
, t
;
5364 gimple_stmt_iterator gsi
;
5365 gimple entry_stmt
, stmt
;
5367 vec
<tree
, va_gc
> *ws_args
;
5369 entry_stmt
= last_stmt (region
->entry
);
5370 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
5371 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
5373 entry_bb
= region
->entry
;
5374 exit_bb
= region
->exit
;
5378 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
5379 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
5380 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
5383 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5384 and the inner statement contains the name of the built-in function
5386 ws_args
= region
->inner
->ws_args
;
5387 else if (is_combined_parallel (region
))
5388 ws_args
= region
->ws_args
;
5392 if (child_cfun
->cfg
)
5394 /* Due to inlining, it may happen that we have already outlined
5395 the region, in which case all we need to do is make the
5396 sub-graph unreachable and emit the parallel call. */
5397 edge entry_succ_e
, exit_succ_e
;
5399 entry_succ_e
= single_succ_edge (entry_bb
);
5401 gsi
= gsi_last_bb (entry_bb
);
5402 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
5403 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
5404 gsi_remove (&gsi
, true);
5409 exit_succ_e
= single_succ_edge (exit_bb
);
5410 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
5412 remove_edge_and_dominated_blocks (entry_succ_e
);
5416 unsigned srcidx
, dstidx
, num
;
5418 /* If the parallel region needs data sent from the parent
5419 function, then the very first statement (except possible
5420 tree profile counter updates) of the parallel body
5421 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5422 &.OMP_DATA_O is passed as an argument to the child function,
5423 we need to replace it with the argument as seen by the child
5426 In most cases, this will end up being the identity assignment
5427 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5428 a function call that has been inlined, the original PARM_DECL
5429 .OMP_DATA_I may have been converted into a different local
5430 variable. In which case, we need to keep the assignment. */
5431 if (gimple_omp_taskreg_data_arg (entry_stmt
))
5433 basic_block entry_succ_bb
= single_succ (entry_bb
);
5435 gimple parcopy_stmt
= NULL
;
5437 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
5441 gcc_assert (!gsi_end_p (gsi
));
5442 stmt
= gsi_stmt (gsi
);
5443 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
5446 if (gimple_num_ops (stmt
) == 2)
5448 tree arg
= gimple_assign_rhs1 (stmt
);
5450 /* We're ignore the subcode because we're
5451 effectively doing a STRIP_NOPS. */
5453 if (TREE_CODE (arg
) == ADDR_EXPR
5454 && TREE_OPERAND (arg
, 0)
5455 == gimple_omp_taskreg_data_arg (entry_stmt
))
5457 parcopy_stmt
= stmt
;
5463 gcc_assert (parcopy_stmt
!= NULL
);
5464 arg
= DECL_ARGUMENTS (child_fn
);
5466 if (!gimple_in_ssa_p (cfun
))
5468 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5469 gsi_remove (&gsi
, true);
5472 /* ?? Is setting the subcode really necessary ?? */
5473 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5474 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5479 /* If we are in ssa form, we must load the value from the default
5480 definition of the argument. That should not be defined now,
5481 since the argument is not used uninitialized. */
5482 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5483 narg
= make_ssa_name (arg
, gimple_build_nop ());
5484 set_ssa_default_def (cfun
, arg
, narg
);
5485 /* ?? Is setting the subcode really necessary ?? */
5486 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5487 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5488 update_stmt (parcopy_stmt
);
5492 /* Declare local variables needed in CHILD_CFUN. */
5493 block
= DECL_INITIAL (child_fn
);
5494 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5495 /* The gimplifier could record temporaries in parallel/task block
5496 rather than in containing function's local_decls chain,
5497 which would mean cgraph missed finalizing them. Do it now. */
5498 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5499 if (TREE_CODE (t
) == VAR_DECL
5501 && !DECL_EXTERNAL (t
))
5502 varpool_node::finalize_decl (t
);
5503 DECL_SAVED_TREE (child_fn
) = NULL
;
5504 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5505 gimple_set_body (child_fn
, NULL
);
5506 TREE_USED (block
) = 1;
5508 /* Reset DECL_CONTEXT on function arguments. */
5509 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5510 DECL_CONTEXT (t
) = child_fn
;
5512 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5513 so that it can be moved to the child function. */
5514 gsi
= gsi_last_bb (entry_bb
);
5515 stmt
= gsi_stmt (gsi
);
5516 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5517 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5518 gsi_remove (&gsi
, true);
5519 e
= split_block (entry_bb
, stmt
);
5521 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5523 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5526 gsi
= gsi_last_bb (exit_bb
);
5527 gcc_assert (!gsi_end_p (gsi
)
5528 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5529 stmt
= gimple_build_return (NULL
);
5530 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5531 gsi_remove (&gsi
, true);
5534 /* Move the parallel region into CHILD_CFUN. */
5536 if (gimple_in_ssa_p (cfun
))
5538 init_tree_ssa (child_cfun
);
5539 init_ssa_operands (child_cfun
);
5540 child_cfun
->gimple_df
->in_ssa_p
= true;
5544 block
= gimple_block (entry_stmt
);
5546 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5548 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5549 /* When the OMP expansion process cannot guarantee an up-to-date
5550 loop tree arrange for the child function to fixup loops. */
5551 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5552 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5554 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5555 num
= vec_safe_length (child_cfun
->local_decls
);
5556 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5558 t
= (*child_cfun
->local_decls
)[srcidx
];
5559 if (DECL_CONTEXT (t
) == cfun
->decl
)
5561 if (srcidx
!= dstidx
)
5562 (*child_cfun
->local_decls
)[dstidx
] = t
;
5566 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5568 /* Inform the callgraph about the new function. */
5569 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5570 cgraph_node::add_new_function (child_fn
, true);
5572 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5573 fixed in a following pass. */
5574 push_cfun (child_cfun
);
5576 optimize_omp_library_calls (entry_stmt
);
5577 cgraph_edge::rebuild_edges ();
5579 /* Some EH regions might become dead, see PR34608. If
5580 pass_cleanup_cfg isn't the first pass to happen with the
5581 new child, these dead EH edges might cause problems.
5582 Clean them up now. */
5583 if (flag_exceptions
)
5586 bool changed
= false;
5588 FOR_EACH_BB_FN (bb
, cfun
)
5589 changed
|= gimple_purge_dead_eh_edges (bb
);
5591 cleanup_tree_cfg ();
5593 if (gimple_in_ssa_p (cfun
))
5594 update_ssa (TODO_update_ssa
);
5598 /* Emit a library call to launch the children threads. */
5600 expand_cilk_for_call (new_bb
,
5601 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5602 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5603 expand_parallel_call (region
, new_bb
,
5604 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5606 expand_task_call (new_bb
, as_a
<gomp_task
*> (entry_stmt
));
5607 if (gimple_in_ssa_p (cfun
))
5608 update_ssa (TODO_update_ssa_only_virtuals
);
5612 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5613 of the combined collapse > 1 loop constructs, generate code like:
5614 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5619 count3 = (adj + N32 - N31) / STEP3;
5620 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5625 count2 = (adj + N22 - N21) / STEP2;
5626 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5631 count1 = (adj + N12 - N11) / STEP1;
5632 count = count1 * count2 * count3;
5633 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5635 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5636 of the combined loop constructs, just initialize COUNTS array
5637 from the _looptemp_ clauses. */
5639 /* NOTE: It *could* be better to moosh all of the BBs together,
5640 creating one larger BB with all the computation and the unexpected
5641 jump at the end. I.e.
5643 bool zero3, zero2, zero1, zero;
5646 count3 = (N32 - N31) /[cl] STEP3;
5648 count2 = (N22 - N21) /[cl] STEP2;
5650 count1 = (N12 - N11) /[cl] STEP1;
5651 zero = zero3 || zero2 || zero1;
5652 count = count1 * count2 * count3;
5653 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5655 After all, we expect the zero=false, and thus we expect to have to
5656 evaluate all of the comparison expressions, so short-circuiting
5657 oughtn't be a win. Since the condition isn't protecting a
5658 denominator, we're not concerned about divide-by-zero, so we can
5659 fully evaluate count even if a numerator turned out to be wrong.
5661 It seems like putting this all together would create much better
5662 scheduling opportunities, and less pressure on the chip's branch
5666 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5667 basic_block
&entry_bb
, tree
*counts
,
5668 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5669 basic_block
&l2_dom_bb
)
5671 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5675 /* Collapsed loops need work for expansion into SSA form. */
5676 gcc_assert (!gimple_in_ssa_p (cfun
));
5678 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5679 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5681 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5682 isn't supposed to be handled, as the inner loop doesn't
5684 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5685 OMP_CLAUSE__LOOPTEMP_
);
5686 gcc_assert (innerc
);
5687 for (i
= 0; i
< fd
->collapse
; i
++)
5689 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5690 OMP_CLAUSE__LOOPTEMP_
);
5691 gcc_assert (innerc
);
5693 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5695 counts
[0] = NULL_TREE
;
5700 for (i
= 0; i
< fd
->collapse
; i
++)
5702 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5704 if (SSA_VAR_P (fd
->loop
.n2
)
5705 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5706 fold_convert (itype
, fd
->loops
[i
].n1
),
5707 fold_convert (itype
, fd
->loops
[i
].n2
)))
5708 == NULL_TREE
|| !integer_onep (t
)))
5712 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5713 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5714 true, GSI_SAME_STMT
);
5715 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5716 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5717 true, GSI_SAME_STMT
);
5718 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5719 NULL_TREE
, NULL_TREE
);
5720 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
5721 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
5722 expand_omp_regimplify_p
, NULL
, NULL
)
5723 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
5724 expand_omp_regimplify_p
, NULL
, NULL
))
5726 *gsi
= gsi_for_stmt (cond_stmt
);
5727 gimple_regimplify_operands (cond_stmt
, gsi
);
5729 e
= split_block (entry_bb
, cond_stmt
);
5730 if (zero_iter_bb
== NULL
)
5732 gassign
*assign_stmt
;
5733 first_zero_iter
= i
;
5734 zero_iter_bb
= create_empty_bb (entry_bb
);
5735 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5736 *gsi
= gsi_after_labels (zero_iter_bb
);
5737 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
5738 build_zero_cst (type
));
5739 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
5740 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5743 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5744 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5745 e
->flags
= EDGE_TRUE_VALUE
;
5746 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5747 if (l2_dom_bb
== NULL
)
5748 l2_dom_bb
= entry_bb
;
5750 *gsi
= gsi_last_bb (entry_bb
);
5753 if (POINTER_TYPE_P (itype
))
5754 itype
= signed_type_for (itype
);
5755 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5757 t
= fold_build2 (PLUS_EXPR
, itype
,
5758 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5759 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5760 fold_convert (itype
, fd
->loops
[i
].n2
));
5761 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5762 fold_convert (itype
, fd
->loops
[i
].n1
));
5763 /* ?? We could probably use CEIL_DIV_EXPR instead of
5764 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5765 generate the same code in the end because generically we
5766 don't know that the values involved must be negative for
5768 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5769 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5770 fold_build1 (NEGATE_EXPR
, itype
, t
),
5771 fold_build1 (NEGATE_EXPR
, itype
,
5772 fold_convert (itype
,
5773 fd
->loops
[i
].step
)));
5775 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5776 fold_convert (itype
, fd
->loops
[i
].step
));
5777 t
= fold_convert (type
, t
);
5778 if (TREE_CODE (t
) == INTEGER_CST
)
5782 counts
[i
] = create_tmp_reg (type
, ".count");
5783 expand_omp_build_assign (gsi
, counts
[i
], t
);
5785 if (SSA_VAR_P (fd
->loop
.n2
))
5790 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5791 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5797 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5799 V3 = N31 + (T % count3) * STEP3;
5801 V2 = N21 + (T % count2) * STEP2;
5803 V1 = N11 + T * STEP1;
5804 if this loop doesn't have an inner loop construct combined with it.
5805 If it does have an inner loop construct combined with it and the
5806 iteration count isn't known constant, store values from counts array
5807 into its _looptemp_ temporaries instead. */
5810 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5811 tree
*counts
, gimple inner_stmt
, tree startvar
)
5814 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5816 /* If fd->loop.n2 is constant, then no propagation of the counts
5817 is needed, they are constant. */
5818 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5821 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5822 ? gimple_omp_parallel_clauses (inner_stmt
)
5823 : gimple_omp_for_clauses (inner_stmt
);
5824 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5825 isn't supposed to be handled, as the inner loop doesn't
5827 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5828 gcc_assert (innerc
);
5829 for (i
= 0; i
< fd
->collapse
; i
++)
5831 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5832 OMP_CLAUSE__LOOPTEMP_
);
5833 gcc_assert (innerc
);
5836 tree tem
= OMP_CLAUSE_DECL (innerc
);
5837 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5838 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5839 false, GSI_CONTINUE_LINKING
);
5840 gassign
*stmt
= gimple_build_assign (tem
, t
);
5841 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5847 tree type
= TREE_TYPE (fd
->loop
.v
);
5848 tree tem
= create_tmp_reg (type
, ".tem");
5849 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
5850 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5852 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5854 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5856 if (POINTER_TYPE_P (vtype
))
5857 itype
= signed_type_for (vtype
);
5859 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5862 t
= fold_convert (itype
, t
);
5863 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5864 fold_convert (itype
, fd
->loops
[i
].step
));
5865 if (POINTER_TYPE_P (vtype
))
5866 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5868 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5869 t
= force_gimple_operand_gsi (gsi
, t
,
5870 DECL_P (fd
->loops
[i
].v
)
5871 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5873 GSI_CONTINUE_LINKING
);
5874 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5875 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5878 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5879 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5880 false, GSI_CONTINUE_LINKING
);
5881 stmt
= gimple_build_assign (tem
, t
);
5882 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5888 /* Helper function for expand_omp_for_*. Generate code like:
5891 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5895 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5902 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5903 basic_block body_bb
)
5905 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5907 gimple_stmt_iterator gsi
;
5913 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5915 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5917 bb
= create_empty_bb (last_bb
);
5918 add_bb_to_loop (bb
, last_bb
->loop_father
);
5919 gsi
= gsi_start_bb (bb
);
5921 if (i
< fd
->collapse
- 1)
5923 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5924 e
->probability
= REG_BR_PROB_BASE
/ 8;
5926 t
= fd
->loops
[i
+ 1].n1
;
5927 t
= force_gimple_operand_gsi (&gsi
, t
,
5928 DECL_P (fd
->loops
[i
+ 1].v
)
5929 && TREE_ADDRESSABLE (fd
->loops
[i
5932 GSI_CONTINUE_LINKING
);
5933 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5934 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5939 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5941 if (POINTER_TYPE_P (vtype
))
5942 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5944 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5945 t
= force_gimple_operand_gsi (&gsi
, t
,
5946 DECL_P (fd
->loops
[i
].v
)
5947 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5948 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5949 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5950 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5954 t
= fd
->loops
[i
].n2
;
5955 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5956 false, GSI_CONTINUE_LINKING
);
5957 tree v
= fd
->loops
[i
].v
;
5958 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5959 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5960 false, GSI_CONTINUE_LINKING
);
5961 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5962 stmt
= gimple_build_cond_empty (t
);
5963 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5964 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5965 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5968 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5976 /* A subroutine of expand_omp_for. Generate code for a parallel
5977 loop with any schedule. Given parameters:
5979 for (V = N1; V cond N2; V += STEP) BODY;
5981 where COND is "<" or ">", we generate pseudocode
5983 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5984 if (more) goto L0; else goto L3;
5991 if (V cond iend) goto L1; else goto L2;
5993 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5996 If this is a combined omp parallel loop, instead of the call to
5997 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5998 If this is gimple_omp_for_combined_p loop, then instead of assigning
5999 V and iend in L0 we assign the first two _looptemp_ clause decls of the
6000 inner GIMPLE_OMP_FOR and V += STEP; and
6001 if (V cond iend) goto L1; else goto L2; are removed.
6003 For collapsed loops, given parameters:
6005 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6006 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6007 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6010 we generate pseudocode
6012 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6017 count3 = (adj + N32 - N31) / STEP3;
6018 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6023 count2 = (adj + N22 - N21) / STEP2;
6024 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6029 count1 = (adj + N12 - N11) / STEP1;
6030 count = count1 * count2 * count3;
6035 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6036 if (more) goto L0; else goto L3;
6040 V3 = N31 + (T % count3) * STEP3;
6042 V2 = N21 + (T % count2) * STEP2;
6044 V1 = N11 + T * STEP1;
6049 if (V < iend) goto L10; else goto L2;
6052 if (V3 cond3 N32) goto L1; else goto L11;
6056 if (V2 cond2 N22) goto L1; else goto L12;
6062 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6068 expand_omp_for_generic (struct omp_region
*region
,
6069 struct omp_for_data
*fd
,
6070 enum built_in_function start_fn
,
6071 enum built_in_function next_fn
,
6074 tree type
, istart0
, iend0
, iend
;
6075 tree t
, vmain
, vback
, bias
= NULL_TREE
;
6076 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
6077 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
6078 gimple_stmt_iterator gsi
;
6079 gassign
*assign_stmt
;
6080 bool in_combined_parallel
= is_combined_parallel (region
);
6081 bool broken_loop
= region
->cont
== NULL
;
6083 tree
*counts
= NULL
;
6086 gcc_assert (!broken_loop
|| !in_combined_parallel
);
6087 gcc_assert (fd
->iter_type
== long_integer_type_node
6088 || !in_combined_parallel
);
6090 type
= TREE_TYPE (fd
->loop
.v
);
6091 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
6092 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
6093 TREE_ADDRESSABLE (istart0
) = 1;
6094 TREE_ADDRESSABLE (iend0
) = 1;
6096 /* See if we need to bias by LLONG_MIN. */
6097 if (fd
->iter_type
== long_long_unsigned_type_node
6098 && TREE_CODE (type
) == INTEGER_TYPE
6099 && !TYPE_UNSIGNED (type
))
6103 if (fd
->loop
.cond_code
== LT_EXPR
)
6106 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6110 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6113 if (TREE_CODE (n1
) != INTEGER_CST
6114 || TREE_CODE (n2
) != INTEGER_CST
6115 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
6116 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
6119 entry_bb
= region
->entry
;
6120 cont_bb
= region
->cont
;
6122 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6123 gcc_assert (broken_loop
6124 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6125 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6126 l1_bb
= single_succ (l0_bb
);
6129 l2_bb
= create_empty_bb (cont_bb
);
6130 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
6131 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6135 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6136 exit_bb
= region
->exit
;
6138 gsi
= gsi_last_bb (entry_bb
);
6140 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6141 if (fd
->collapse
> 1)
6143 int first_zero_iter
= -1;
6144 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
6146 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6147 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6148 zero_iter_bb
, first_zero_iter
,
6153 /* Some counts[i] vars might be uninitialized if
6154 some loop has zero iterations. But the body shouldn't
6155 be executed in that case, so just avoid uninit warnings. */
6156 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
6157 if (SSA_VAR_P (counts
[i
]))
6158 TREE_NO_WARNING (counts
[i
]) = 1;
6160 e
= split_block (entry_bb
, gsi_stmt (gsi
));
6162 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
6163 gsi
= gsi_last_bb (entry_bb
);
6164 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
6165 get_immediate_dominator (CDI_DOMINATORS
,
6169 if (in_combined_parallel
)
6171 /* In a combined parallel loop, emit a call to
6172 GOMP_loop_foo_next. */
6173 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6174 build_fold_addr_expr (istart0
),
6175 build_fold_addr_expr (iend0
));
6179 tree t0
, t1
, t2
, t3
, t4
;
6180 /* If this is not a combined parallel loop, emit a call to
6181 GOMP_loop_foo_start in ENTRY_BB. */
6182 t4
= build_fold_addr_expr (iend0
);
6183 t3
= build_fold_addr_expr (istart0
);
6184 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
6187 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6189 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6190 OMP_CLAUSE__LOOPTEMP_
);
6191 gcc_assert (innerc
);
6192 t0
= OMP_CLAUSE_DECL (innerc
);
6193 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6194 OMP_CLAUSE__LOOPTEMP_
);
6195 gcc_assert (innerc
);
6196 t1
= OMP_CLAUSE_DECL (innerc
);
6198 if (POINTER_TYPE_P (TREE_TYPE (t0
))
6199 && TYPE_PRECISION (TREE_TYPE (t0
))
6200 != TYPE_PRECISION (fd
->iter_type
))
6202 /* Avoid casting pointers to integer of a different size. */
6203 tree itype
= signed_type_for (type
);
6204 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
6205 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
6209 t1
= fold_convert (fd
->iter_type
, t1
);
6210 t0
= fold_convert (fd
->iter_type
, t0
);
6214 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
6215 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
6217 if (fd
->iter_type
== long_integer_type_node
)
6221 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6222 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6223 6, t0
, t1
, t2
, t
, t3
, t4
);
6226 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6227 5, t0
, t1
, t2
, t3
, t4
);
6235 /* The GOMP_loop_ull_*start functions have additional boolean
6236 argument, true for < loops and false for > loops.
6237 In Fortran, the C bool type can be different from
6238 boolean_type_node. */
6239 bfn_decl
= builtin_decl_explicit (start_fn
);
6240 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
6241 t5
= build_int_cst (c_bool_type
,
6242 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
6245 tree bfn_decl
= builtin_decl_explicit (start_fn
);
6246 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6247 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
6250 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6251 6, t5
, t0
, t1
, t2
, t3
, t4
);
6254 if (TREE_TYPE (t
) != boolean_type_node
)
6255 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6256 t
, build_int_cst (TREE_TYPE (t
), 0));
6257 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6258 true, GSI_SAME_STMT
);
6259 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6261 /* Remove the GIMPLE_OMP_FOR statement. */
6262 gsi_remove (&gsi
, true);
6264 /* Iteration setup for sequential loop goes in L0_BB. */
6265 tree startvar
= fd
->loop
.v
;
6266 tree endvar
= NULL_TREE
;
6268 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6270 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
6271 && gimple_omp_for_kind (inner_stmt
)
6272 == GF_OMP_FOR_KIND_SIMD
);
6273 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
6274 OMP_CLAUSE__LOOPTEMP_
);
6275 gcc_assert (innerc
);
6276 startvar
= OMP_CLAUSE_DECL (innerc
);
6277 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6278 OMP_CLAUSE__LOOPTEMP_
);
6279 gcc_assert (innerc
);
6280 endvar
= OMP_CLAUSE_DECL (innerc
);
6283 gsi
= gsi_start_bb (l0_bb
);
6286 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6287 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6288 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6289 t
= fold_convert (TREE_TYPE (startvar
), t
);
6290 t
= force_gimple_operand_gsi (&gsi
, t
,
6292 && TREE_ADDRESSABLE (startvar
),
6293 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6294 assign_stmt
= gimple_build_assign (startvar
, t
);
6295 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6299 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6300 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6301 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6302 t
= fold_convert (TREE_TYPE (startvar
), t
);
6303 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6304 false, GSI_CONTINUE_LINKING
);
6307 assign_stmt
= gimple_build_assign (endvar
, iend
);
6308 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6309 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
6310 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
6312 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
6313 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6315 if (fd
->collapse
> 1)
6316 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6320 /* Code to control the increment and predicate for the sequential
6321 loop goes in the CONT_BB. */
6322 gsi
= gsi_last_bb (cont_bb
);
6323 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6324 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6325 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6326 vback
= gimple_omp_continue_control_def (cont_stmt
);
6328 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6330 if (POINTER_TYPE_P (type
))
6331 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
6333 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
6334 t
= force_gimple_operand_gsi (&gsi
, t
,
6336 && TREE_ADDRESSABLE (vback
),
6337 NULL_TREE
, true, GSI_SAME_STMT
);
6338 assign_stmt
= gimple_build_assign (vback
, t
);
6339 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6341 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6342 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
6344 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6345 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6348 /* Remove GIMPLE_OMP_CONTINUE. */
6349 gsi_remove (&gsi
, true);
6351 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6352 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
6354 /* Emit code to get the next parallel iteration in L2_BB. */
6355 gsi
= gsi_start_bb (l2_bb
);
6357 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6358 build_fold_addr_expr (istart0
),
6359 build_fold_addr_expr (iend0
));
6360 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6361 false, GSI_CONTINUE_LINKING
);
6362 if (TREE_TYPE (t
) != boolean_type_node
)
6363 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6364 t
, build_int_cst (TREE_TYPE (t
), 0));
6365 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6366 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
6369 /* Add the loop cleanup function. */
6370 gsi
= gsi_last_bb (exit_bb
);
6371 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6372 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
6373 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6374 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
6376 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
6377 gcall
*call_stmt
= gimple_build_call (t
, 0);
6378 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6379 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
6380 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
6381 gsi_remove (&gsi
, true);
6383 /* Connect the new blocks. */
6384 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
6385 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
6391 e
= find_edge (cont_bb
, l3_bb
);
6392 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
6394 phis
= phi_nodes (l3_bb
);
6395 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6397 gimple phi
= gsi_stmt (gsi
);
6398 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
6399 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
6403 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
6404 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
6405 e
= find_edge (cont_bb
, l1_bb
);
6406 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6411 else if (fd
->collapse
> 1)
6414 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6417 e
->flags
= EDGE_TRUE_VALUE
;
6420 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6421 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
6425 e
= find_edge (cont_bb
, l2_bb
);
6426 e
->flags
= EDGE_FALLTHRU
;
6428 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6430 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
6431 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
6432 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
6433 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
6434 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
6435 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
6436 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
6437 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
6439 struct loop
*outer_loop
= alloc_loop ();
6440 outer_loop
->header
= l0_bb
;
6441 outer_loop
->latch
= l2_bb
;
6442 add_loop (outer_loop
, l0_bb
->loop_father
);
6444 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6446 struct loop
*loop
= alloc_loop ();
6447 loop
->header
= l1_bb
;
6448 /* The loop may have multiple latches. */
6449 add_loop (loop
, outer_loop
);
6455 /* A subroutine of expand_omp_for. Generate code for a parallel
6456 loop with static schedule and no specified chunk size. Given
6459 for (V = N1; V cond N2; V += STEP) BODY;
6461 where COND is "<" or ">", we generate pseudocode
6463 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6468 if ((__typeof (V)) -1 > 0 && cond is >)
6469 n = -(adj + N2 - N1) / -STEP;
6471 n = (adj + N2 - N1) / STEP;
6474 if (threadid < tt) goto L3; else goto L4;
6479 s0 = q * threadid + tt;
6482 if (s0 >= e0) goto L2; else goto L0;
6488 if (V cond e) goto L1;
6493 expand_omp_for_static_nochunk (struct omp_region
*region
,
6494 struct omp_for_data
*fd
,
6497 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6498 tree type
, itype
, vmain
, vback
;
6499 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6500 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6502 gimple_stmt_iterator gsi
;
6504 bool broken_loop
= region
->cont
== NULL
;
6505 tree
*counts
= NULL
;
6508 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6509 != GF_OMP_FOR_KIND_OACC_LOOP
)
6512 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6513 if (POINTER_TYPE_P (type
))
6514 itype
= signed_type_for (type
);
6516 entry_bb
= region
->entry
;
6517 cont_bb
= region
->cont
;
6518 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6519 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6520 gcc_assert (broken_loop
6521 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6522 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6523 body_bb
= single_succ (seq_start_bb
);
6526 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6527 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6529 exit_bb
= region
->exit
;
6531 /* Iteration space partitioning goes in ENTRY_BB. */
6532 gsi
= gsi_last_bb (entry_bb
);
6533 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6535 if (fd
->collapse
> 1)
6537 int first_zero_iter
= -1;
6538 basic_block l2_dom_bb
= NULL
;
6540 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6541 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6542 fin_bb
, first_zero_iter
,
6546 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6547 t
= integer_one_node
;
6549 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6550 fold_convert (type
, fd
->loop
.n1
),
6551 fold_convert (type
, fd
->loop
.n2
));
6552 if (fd
->collapse
== 1
6553 && TYPE_UNSIGNED (type
)
6554 && (t
== NULL_TREE
|| !integer_onep (t
)))
6556 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6557 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6558 true, GSI_SAME_STMT
);
6559 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6560 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6561 true, GSI_SAME_STMT
);
6562 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6563 NULL_TREE
, NULL_TREE
);
6564 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6565 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6566 expand_omp_regimplify_p
, NULL
, NULL
)
6567 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6568 expand_omp_regimplify_p
, NULL
, NULL
))
6570 gsi
= gsi_for_stmt (cond_stmt
);
6571 gimple_regimplify_operands (cond_stmt
, &gsi
);
6573 ep
= split_block (entry_bb
, cond_stmt
);
6574 ep
->flags
= EDGE_TRUE_VALUE
;
6575 entry_bb
= ep
->dest
;
6576 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6577 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6578 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6579 if (gimple_in_ssa_p (cfun
))
6581 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6582 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6583 !gsi_end_p (gpi
); gsi_next (&gpi
))
6585 gphi
*phi
= gpi
.phi ();
6586 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6587 ep
, UNKNOWN_LOCATION
);
6590 gsi
= gsi_last_bb (entry_bb
);
6593 switch (gimple_omp_for_kind (fd
->for_stmt
))
6595 case GF_OMP_FOR_KIND_FOR
:
6596 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6597 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6599 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6600 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6601 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6603 case GF_OMP_FOR_KIND_OACC_LOOP
:
6604 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6605 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
6610 nthreads
= build_call_expr (nthreads
, 0);
6611 nthreads
= fold_convert (itype
, nthreads
);
6612 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
6613 true, GSI_SAME_STMT
);
6614 threadid
= build_call_expr (threadid
, 0);
6615 threadid
= fold_convert (itype
, threadid
);
6616 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
6617 true, GSI_SAME_STMT
);
6621 step
= fd
->loop
.step
;
6622 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6624 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6625 OMP_CLAUSE__LOOPTEMP_
);
6626 gcc_assert (innerc
);
6627 n1
= OMP_CLAUSE_DECL (innerc
);
6628 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6629 OMP_CLAUSE__LOOPTEMP_
);
6630 gcc_assert (innerc
);
6631 n2
= OMP_CLAUSE_DECL (innerc
);
6633 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6634 true, NULL_TREE
, true, GSI_SAME_STMT
);
6635 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6636 true, NULL_TREE
, true, GSI_SAME_STMT
);
6637 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6638 true, NULL_TREE
, true, GSI_SAME_STMT
);
6640 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6641 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6642 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6643 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6644 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6645 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6646 fold_build1 (NEGATE_EXPR
, itype
, t
),
6647 fold_build1 (NEGATE_EXPR
, itype
, step
));
6649 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6650 t
= fold_convert (itype
, t
);
6651 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6653 q
= create_tmp_reg (itype
, "q");
6654 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6655 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6656 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6658 tt
= create_tmp_reg (itype
, "tt");
6659 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6660 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6661 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6663 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6664 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6665 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6667 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
6668 gsi
= gsi_last_bb (second_bb
);
6669 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6671 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6673 gassign
*assign_stmt
6674 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
6675 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6677 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
6678 gsi
= gsi_last_bb (third_bb
);
6679 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6681 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6682 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6683 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6685 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6686 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6688 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6689 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6691 /* Remove the GIMPLE_OMP_FOR statement. */
6692 gsi_remove (&gsi
, true);
6694 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6695 gsi
= gsi_start_bb (seq_start_bb
);
6697 tree startvar
= fd
->loop
.v
;
6698 tree endvar
= NULL_TREE
;
6700 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6702 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6703 ? gimple_omp_parallel_clauses (inner_stmt
)
6704 : gimple_omp_for_clauses (inner_stmt
);
6705 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6706 gcc_assert (innerc
);
6707 startvar
= OMP_CLAUSE_DECL (innerc
);
6708 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6709 OMP_CLAUSE__LOOPTEMP_
);
6710 gcc_assert (innerc
);
6711 endvar
= OMP_CLAUSE_DECL (innerc
);
6713 t
= fold_convert (itype
, s0
);
6714 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6715 if (POINTER_TYPE_P (type
))
6716 t
= fold_build_pointer_plus (n1
, t
);
6718 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6719 t
= fold_convert (TREE_TYPE (startvar
), t
);
6720 t
= force_gimple_operand_gsi (&gsi
, t
,
6722 && TREE_ADDRESSABLE (startvar
),
6723 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6724 assign_stmt
= gimple_build_assign (startvar
, t
);
6725 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6727 t
= fold_convert (itype
, e0
);
6728 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6729 if (POINTER_TYPE_P (type
))
6730 t
= fold_build_pointer_plus (n1
, t
);
6732 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6733 t
= fold_convert (TREE_TYPE (startvar
), t
);
6734 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6735 false, GSI_CONTINUE_LINKING
);
6738 assign_stmt
= gimple_build_assign (endvar
, e
);
6739 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6740 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6741 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6743 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
6744 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6746 if (fd
->collapse
> 1)
6747 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6751 /* The code controlling the sequential loop replaces the
6752 GIMPLE_OMP_CONTINUE. */
6753 gsi
= gsi_last_bb (cont_bb
);
6754 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6755 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6756 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6757 vback
= gimple_omp_continue_control_def (cont_stmt
);
6759 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6761 if (POINTER_TYPE_P (type
))
6762 t
= fold_build_pointer_plus (vmain
, step
);
6764 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6765 t
= force_gimple_operand_gsi (&gsi
, t
,
6767 && TREE_ADDRESSABLE (vback
),
6768 NULL_TREE
, true, GSI_SAME_STMT
);
6769 assign_stmt
= gimple_build_assign (vback
, t
);
6770 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6772 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6773 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6775 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6778 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6779 gsi_remove (&gsi
, true);
6781 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6782 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6785 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6786 gsi
= gsi_last_bb (exit_bb
);
6787 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6789 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6790 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
6791 gcc_checking_assert (t
== NULL_TREE
);
6793 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6795 gsi_remove (&gsi
, true);
6797 /* Connect all the blocks. */
6798 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6799 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6800 ep
= find_edge (entry_bb
, second_bb
);
6801 ep
->flags
= EDGE_TRUE_VALUE
;
6802 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6803 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6804 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6808 ep
= find_edge (cont_bb
, body_bb
);
6809 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6814 else if (fd
->collapse
> 1)
6817 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6820 ep
->flags
= EDGE_TRUE_VALUE
;
6821 find_edge (cont_bb
, fin_bb
)->flags
6822 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6825 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6826 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6827 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6829 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6830 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6831 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6832 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6834 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6836 struct loop
*loop
= alloc_loop ();
6837 loop
->header
= body_bb
;
6838 if (collapse_bb
== NULL
)
6839 loop
->latch
= cont_bb
;
6840 add_loop (loop
, body_bb
->loop_father
);
6845 /* A subroutine of expand_omp_for. Generate code for a parallel
6846 loop with static schedule and a specified chunk size. Given
6849 for (V = N1; V cond N2; V += STEP) BODY;
6851 where COND is "<" or ">", we generate pseudocode
6853 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6858 if ((__typeof (V)) -1 > 0 && cond is >)
6859 n = -(adj + N2 - N1) / -STEP;
6861 n = (adj + N2 - N1) / STEP;
6863 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6864 here so that V is defined
6865 if the loop is not entered
6867 s0 = (trip * nthreads + threadid) * CHUNK;
6868 e0 = min(s0 + CHUNK, n);
6869 if (s0 < n) goto L1; else goto L4;
6876 if (V cond e) goto L2; else goto L3;
6884 expand_omp_for_static_chunk (struct omp_region
*region
,
6885 struct omp_for_data
*fd
, gimple inner_stmt
)
6887 tree n
, s0
, e0
, e
, t
;
6888 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6889 tree type
, itype
, vmain
, vback
, vextra
;
6890 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6891 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6892 gimple_stmt_iterator gsi
;
6894 bool broken_loop
= region
->cont
== NULL
;
6895 tree
*counts
= NULL
;
6898 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6899 != GF_OMP_FOR_KIND_OACC_LOOP
)
6902 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6903 if (POINTER_TYPE_P (type
))
6904 itype
= signed_type_for (type
);
6906 entry_bb
= region
->entry
;
6907 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6909 iter_part_bb
= se
->dest
;
6910 cont_bb
= region
->cont
;
6911 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6912 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6913 gcc_assert (broken_loop
6914 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6915 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6916 body_bb
= single_succ (seq_start_bb
);
6919 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6920 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6921 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6923 exit_bb
= region
->exit
;
6925 /* Trip and adjustment setup goes in ENTRY_BB. */
6926 gsi
= gsi_last_bb (entry_bb
);
6927 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6929 if (fd
->collapse
> 1)
6931 int first_zero_iter
= -1;
6932 basic_block l2_dom_bb
= NULL
;
6934 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6935 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6936 fin_bb
, first_zero_iter
,
6940 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6941 t
= integer_one_node
;
6943 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6944 fold_convert (type
, fd
->loop
.n1
),
6945 fold_convert (type
, fd
->loop
.n2
));
6946 if (fd
->collapse
== 1
6947 && TYPE_UNSIGNED (type
)
6948 && (t
== NULL_TREE
|| !integer_onep (t
)))
6950 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6951 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6952 true, GSI_SAME_STMT
);
6953 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6954 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6955 true, GSI_SAME_STMT
);
6956 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6957 NULL_TREE
, NULL_TREE
);
6958 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6959 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6960 expand_omp_regimplify_p
, NULL
, NULL
)
6961 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6962 expand_omp_regimplify_p
, NULL
, NULL
))
6964 gsi
= gsi_for_stmt (cond_stmt
);
6965 gimple_regimplify_operands (cond_stmt
, &gsi
);
6967 se
= split_block (entry_bb
, cond_stmt
);
6968 se
->flags
= EDGE_TRUE_VALUE
;
6969 entry_bb
= se
->dest
;
6970 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6971 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6972 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6973 if (gimple_in_ssa_p (cfun
))
6975 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6976 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6977 !gsi_end_p (gpi
); gsi_next (&gpi
))
6979 gphi
*phi
= gpi
.phi ();
6980 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6981 se
, UNKNOWN_LOCATION
);
6984 gsi
= gsi_last_bb (entry_bb
);
6987 switch (gimple_omp_for_kind (fd
->for_stmt
))
6989 case GF_OMP_FOR_KIND_FOR
:
6990 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6991 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6993 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6994 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6995 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6997 case GF_OMP_FOR_KIND_OACC_LOOP
:
6998 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6999 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
7004 nthreads
= build_call_expr (nthreads
, 0);
7005 nthreads
= fold_convert (itype
, nthreads
);
7006 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
7007 true, GSI_SAME_STMT
);
7008 threadid
= build_call_expr (threadid
, 0);
7009 threadid
= fold_convert (itype
, threadid
);
7010 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
7011 true, GSI_SAME_STMT
);
7015 step
= fd
->loop
.step
;
7016 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7018 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7019 OMP_CLAUSE__LOOPTEMP_
);
7020 gcc_assert (innerc
);
7021 n1
= OMP_CLAUSE_DECL (innerc
);
7022 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7023 OMP_CLAUSE__LOOPTEMP_
);
7024 gcc_assert (innerc
);
7025 n2
= OMP_CLAUSE_DECL (innerc
);
7027 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
7028 true, NULL_TREE
, true, GSI_SAME_STMT
);
7029 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
7030 true, NULL_TREE
, true, GSI_SAME_STMT
);
7031 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
7032 true, NULL_TREE
, true, GSI_SAME_STMT
);
7034 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
7035 true, NULL_TREE
, true, GSI_SAME_STMT
);
7037 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
7038 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
7039 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
7040 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
7041 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
7042 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7043 fold_build1 (NEGATE_EXPR
, itype
, t
),
7044 fold_build1 (NEGATE_EXPR
, itype
, step
));
7046 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
7047 t
= fold_convert (itype
, t
);
7048 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7049 true, GSI_SAME_STMT
);
7051 trip_var
= create_tmp_reg (itype
, ".trip");
7052 if (gimple_in_ssa_p (cfun
))
7054 trip_init
= make_ssa_name (trip_var
);
7055 trip_main
= make_ssa_name (trip_var
);
7056 trip_back
= make_ssa_name (trip_var
);
7060 trip_init
= trip_var
;
7061 trip_main
= trip_var
;
7062 trip_back
= trip_var
;
7065 gassign
*assign_stmt
7066 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
7067 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7069 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
7070 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7071 if (POINTER_TYPE_P (type
))
7072 t
= fold_build_pointer_plus (n1
, t
);
7074 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7075 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7076 true, GSI_SAME_STMT
);
7078 /* Remove the GIMPLE_OMP_FOR. */
7079 gsi_remove (&gsi
, true);
7081 /* Iteration space partitioning goes in ITER_PART_BB. */
7082 gsi
= gsi_last_bb (iter_part_bb
);
7084 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
7085 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
7086 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
7087 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7088 false, GSI_CONTINUE_LINKING
);
7090 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
7091 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
7092 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7093 false, GSI_CONTINUE_LINKING
);
7095 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
7096 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
7098 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7099 gsi
= gsi_start_bb (seq_start_bb
);
7101 tree startvar
= fd
->loop
.v
;
7102 tree endvar
= NULL_TREE
;
7104 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7106 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
7107 ? gimple_omp_parallel_clauses (inner_stmt
)
7108 : gimple_omp_for_clauses (inner_stmt
);
7109 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7110 gcc_assert (innerc
);
7111 startvar
= OMP_CLAUSE_DECL (innerc
);
7112 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7113 OMP_CLAUSE__LOOPTEMP_
);
7114 gcc_assert (innerc
);
7115 endvar
= OMP_CLAUSE_DECL (innerc
);
7118 t
= fold_convert (itype
, s0
);
7119 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7120 if (POINTER_TYPE_P (type
))
7121 t
= fold_build_pointer_plus (n1
, t
);
7123 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7124 t
= fold_convert (TREE_TYPE (startvar
), t
);
7125 t
= force_gimple_operand_gsi (&gsi
, t
,
7127 && TREE_ADDRESSABLE (startvar
),
7128 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7129 assign_stmt
= gimple_build_assign (startvar
, t
);
7130 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7132 t
= fold_convert (itype
, e0
);
7133 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7134 if (POINTER_TYPE_P (type
))
7135 t
= fold_build_pointer_plus (n1
, t
);
7137 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7138 t
= fold_convert (TREE_TYPE (startvar
), t
);
7139 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7140 false, GSI_CONTINUE_LINKING
);
7143 assign_stmt
= gimple_build_assign (endvar
, e
);
7144 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7145 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
7146 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
7148 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
7149 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7151 if (fd
->collapse
> 1)
7152 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
7156 /* The code controlling the sequential loop goes in CONT_BB,
7157 replacing the GIMPLE_OMP_CONTINUE. */
7158 gsi
= gsi_last_bb (cont_bb
);
7159 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
7160 vmain
= gimple_omp_continue_control_use (cont_stmt
);
7161 vback
= gimple_omp_continue_control_def (cont_stmt
);
7163 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7165 if (POINTER_TYPE_P (type
))
7166 t
= fold_build_pointer_plus (vmain
, step
);
7168 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
7169 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
7170 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7171 true, GSI_SAME_STMT
);
7172 assign_stmt
= gimple_build_assign (vback
, t
);
7173 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7175 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
7176 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
7178 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
7181 /* Remove GIMPLE_OMP_CONTINUE. */
7182 gsi_remove (&gsi
, true);
7184 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
7185 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
7187 /* Trip update code goes into TRIP_UPDATE_BB. */
7188 gsi
= gsi_start_bb (trip_update_bb
);
7190 t
= build_int_cst (itype
, 1);
7191 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
7192 assign_stmt
= gimple_build_assign (trip_back
, t
);
7193 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7196 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7197 gsi
= gsi_last_bb (exit_bb
);
7198 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
7200 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
7201 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
7202 gcc_checking_assert (t
== NULL_TREE
);
7204 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
7206 gsi_remove (&gsi
, true);
7208 /* Connect the new blocks. */
7209 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
7210 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
7214 se
= find_edge (cont_bb
, body_bb
);
7215 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7220 else if (fd
->collapse
> 1)
7223 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
7226 se
->flags
= EDGE_TRUE_VALUE
;
7227 find_edge (cont_bb
, trip_update_bb
)->flags
7228 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
7230 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
7233 if (gimple_in_ssa_p (cfun
))
7241 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
7243 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7244 remove arguments of the phi nodes in fin_bb. We need to create
7245 appropriate phi nodes in iter_part_bb instead. */
7246 se
= single_pred_edge (fin_bb
);
7247 re
= single_succ_edge (trip_update_bb
);
7248 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
7249 ene
= single_succ_edge (entry_bb
);
7251 psi
= gsi_start_phis (fin_bb
);
7252 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
7253 gsi_next (&psi
), ++i
)
7256 source_location locus
;
7259 t
= gimple_phi_result (phi
);
7260 gcc_assert (t
== redirect_edge_var_map_result (vm
));
7261 nphi
= create_phi_node (t
, iter_part_bb
);
7263 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
7264 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
7266 /* A special case -- fd->loop.v is not yet computed in
7267 iter_part_bb, we need to use vextra instead. */
7268 if (t
== fd
->loop
.v
)
7270 add_phi_arg (nphi
, t
, ene
, locus
);
7271 locus
= redirect_edge_var_map_location (vm
);
7272 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
7274 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
7275 redirect_edge_var_map_clear (re
);
7278 psi
= gsi_start_phis (fin_bb
);
7279 if (gsi_end_p (psi
))
7281 remove_phi_node (&psi
, false);
7284 /* Make phi node for trip. */
7285 phi
= create_phi_node (trip_main
, iter_part_bb
);
7286 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
7288 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
7293 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
7294 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
7295 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
7296 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
7297 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
7298 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
7299 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
7300 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
7301 recompute_dominator (CDI_DOMINATORS
, body_bb
));
7305 struct loop
*trip_loop
= alloc_loop ();
7306 trip_loop
->header
= iter_part_bb
;
7307 trip_loop
->latch
= trip_update_bb
;
7308 add_loop (trip_loop
, iter_part_bb
->loop_father
);
7310 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7312 struct loop
*loop
= alloc_loop ();
7313 loop
->header
= body_bb
;
7314 if (collapse_bb
== NULL
)
7315 loop
->latch
= cont_bb
;
7316 add_loop (loop
, trip_loop
);
7321 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7323 for (V = N1; V cond N2; V += STEP) BODY;
7325 where COND is "<" or ">" or "!=", we generate pseudocode
7327 for (ind_var = low; ind_var < high; ind_var++)
7329 V = n1 + (ind_var * STEP)
7334 In the above pseudocode, low and high are function parameters of the
7335 child function. In the function below, we are inserting a temp.
7336 variable that will be making a call to two OMP functions that will not be
7337 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7338 with _Cilk_for). These functions are replaced with low and high
7339 by the function that handles taskreg. */
7343 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
7345 bool broken_loop
= region
->cont
== NULL
;
7346 basic_block entry_bb
= region
->entry
;
7347 basic_block cont_bb
= region
->cont
;
7349 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7350 gcc_assert (broken_loop
7351 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7352 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7353 basic_block l1_bb
, l2_bb
;
7357 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7358 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7359 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7360 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7364 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7365 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7366 l2_bb
= single_succ (l1_bb
);
7368 basic_block exit_bb
= region
->exit
;
7369 basic_block l2_dom_bb
= NULL
;
7371 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
7373 /* Below statements until the "tree high_val = ..." are pseudo statements
7374 used to pass information to be used by expand_omp_taskreg.
7375 low_val and high_val will be replaced by the __low and __high
7376 parameter from the child function.
7378 The call_exprs part is a place-holder, it is mainly used
7379 to distinctly identify to the top-level part that this is
7380 where we should put low and high (reasoning given in header
7384 = gimple_omp_parallel_child_fn (
7385 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
7386 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
7387 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
7389 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
7391 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
7394 gcc_assert (low_val
&& high_val
);
7396 tree type
= TREE_TYPE (low_val
);
7397 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
7398 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7400 /* Not needed in SSA form right now. */
7401 gcc_assert (!gimple_in_ssa_p (cfun
));
7402 if (l2_dom_bb
== NULL
)
7408 gimple stmt
= gimple_build_assign (ind_var
, n1
);
7410 /* Replace the GIMPLE_OMP_FOR statement. */
7411 gsi_replace (&gsi
, stmt
, true);
7415 /* Code to control the increment goes in the CONT_BB. */
7416 gsi
= gsi_last_bb (cont_bb
);
7417 stmt
= gsi_stmt (gsi
);
7418 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7419 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
7420 build_one_cst (type
));
7422 /* Replace GIMPLE_OMP_CONTINUE. */
7423 gsi_replace (&gsi
, stmt
, true);
7426 /* Emit the condition in L1_BB. */
7427 gsi
= gsi_after_labels (l1_bb
);
7428 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
7429 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
7431 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
7432 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7433 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7435 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7436 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
7437 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
7438 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7440 /* The condition is always '<' since the runtime will fill in the low
7442 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
7443 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7445 /* Remove GIMPLE_OMP_RETURN. */
7446 gsi
= gsi_last_bb (exit_bb
);
7447 gsi_remove (&gsi
, true);
7449 /* Connect the new blocks. */
7450 remove_edge (FALLTHRU_EDGE (entry_bb
));
7455 remove_edge (BRANCH_EDGE (entry_bb
));
7456 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7458 e
= BRANCH_EDGE (l1_bb
);
7459 ne
= FALLTHRU_EDGE (l1_bb
);
7460 e
->flags
= EDGE_TRUE_VALUE
;
7464 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7466 ne
= single_succ_edge (l1_bb
);
7467 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7470 ne
->flags
= EDGE_FALSE_VALUE
;
7471 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7472 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7474 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7475 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7476 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7480 struct loop
*loop
= alloc_loop ();
7481 loop
->header
= l1_bb
;
7482 loop
->latch
= cont_bb
;
7483 add_loop (loop
, l1_bb
->loop_father
);
7484 loop
->safelen
= INT_MAX
;
7487 /* Pick the correct library function based on the precision of the
7488 induction variable type. */
7489 tree lib_fun
= NULL_TREE
;
7490 if (TYPE_PRECISION (type
) == 32)
7491 lib_fun
= cilk_for_32_fndecl
;
7492 else if (TYPE_PRECISION (type
) == 64)
7493 lib_fun
= cilk_for_64_fndecl
;
7497 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7499 /* WS_ARGS contains the library function flavor to call:
7500 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7501 user-defined grain value. If the user does not define one, then zero
7502 is passed in by the parser. */
7503 vec_alloc (region
->ws_args
, 2);
7504 region
->ws_args
->quick_push (lib_fun
);
7505 region
->ws_args
->quick_push (fd
->chunk_size
);
7508 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7509 loop. Given parameters:
7511 for (V = N1; V cond N2; V += STEP) BODY;
7513 where COND is "<" or ">", we generate pseudocode
7521 if (V cond N2) goto L0; else goto L2;
7524 For collapsed loops, given parameters:
7526 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7527 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7528 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7531 we generate pseudocode
7537 count3 = (adj + N32 - N31) / STEP3;
7542 count2 = (adj + N22 - N21) / STEP2;
7547 count1 = (adj + N12 - N11) / STEP1;
7548 count = count1 * count2 * count3;
7558 V2 += (V3 cond3 N32) ? 0 : STEP2;
7559 V3 = (V3 cond3 N32) ? V3 : N31;
7560 V1 += (V2 cond2 N22) ? 0 : STEP1;
7561 V2 = (V2 cond2 N22) ? V2 : N21;
7563 if (V < count) goto L0; else goto L2;
7569 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7572 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7573 gimple_stmt_iterator gsi
;
7576 bool broken_loop
= region
->cont
== NULL
;
7578 tree
*counts
= NULL
;
7580 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7581 OMP_CLAUSE_SAFELEN
);
7582 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7583 OMP_CLAUSE__SIMDUID_
);
7586 type
= TREE_TYPE (fd
->loop
.v
);
7587 entry_bb
= region
->entry
;
7588 cont_bb
= region
->cont
;
7589 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7590 gcc_assert (broken_loop
7591 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7592 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7595 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7596 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7597 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7598 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7602 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7603 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7604 l2_bb
= single_succ (l1_bb
);
7606 exit_bb
= region
->exit
;
7609 gsi
= gsi_last_bb (entry_bb
);
7611 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7612 /* Not needed in SSA form right now. */
7613 gcc_assert (!gimple_in_ssa_p (cfun
));
7614 if (fd
->collapse
> 1)
7616 int first_zero_iter
= -1;
7617 basic_block zero_iter_bb
= l2_bb
;
7619 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7620 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7621 zero_iter_bb
, first_zero_iter
,
7624 if (l2_dom_bb
== NULL
)
7629 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7631 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7632 OMP_CLAUSE__LOOPTEMP_
);
7633 gcc_assert (innerc
);
7634 n1
= OMP_CLAUSE_DECL (innerc
);
7635 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7636 OMP_CLAUSE__LOOPTEMP_
);
7637 gcc_assert (innerc
);
7638 n2
= OMP_CLAUSE_DECL (innerc
);
7639 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7640 fold_convert (type
, n1
));
7641 if (fd
->collapse
> 1)
7644 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7650 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7651 fold_convert (type
, fd
->loop
.n1
));
7652 if (fd
->collapse
> 1)
7653 for (i
= 0; i
< fd
->collapse
; i
++)
7655 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7656 if (POINTER_TYPE_P (itype
))
7657 itype
= signed_type_for (itype
);
7658 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7659 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7663 /* Remove the GIMPLE_OMP_FOR statement. */
7664 gsi_remove (&gsi
, true);
7668 /* Code to control the increment goes in the CONT_BB. */
7669 gsi
= gsi_last_bb (cont_bb
);
7670 stmt
= gsi_stmt (gsi
);
7671 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7673 if (POINTER_TYPE_P (type
))
7674 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7676 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7677 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7679 if (fd
->collapse
> 1)
7681 i
= fd
->collapse
- 1;
7682 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7684 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7685 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7689 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7691 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7694 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7696 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7698 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7699 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7700 if (POINTER_TYPE_P (itype2
))
7701 itype2
= signed_type_for (itype2
);
7702 t
= build3 (COND_EXPR
, itype2
,
7703 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7705 fold_convert (itype
, fd
->loops
[i
].n2
)),
7706 build_int_cst (itype2
, 0),
7707 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7708 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7709 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7711 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7712 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7714 t
= build3 (COND_EXPR
, itype
,
7715 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7717 fold_convert (itype
, fd
->loops
[i
].n2
)),
7719 fold_convert (itype
, fd
->loops
[i
].n1
));
7720 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7724 /* Remove GIMPLE_OMP_CONTINUE. */
7725 gsi_remove (&gsi
, true);
7728 /* Emit the condition in L1_BB. */
7729 gsi
= gsi_start_bb (l1_bb
);
7731 t
= fold_convert (type
, n2
);
7732 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7733 false, GSI_CONTINUE_LINKING
);
7734 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7735 cond_stmt
= gimple_build_cond_empty (t
);
7736 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
7737 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7739 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7742 gsi
= gsi_for_stmt (cond_stmt
);
7743 gimple_regimplify_operands (cond_stmt
, &gsi
);
7746 /* Remove GIMPLE_OMP_RETURN. */
7747 gsi
= gsi_last_bb (exit_bb
);
7748 gsi_remove (&gsi
, true);
7750 /* Connect the new blocks. */
7751 remove_edge (FALLTHRU_EDGE (entry_bb
));
7755 remove_edge (BRANCH_EDGE (entry_bb
));
7756 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7758 e
= BRANCH_EDGE (l1_bb
);
7759 ne
= FALLTHRU_EDGE (l1_bb
);
7760 e
->flags
= EDGE_TRUE_VALUE
;
7764 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7766 ne
= single_succ_edge (l1_bb
);
7767 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7770 ne
->flags
= EDGE_FALSE_VALUE
;
7771 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7772 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7774 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7775 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7776 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7780 struct loop
*loop
= alloc_loop ();
7781 loop
->header
= l1_bb
;
7782 loop
->latch
= cont_bb
;
7783 add_loop (loop
, l1_bb
->loop_father
);
7784 if (safelen
== NULL_TREE
)
7785 loop
->safelen
= INT_MAX
;
7788 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7789 if (TREE_CODE (safelen
) != INTEGER_CST
)
7791 else if (!tree_fits_uhwi_p (safelen
)
7792 || tree_to_uhwi (safelen
) > INT_MAX
)
7793 loop
->safelen
= INT_MAX
;
7795 loop
->safelen
= tree_to_uhwi (safelen
);
7796 if (loop
->safelen
== 1)
7801 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7802 cfun
->has_simduid_loops
= true;
7804 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7806 if ((flag_tree_loop_vectorize
7807 || (!global_options_set
.x_flag_tree_loop_vectorize
7808 && !global_options_set
.x_flag_tree_vectorize
))
7809 && flag_tree_loop_optimize
7810 && loop
->safelen
> 1)
7812 loop
->force_vectorize
= true;
7813 cfun
->has_force_vectorize_loops
= true;
7819 /* Expand the OMP loop defined by REGION. */
7822 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7824 struct omp_for_data fd
;
7825 struct omp_for_data_loop
*loops
;
7828 = (struct omp_for_data_loop
*)
7829 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7830 * sizeof (struct omp_for_data_loop
));
7831 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
7833 region
->sched_kind
= fd
.sched_kind
;
7835 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7836 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7837 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7840 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7841 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7842 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7845 /* If there isn't a continue then this is a degerate case where
7846 the introduction of abnormal edges during lowering will prevent
7847 original loops from being detected. Fix that up. */
7848 loops_state_set (LOOPS_NEED_FIXUP
);
7850 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7851 expand_omp_simd (region
, &fd
);
7852 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7853 expand_cilk_for (region
, &fd
);
7854 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7855 && !fd
.have_ordered
)
7857 if (fd
.chunk_size
== NULL
)
7858 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7860 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7864 int fn_index
, start_ix
, next_ix
;
7866 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7867 == GF_OMP_FOR_KIND_FOR
);
7868 if (fd
.chunk_size
== NULL
7869 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7870 fd
.chunk_size
= integer_zero_node
;
7871 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7872 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7873 ? 3 : fd
.sched_kind
;
7874 fn_index
+= fd
.have_ordered
* 4;
7875 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7876 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7877 if (fd
.iter_type
== long_long_unsigned_type_node
)
7879 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7880 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7881 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7882 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7884 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7885 (enum built_in_function
) next_ix
, inner_stmt
);
7888 if (gimple_in_ssa_p (cfun
))
7889 update_ssa (TODO_update_ssa_only_virtuals
);
7893 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7895 v = GOMP_sections_start (n);
7912 v = GOMP_sections_next ();
7917 If this is a combined parallel sections, replace the call to
7918 GOMP_sections_start with call to GOMP_sections_next. */
7921 expand_omp_sections (struct omp_region
*region
)
7923 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7925 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7926 gimple_stmt_iterator si
, switch_si
;
7927 gomp_sections
*sections_stmt
;
7929 gomp_continue
*cont
;
7932 struct omp_region
*inner
;
7934 bool exit_reachable
= region
->cont
!= NULL
;
7936 gcc_assert (region
->exit
!= NULL
);
7937 entry_bb
= region
->entry
;
7938 l0_bb
= single_succ (entry_bb
);
7939 l1_bb
= region
->cont
;
7940 l2_bb
= region
->exit
;
7941 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7942 l2
= gimple_block_label (l2_bb
);
7945 /* This can happen if there are reductions. */
7946 len
= EDGE_COUNT (l0_bb
->succs
);
7947 gcc_assert (len
> 0);
7948 e
= EDGE_SUCC (l0_bb
, len
- 1);
7949 si
= gsi_last_bb (e
->dest
);
7952 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7953 l2
= gimple_block_label (e
->dest
);
7955 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7957 si
= gsi_last_bb (e
->dest
);
7959 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7961 l2
= gimple_block_label (e
->dest
);
7967 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7969 default_bb
= create_empty_bb (l0_bb
);
7971 /* We will build a switch() with enough cases for all the
7972 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7973 and a default case to abort if something goes wrong. */
7974 len
= EDGE_COUNT (l0_bb
->succs
);
7976 /* Use vec::quick_push on label_vec throughout, since we know the size
7978 auto_vec
<tree
> label_vec (len
);
7980 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7981 GIMPLE_OMP_SECTIONS statement. */
7982 si
= gsi_last_bb (entry_bb
);
7983 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
7984 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7985 vin
= gimple_omp_sections_control (sections_stmt
);
7986 if (!is_combined_parallel (region
))
7988 /* If we are not inside a combined parallel+sections region,
7989 call GOMP_sections_start. */
7990 t
= build_int_cst (unsigned_type_node
, len
- 1);
7991 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7992 stmt
= gimple_build_call (u
, 1, t
);
7996 /* Otherwise, call GOMP_sections_next. */
7997 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7998 stmt
= gimple_build_call (u
, 0);
8000 gimple_call_set_lhs (stmt
, vin
);
8001 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8002 gsi_remove (&si
, true);
8004 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8006 switch_si
= gsi_last_bb (l0_bb
);
8007 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
8010 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
8011 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
8012 vmain
= gimple_omp_continue_control_use (cont
);
8013 vnext
= gimple_omp_continue_control_def (cont
);
8021 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
8022 label_vec
.quick_push (t
);
8025 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8026 for (inner
= region
->inner
, casei
= 1;
8028 inner
= inner
->next
, i
++, casei
++)
8030 basic_block s_entry_bb
, s_exit_bb
;
8032 /* Skip optional reduction region. */
8033 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
8040 s_entry_bb
= inner
->entry
;
8041 s_exit_bb
= inner
->exit
;
8043 t
= gimple_block_label (s_entry_bb
);
8044 u
= build_int_cst (unsigned_type_node
, casei
);
8045 u
= build_case_label (u
, NULL
, t
);
8046 label_vec
.quick_push (u
);
8048 si
= gsi_last_bb (s_entry_bb
);
8049 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
8050 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
8051 gsi_remove (&si
, true);
8052 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
8054 if (s_exit_bb
== NULL
)
8057 si
= gsi_last_bb (s_exit_bb
);
8058 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8059 gsi_remove (&si
, true);
8061 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
8064 /* Error handling code goes in DEFAULT_BB. */
8065 t
= gimple_block_label (default_bb
);
8066 u
= build_case_label (NULL
, NULL
, t
);
8067 make_edge (l0_bb
, default_bb
, 0);
8068 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
8070 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
8071 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
8072 gsi_remove (&switch_si
, true);
8074 si
= gsi_start_bb (default_bb
);
8075 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
8076 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
8082 /* Code to get the next section goes in L1_BB. */
8083 si
= gsi_last_bb (l1_bb
);
8084 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
8086 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8087 stmt
= gimple_build_call (bfn_decl
, 0);
8088 gimple_call_set_lhs (stmt
, vnext
);
8089 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8090 gsi_remove (&si
, true);
8092 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
8095 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8096 si
= gsi_last_bb (l2_bb
);
8097 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
8098 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
8099 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
8100 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
8102 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
8103 stmt
= gimple_build_call (t
, 0);
8104 if (gimple_omp_return_lhs (gsi_stmt (si
)))
8105 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
8106 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8107 gsi_remove (&si
, true);
8109 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
8113 /* Expand code for an OpenMP single directive. We've already expanded
8114 much of the code, here we simply place the GOMP_barrier call. */
8117 expand_omp_single (struct omp_region
*region
)
8119 basic_block entry_bb
, exit_bb
;
8120 gimple_stmt_iterator si
;
8122 entry_bb
= region
->entry
;
8123 exit_bb
= region
->exit
;
8125 si
= gsi_last_bb (entry_bb
);
8126 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
8127 gsi_remove (&si
, true);
8128 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8130 si
= gsi_last_bb (exit_bb
);
8131 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
8133 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
8134 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
8136 gsi_remove (&si
, true);
8137 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8141 /* Generic expansion for OpenMP synchronization directives: master,
8142 ordered and critical. All we need to do here is remove the entry
8143 and exit markers for REGION. */
8146 expand_omp_synch (struct omp_region
*region
)
8148 basic_block entry_bb
, exit_bb
;
8149 gimple_stmt_iterator si
;
8151 entry_bb
= region
->entry
;
8152 exit_bb
= region
->exit
;
8154 si
= gsi_last_bb (entry_bb
);
8155 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
8156 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
8157 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
8158 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
8159 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
8160 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
8161 gsi_remove (&si
, true);
8162 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8166 si
= gsi_last_bb (exit_bb
);
8167 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8168 gsi_remove (&si
, true);
8169 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8173 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8174 operation as a normal volatile load. */
8177 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
8178 tree loaded_val
, int index
)
8180 enum built_in_function tmpbase
;
8181 gimple_stmt_iterator gsi
;
8182 basic_block store_bb
;
8185 tree decl
, call
, type
, itype
;
8187 gsi
= gsi_last_bb (load_bb
);
8188 stmt
= gsi_stmt (gsi
);
8189 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8190 loc
= gimple_location (stmt
);
8192 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8193 is smaller than word size, then expand_atomic_load assumes that the load
8194 is atomic. We could avoid the builtin entirely in this case. */
8196 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8197 decl
= builtin_decl_explicit (tmpbase
);
8198 if (decl
== NULL_TREE
)
8201 type
= TREE_TYPE (loaded_val
);
8202 itype
= TREE_TYPE (TREE_TYPE (decl
));
8204 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
8205 build_int_cst (NULL
,
8206 gimple_omp_atomic_seq_cst_p (stmt
)
8208 : MEMMODEL_RELAXED
));
8209 if (!useless_type_conversion_p (type
, itype
))
8210 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8211 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8213 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8214 gsi_remove (&gsi
, true);
8216 store_bb
= single_succ (load_bb
);
8217 gsi
= gsi_last_bb (store_bb
);
8218 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8219 gsi_remove (&gsi
, true);
8221 if (gimple_in_ssa_p (cfun
))
8222 update_ssa (TODO_update_ssa_no_phi
);
8227 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8228 operation as a normal volatile store. */
8231 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
8232 tree loaded_val
, tree stored_val
, int index
)
8234 enum built_in_function tmpbase
;
8235 gimple_stmt_iterator gsi
;
8236 basic_block store_bb
= single_succ (load_bb
);
8239 tree decl
, call
, type
, itype
;
8243 gsi
= gsi_last_bb (load_bb
);
8244 stmt
= gsi_stmt (gsi
);
8245 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8247 /* If the load value is needed, then this isn't a store but an exchange. */
8248 exchange
= gimple_omp_atomic_need_value_p (stmt
);
8250 gsi
= gsi_last_bb (store_bb
);
8251 stmt
= gsi_stmt (gsi
);
8252 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
8253 loc
= gimple_location (stmt
);
8255 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8256 is smaller than word size, then expand_atomic_store assumes that the store
8257 is atomic. We could avoid the builtin entirely in this case. */
8259 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
8260 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
8261 decl
= builtin_decl_explicit (tmpbase
);
8262 if (decl
== NULL_TREE
)
8265 type
= TREE_TYPE (stored_val
);
8267 /* Dig out the type of the function's second argument. */
8268 itype
= TREE_TYPE (decl
);
8269 itype
= TYPE_ARG_TYPES (itype
);
8270 itype
= TREE_CHAIN (itype
);
8271 itype
= TREE_VALUE (itype
);
8272 imode
= TYPE_MODE (itype
);
8274 if (exchange
&& !can_atomic_exchange_p (imode
, true))
8277 if (!useless_type_conversion_p (itype
, type
))
8278 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
8279 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
8280 build_int_cst (NULL
,
8281 gimple_omp_atomic_seq_cst_p (stmt
)
8283 : MEMMODEL_RELAXED
));
8286 if (!useless_type_conversion_p (type
, itype
))
8287 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8288 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8291 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8292 gsi_remove (&gsi
, true);
8294 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8295 gsi
= gsi_last_bb (load_bb
);
8296 gsi_remove (&gsi
, true);
8298 if (gimple_in_ssa_p (cfun
))
8299 update_ssa (TODO_update_ssa_no_phi
);
8304 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8305 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8306 size of the data type, and thus usable to find the index of the builtin
8307 decl. Returns false if the expression is not of the proper form. */
8310 expand_omp_atomic_fetch_op (basic_block load_bb
,
8311 tree addr
, tree loaded_val
,
8312 tree stored_val
, int index
)
8314 enum built_in_function oldbase
, newbase
, tmpbase
;
8315 tree decl
, itype
, call
;
8317 basic_block store_bb
= single_succ (load_bb
);
8318 gimple_stmt_iterator gsi
;
8321 enum tree_code code
;
8322 bool need_old
, need_new
;
8326 /* We expect to find the following sequences:
8329 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8332 val = tmp OP something; (or: something OP tmp)
8333 GIMPLE_OMP_STORE (val)
8335 ???FIXME: Allow a more flexible sequence.
8336 Perhaps use data flow to pick the statements.
8340 gsi
= gsi_after_labels (store_bb
);
8341 stmt
= gsi_stmt (gsi
);
8342 loc
= gimple_location (stmt
);
8343 if (!is_gimple_assign (stmt
))
8346 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
8348 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
8349 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
8350 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
8351 gcc_checking_assert (!need_old
|| !need_new
);
8353 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
8356 /* Check for one of the supported fetch-op operations. */
8357 code
= gimple_assign_rhs_code (stmt
);
8361 case POINTER_PLUS_EXPR
:
8362 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
8363 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
8366 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
8367 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
8370 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
8371 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
8374 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
8375 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
8378 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
8379 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
8385 /* Make sure the expression is of the proper form. */
8386 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
8387 rhs
= gimple_assign_rhs2 (stmt
);
8388 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
8389 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
8390 rhs
= gimple_assign_rhs1 (stmt
);
8394 tmpbase
= ((enum built_in_function
)
8395 ((need_new
? newbase
: oldbase
) + index
+ 1));
8396 decl
= builtin_decl_explicit (tmpbase
);
8397 if (decl
== NULL_TREE
)
8399 itype
= TREE_TYPE (TREE_TYPE (decl
));
8400 imode
= TYPE_MODE (itype
);
8402 /* We could test all of the various optabs involved, but the fact of the
8403 matter is that (with the exception of i486 vs i586 and xadd) all targets
8404 that support any atomic operaton optab also implements compare-and-swap.
8405 Let optabs.c take care of expanding any compare-and-swap loop. */
8406 if (!can_compare_and_swap_p (imode
, true))
8409 gsi
= gsi_last_bb (load_bb
);
8410 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8412 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8413 It only requires that the operation happen atomically. Thus we can
8414 use the RELAXED memory model. */
8415 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
8416 fold_convert_loc (loc
, itype
, rhs
),
8417 build_int_cst (NULL
,
8418 seq_cst
? MEMMODEL_SEQ_CST
8419 : MEMMODEL_RELAXED
));
8421 if (need_old
|| need_new
)
8423 lhs
= need_old
? loaded_val
: stored_val
;
8424 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
8425 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
8428 call
= fold_convert_loc (loc
, void_type_node
, call
);
8429 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8430 gsi_remove (&gsi
, true);
8432 gsi
= gsi_last_bb (store_bb
);
8433 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8434 gsi_remove (&gsi
, true);
8435 gsi
= gsi_last_bb (store_bb
);
8436 gsi_remove (&gsi
, true);
8438 if (gimple_in_ssa_p (cfun
))
8439 update_ssa (TODO_update_ssa_no_phi
);
8444 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8448 newval = rhs; // with oldval replacing *addr in rhs
8449 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8450 if (oldval != newval)
8453 INDEX is log2 of the size of the data type, and thus usable to find the
8454 index of the builtin decl. */
8457 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
8458 tree addr
, tree loaded_val
, tree stored_val
,
8461 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
8462 tree type
, itype
, cmpxchg
, iaddr
;
8463 gimple_stmt_iterator si
;
8464 basic_block loop_header
= single_succ (load_bb
);
8467 enum built_in_function fncode
;
8469 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8470 order to use the RELAXED memory model effectively. */
8471 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8473 cmpxchg
= builtin_decl_explicit (fncode
);
8474 if (cmpxchg
== NULL_TREE
)
8476 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8477 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
8479 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
8482 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8483 si
= gsi_last_bb (load_bb
);
8484 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8486 /* For floating-point values, we'll need to view-convert them to integers
8487 so that we can perform the atomic compare and swap. Simplify the
8488 following code by always setting up the "i"ntegral variables. */
8489 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
8493 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
8496 = force_gimple_operand_gsi (&si
,
8497 fold_convert (TREE_TYPE (iaddr
), addr
),
8498 false, NULL_TREE
, true, GSI_SAME_STMT
);
8499 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
8500 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8501 loadedi
= create_tmp_var (itype
);
8502 if (gimple_in_ssa_p (cfun
))
8503 loadedi
= make_ssa_name (loadedi
);
8508 loadedi
= loaded_val
;
8511 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8512 tree loaddecl
= builtin_decl_explicit (fncode
);
8515 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8516 build_call_expr (loaddecl
, 2, iaddr
,
8517 build_int_cst (NULL_TREE
,
8518 MEMMODEL_RELAXED
)));
8520 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8521 build_int_cst (TREE_TYPE (iaddr
), 0));
8524 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8527 /* Move the value to the LOADEDI temporary. */
8528 if (gimple_in_ssa_p (cfun
))
8530 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8531 phi
= create_phi_node (loadedi
, loop_header
);
8532 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8536 gsi_insert_before (&si
,
8537 gimple_build_assign (loadedi
, initial
),
8539 if (loadedi
!= loaded_val
)
8541 gimple_stmt_iterator gsi2
;
8544 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8545 gsi2
= gsi_start_bb (loop_header
);
8546 if (gimple_in_ssa_p (cfun
))
8549 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8550 true, GSI_SAME_STMT
);
8551 stmt
= gimple_build_assign (loaded_val
, x
);
8552 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8556 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8557 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8558 true, GSI_SAME_STMT
);
8561 gsi_remove (&si
, true);
8563 si
= gsi_last_bb (store_bb
);
8564 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8567 storedi
= stored_val
;
8570 force_gimple_operand_gsi (&si
,
8571 build1 (VIEW_CONVERT_EXPR
, itype
,
8572 stored_val
), true, NULL_TREE
, true,
8575 /* Build the compare&swap statement. */
8576 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8577 new_storedi
= force_gimple_operand_gsi (&si
,
8578 fold_convert (TREE_TYPE (loadedi
),
8581 true, GSI_SAME_STMT
);
8583 if (gimple_in_ssa_p (cfun
))
8587 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
8588 stmt
= gimple_build_assign (old_vali
, loadedi
);
8589 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8591 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8592 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8595 /* Note that we always perform the comparison as an integer, even for
8596 floating point. This allows the atomic operation to properly
8597 succeed even with NaNs and -0.0. */
8598 stmt
= gimple_build_cond_empty
8599 (build2 (NE_EXPR
, boolean_type_node
,
8600 new_storedi
, old_vali
));
8601 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8604 e
= single_succ_edge (store_bb
);
8605 e
->flags
&= ~EDGE_FALLTHRU
;
8606 e
->flags
|= EDGE_FALSE_VALUE
;
8608 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8610 /* Copy the new value to loadedi (we already did that before the condition
8611 if we are not in SSA). */
8612 if (gimple_in_ssa_p (cfun
))
8614 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8615 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8618 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8619 gsi_remove (&si
, true);
8621 struct loop
*loop
= alloc_loop ();
8622 loop
->header
= loop_header
;
8623 loop
->latch
= store_bb
;
8624 add_loop (loop
, loop_header
->loop_father
);
8626 if (gimple_in_ssa_p (cfun
))
8627 update_ssa (TODO_update_ssa_no_phi
);
8632 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8634 GOMP_atomic_start ();
8638 The result is not globally atomic, but works so long as all parallel
8639 references are within #pragma omp atomic directives. According to
8640 responses received from omp@openmp.org, appears to be within spec.
8641 Which makes sense, since that's how several other compilers handle
8642 this situation as well.
8643 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8644 expanding. STORED_VAL is the operand of the matching
8645 GIMPLE_OMP_ATOMIC_STORE.
8648 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8652 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8657 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8658 tree addr
, tree loaded_val
, tree stored_val
)
8660 gimple_stmt_iterator si
;
8664 si
= gsi_last_bb (load_bb
);
8665 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8667 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8668 t
= build_call_expr (t
, 0);
8669 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8671 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8672 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8673 gsi_remove (&si
, true);
8675 si
= gsi_last_bb (store_bb
);
8676 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8678 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8680 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8682 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8683 t
= build_call_expr (t
, 0);
8684 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8685 gsi_remove (&si
, true);
8687 if (gimple_in_ssa_p (cfun
))
8688 update_ssa (TODO_update_ssa_no_phi
);
8692 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8693 using expand_omp_atomic_fetch_op. If it failed, we try to
8694 call expand_omp_atomic_pipeline, and if it fails too, the
8695 ultimate fallback is wrapping the operation in a mutex
8696 (expand_omp_atomic_mutex). REGION is the atomic region built
8697 by build_omp_regions_1(). */
8700 expand_omp_atomic (struct omp_region
*region
)
8702 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8703 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
8704 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
8705 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8706 tree addr
= gimple_omp_atomic_load_rhs (load
);
8707 tree stored_val
= gimple_omp_atomic_store_val (store
);
8708 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8709 HOST_WIDE_INT index
;
8711 /* Make sure the type is one of the supported sizes. */
8712 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8713 index
= exact_log2 (index
);
8714 if (index
>= 0 && index
<= 4)
8716 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8718 /* __sync builtins require strict data alignment. */
8719 if (exact_log2 (align
) >= index
)
8722 if (loaded_val
== stored_val
8723 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8724 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8725 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8726 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8730 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8731 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8732 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8733 && store_bb
== single_succ (load_bb
)
8734 && first_stmt (store_bb
) == store
8735 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8739 /* When possible, use specialized atomic update functions. */
8740 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8741 && store_bb
== single_succ (load_bb
)
8742 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8743 loaded_val
, stored_val
, index
))
8746 /* If we don't have specialized __sync builtins, try and implement
8747 as a compare and swap loop. */
8748 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8749 loaded_val
, stored_val
, index
))
8754 /* The ultimate fallback is wrapping the operation in a mutex. */
8755 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8759 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8762 expand_omp_target (struct omp_region
*region
)
8764 basic_block entry_bb
, exit_bb
, new_bb
;
8765 struct function
*child_cfun
;
8766 tree child_fn
, block
, t
;
8767 gimple_stmt_iterator gsi
;
8768 gomp_target
*entry_stmt
;
8771 bool offloaded
, data_region
;
8773 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
8774 new_bb
= region
->entry
;
8776 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
8777 switch (gimple_omp_target_kind (entry_stmt
))
8779 case GF_OMP_TARGET_KIND_REGION
:
8780 case GF_OMP_TARGET_KIND_UPDATE
:
8781 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8782 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8783 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8784 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8785 data_region
= false;
8787 case GF_OMP_TARGET_KIND_DATA
:
8788 case GF_OMP_TARGET_KIND_OACC_DATA
:
8795 child_fn
= NULL_TREE
;
8799 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8800 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8803 /* Supported by expand_omp_taskreg, but not here. */
8804 if (child_cfun
!= NULL
)
8805 gcc_checking_assert (!child_cfun
->cfg
);
8806 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
8808 entry_bb
= region
->entry
;
8809 exit_bb
= region
->exit
;
8813 unsigned srcidx
, dstidx
, num
;
8815 /* If the offloading region needs data sent from the parent
8816 function, then the very first statement (except possible
8817 tree profile counter updates) of the offloading body
8818 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8819 &.OMP_DATA_O is passed as an argument to the child function,
8820 we need to replace it with the argument as seen by the child
8823 In most cases, this will end up being the identity assignment
8824 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8825 a function call that has been inlined, the original PARM_DECL
8826 .OMP_DATA_I may have been converted into a different local
8827 variable. In which case, we need to keep the assignment. */
8828 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
8831 basic_block entry_succ_bb
= single_succ (entry_bb
);
8832 gimple_stmt_iterator gsi
;
8834 gimple tgtcopy_stmt
= NULL
;
8835 tree sender
= TREE_VEC_ELT (data_arg
, 0);
8837 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8839 gcc_assert (!gsi_end_p (gsi
));
8840 stmt
= gsi_stmt (gsi
);
8841 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8844 if (gimple_num_ops (stmt
) == 2)
8846 tree arg
= gimple_assign_rhs1 (stmt
);
8848 /* We're ignoring the subcode because we're
8849 effectively doing a STRIP_NOPS. */
8851 if (TREE_CODE (arg
) == ADDR_EXPR
8852 && TREE_OPERAND (arg
, 0) == sender
)
8854 tgtcopy_stmt
= stmt
;
8860 gcc_assert (tgtcopy_stmt
!= NULL
);
8861 arg
= DECL_ARGUMENTS (child_fn
);
8863 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8864 gsi_remove (&gsi
, true);
8867 /* Declare local variables needed in CHILD_CFUN. */
8868 block
= DECL_INITIAL (child_fn
);
8869 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8870 /* The gimplifier could record temporaries in the offloading block
8871 rather than in containing function's local_decls chain,
8872 which would mean cgraph missed finalizing them. Do it now. */
8873 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8874 if (TREE_CODE (t
) == VAR_DECL
8876 && !DECL_EXTERNAL (t
))
8877 varpool_node::finalize_decl (t
);
8878 DECL_SAVED_TREE (child_fn
) = NULL
;
8879 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8880 gimple_set_body (child_fn
, NULL
);
8881 TREE_USED (block
) = 1;
8883 /* Reset DECL_CONTEXT on function arguments. */
8884 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8885 DECL_CONTEXT (t
) = child_fn
;
8887 /* Split ENTRY_BB at GIMPLE_*,
8888 so that it can be moved to the child function. */
8889 gsi
= gsi_last_bb (entry_bb
);
8890 stmt
= gsi_stmt (gsi
);
8892 && gimple_code (stmt
) == gimple_code (entry_stmt
));
8893 gsi_remove (&gsi
, true);
8894 e
= split_block (entry_bb
, stmt
);
8896 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8898 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8901 gsi
= gsi_last_bb (exit_bb
);
8902 gcc_assert (!gsi_end_p (gsi
)
8903 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8904 stmt
= gimple_build_return (NULL
);
8905 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8906 gsi_remove (&gsi
, true);
8909 /* Move the offloading region into CHILD_CFUN. */
8911 block
= gimple_block (entry_stmt
);
8913 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8915 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8916 /* When the OMP expansion process cannot guarantee an up-to-date
8917 loop tree arrange for the child function to fixup loops. */
8918 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8919 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8921 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8922 num
= vec_safe_length (child_cfun
->local_decls
);
8923 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8925 t
= (*child_cfun
->local_decls
)[srcidx
];
8926 if (DECL_CONTEXT (t
) == cfun
->decl
)
8928 if (srcidx
!= dstidx
)
8929 (*child_cfun
->local_decls
)[dstidx
] = t
;
8933 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8935 /* Inform the callgraph about the new function. */
8936 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8937 cgraph_node::add_new_function (child_fn
, true);
8939 #ifdef ENABLE_OFFLOADING
8940 /* Add the new function to the offload table. */
8941 vec_safe_push (offload_funcs
, child_fn
);
8944 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8945 fixed in a following pass. */
8946 push_cfun (child_cfun
);
8947 cgraph_edge::rebuild_edges ();
8949 #ifdef ENABLE_OFFLOADING
8950 /* Prevent IPA from removing child_fn as unreachable, since there are no
8951 refs from the parent function to child_fn in offload LTO mode. */
8952 struct cgraph_node
*node
= cgraph_node::get (child_fn
);
8953 node
->mark_force_output ();
8956 /* Some EH regions might become dead, see PR34608. If
8957 pass_cleanup_cfg isn't the first pass to happen with the
8958 new child, these dead EH edges might cause problems.
8959 Clean them up now. */
8960 if (flag_exceptions
)
8963 bool changed
= false;
8965 FOR_EACH_BB_FN (bb
, cfun
)
8966 changed
|= gimple_purge_dead_eh_edges (bb
);
8968 cleanup_tree_cfg ();
8973 /* Emit a library call to launch the offloading region, or do data
8975 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8976 enum built_in_function start_ix
;
8977 location_t clause_loc
;
8979 switch (gimple_omp_target_kind (entry_stmt
))
8981 case GF_OMP_TARGET_KIND_REGION
:
8982 start_ix
= BUILT_IN_GOMP_TARGET
;
8984 case GF_OMP_TARGET_KIND_DATA
:
8985 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8987 case GF_OMP_TARGET_KIND_UPDATE
:
8988 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8990 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8991 start_ix
= BUILT_IN_GOACC_PARALLEL
;
8993 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8994 start_ix
= BUILT_IN_GOACC_KERNELS
;
8996 case GF_OMP_TARGET_KIND_OACC_DATA
:
8997 start_ix
= BUILT_IN_GOACC_DATA_START
;
8999 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9000 start_ix
= BUILT_IN_GOACC_UPDATE
;
9002 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9003 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
9009 clauses
= gimple_omp_target_clauses (entry_stmt
);
9011 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
9012 library choose) and there is no conditional. */
9014 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
9016 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
9018 cond
= OMP_CLAUSE_IF_EXPR (c
);
9020 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
9023 /* Even if we pass it to all library function calls, it is currently only
9024 defined/used for the OpenMP target ones. */
9025 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
9026 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
9027 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
);
9029 device
= OMP_CLAUSE_DEVICE_ID (c
);
9030 clause_loc
= OMP_CLAUSE_LOCATION (c
);
9033 clause_loc
= gimple_location (entry_stmt
);
9035 /* Ensure 'device' is of the correct type. */
9036 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
9038 /* If we found the clause 'if (cond)', build
9039 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
9042 cond
= gimple_boolify (cond
);
9044 basic_block cond_bb
, then_bb
, else_bb
;
9048 tmp_var
= create_tmp_var (TREE_TYPE (device
));
9050 e
= split_block (new_bb
, NULL
);
9053 gsi
= gsi_last_bb (new_bb
);
9055 e
= split_block (new_bb
, gsi_stmt (gsi
));
9061 then_bb
= create_empty_bb (cond_bb
);
9062 else_bb
= create_empty_bb (then_bb
);
9063 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
9064 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
9066 stmt
= gimple_build_cond_empty (cond
);
9067 gsi
= gsi_last_bb (cond_bb
);
9068 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9070 gsi
= gsi_start_bb (then_bb
);
9071 stmt
= gimple_build_assign (tmp_var
, device
);
9072 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9074 gsi
= gsi_start_bb (else_bb
);
9075 stmt
= gimple_build_assign (tmp_var
,
9076 build_int_cst (integer_type_node
,
9077 GOMP_DEVICE_HOST_FALLBACK
));
9078 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9080 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
9081 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
9082 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
9083 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
9084 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
9085 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
9090 gsi
= gsi_last_bb (new_bb
);
9091 t
= gimple_omp_target_data_arg (entry_stmt
);
9094 t1
= size_zero_node
;
9095 t2
= build_zero_cst (ptr_type_node
);
9101 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
9102 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
9103 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
9104 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
9105 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
9109 tree offload_table
= get_offload_symbol_decl ();
9111 /* The maximum number used by any start_ix, without varargs. */
9112 unsigned int argcnt
= 12;
9114 vec_alloc (args
, argcnt
);
9115 args
->quick_push (device
);
9117 args
->quick_push (build_fold_addr_expr (child_fn
));
9118 args
->quick_push (build_fold_addr_expr (offload_table
));
9119 args
->quick_push (t1
);
9120 args
->quick_push (t2
);
9121 args
->quick_push (t3
);
9122 args
->quick_push (t4
);
9125 case BUILT_IN_GOACC_DATA_START
:
9126 case BUILT_IN_GOMP_TARGET
:
9127 case BUILT_IN_GOMP_TARGET_DATA
:
9128 case BUILT_IN_GOMP_TARGET_UPDATE
:
9130 case BUILT_IN_GOACC_KERNELS
:
9131 case BUILT_IN_GOACC_PARALLEL
:
9133 tree t_num_gangs
, t_num_workers
, t_vector_length
;
9135 /* Default values for num_gangs, num_workers, and vector_length. */
9136 t_num_gangs
= t_num_workers
= t_vector_length
9137 = fold_convert_loc (gimple_location (entry_stmt
),
9138 integer_type_node
, integer_one_node
);
9139 /* ..., but if present, use the value specified by the respective
9140 clause, making sure that are of the correct type. */
9141 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
9143 t_num_gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9145 OMP_CLAUSE_NUM_GANGS_EXPR (c
));
9146 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_WORKERS
);
9148 t_num_workers
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9150 OMP_CLAUSE_NUM_WORKERS_EXPR (c
));
9151 c
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
9153 t_vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9155 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c
));
9156 args
->quick_push (t_num_gangs
);
9157 args
->quick_push (t_num_workers
);
9158 args
->quick_push (t_vector_length
);
9161 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9162 case BUILT_IN_GOACC_UPDATE
:
9167 /* Default values for t_async. */
9168 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
9170 build_int_cst (integer_type_node
,
9172 /* ..., but if present, use the value specified by the respective
9173 clause, making sure that is of the correct type. */
9174 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
9176 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9178 OMP_CLAUSE_ASYNC_EXPR (c
));
9180 args
->quick_push (t_async
);
9181 /* Save the index, and... */
9182 t_wait_idx
= args
->length ();
9183 /* ... push a default value. */
9184 args
->quick_push (fold_convert_loc (gimple_location (entry_stmt
),
9186 integer_zero_node
));
9187 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
9192 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
9194 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
9196 args
->safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9198 OMP_CLAUSE_WAIT_EXPR (c
)));
9203 /* Now that we know the number, replace the default value. */
9204 args
->ordered_remove (t_wait_idx
);
9205 args
->quick_insert (t_wait_idx
,
9206 fold_convert_loc (gimple_location (entry_stmt
),
9208 build_int_cst (integer_type_node
, n
)));
9216 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), *args
);
9218 gimple_set_location (g
, gimple_location (entry_stmt
));
9219 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
9223 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
9224 gsi_remove (&gsi
, true);
9229 gsi
= gsi_last_bb (region
->exit
);
9231 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
9232 gsi_remove (&gsi
, true);
9237 /* Expand the parallel region tree rooted at REGION. Expansion
9238 proceeds in depth-first order. Innermost regions are expanded
9239 first. This way, parallel regions that require a new function to
9240 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9241 internal dependencies in their body. */
9244 expand_omp (struct omp_region
*region
)
9248 location_t saved_location
;
9249 gimple inner_stmt
= NULL
;
9251 /* First, determine whether this is a combined parallel+workshare
9253 if (region
->type
== GIMPLE_OMP_PARALLEL
)
9254 determine_parallel_type (region
);
9256 if (region
->type
== GIMPLE_OMP_FOR
9257 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
9258 inner_stmt
= last_stmt (region
->inner
->entry
);
9261 expand_omp (region
->inner
);
9263 saved_location
= input_location
;
9264 if (gimple_has_location (last_stmt (region
->entry
)))
9265 input_location
= gimple_location (last_stmt (region
->entry
));
9267 switch (region
->type
)
9269 case GIMPLE_OMP_PARALLEL
:
9270 case GIMPLE_OMP_TASK
:
9271 expand_omp_taskreg (region
);
9274 case GIMPLE_OMP_FOR
:
9275 expand_omp_for (region
, inner_stmt
);
9278 case GIMPLE_OMP_SECTIONS
:
9279 expand_omp_sections (region
);
9282 case GIMPLE_OMP_SECTION
:
9283 /* Individual omp sections are handled together with their
9284 parent GIMPLE_OMP_SECTIONS region. */
9287 case GIMPLE_OMP_SINGLE
:
9288 expand_omp_single (region
);
9291 case GIMPLE_OMP_MASTER
:
9292 case GIMPLE_OMP_TASKGROUP
:
9293 case GIMPLE_OMP_ORDERED
:
9294 case GIMPLE_OMP_CRITICAL
:
9295 case GIMPLE_OMP_TEAMS
:
9296 expand_omp_synch (region
);
9299 case GIMPLE_OMP_ATOMIC_LOAD
:
9300 expand_omp_atomic (region
);
9303 case GIMPLE_OMP_TARGET
:
9304 expand_omp_target (region
);
9311 input_location
= saved_location
;
9312 region
= region
->next
;
9317 /* Helper for build_omp_regions. Scan the dominator tree starting at
9318 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9319 true, the function ends once a single tree is built (otherwise, whole
9320 forest of OMP constructs may be built). */
9323 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
9326 gimple_stmt_iterator gsi
;
9330 gsi
= gsi_last_bb (bb
);
9331 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
9333 struct omp_region
*region
;
9334 enum gimple_code code
;
9336 stmt
= gsi_stmt (gsi
);
9337 code
= gimple_code (stmt
);
9338 if (code
== GIMPLE_OMP_RETURN
)
9340 /* STMT is the return point out of region PARENT. Mark it
9341 as the exit point and make PARENT the immediately
9342 enclosing region. */
9343 gcc_assert (parent
);
9346 parent
= parent
->outer
;
9348 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
9350 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9351 GIMPLE_OMP_RETURN, but matches with
9352 GIMPLE_OMP_ATOMIC_LOAD. */
9353 gcc_assert (parent
);
9354 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
9357 parent
= parent
->outer
;
9359 else if (code
== GIMPLE_OMP_CONTINUE
)
9361 gcc_assert (parent
);
9364 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
9366 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9367 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9371 region
= new_omp_region (bb
, code
, parent
);
9373 if (code
== GIMPLE_OMP_TARGET
)
9375 switch (gimple_omp_target_kind (stmt
))
9377 case GF_OMP_TARGET_KIND_REGION
:
9378 case GF_OMP_TARGET_KIND_DATA
:
9379 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9380 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9381 case GF_OMP_TARGET_KIND_OACC_DATA
:
9383 case GF_OMP_TARGET_KIND_UPDATE
:
9384 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9385 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9386 /* ..., other than for those stand-alone directives... */
9393 /* ..., this directive becomes the parent for a new region. */
9399 if (single_tree
&& !parent
)
9402 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
9404 son
= next_dom_son (CDI_DOMINATORS
, son
))
9405 build_omp_regions_1 (son
, parent
, single_tree
);
9408 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9412 build_omp_regions_root (basic_block root
)
9414 gcc_assert (root_omp_region
== NULL
);
9415 build_omp_regions_1 (root
, NULL
, true);
9416 gcc_assert (root_omp_region
!= NULL
);
9419 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9422 omp_expand_local (basic_block head
)
9424 build_omp_regions_root (head
);
9425 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
9427 fprintf (dump_file
, "\nOMP region tree\n\n");
9428 dump_omp_region (dump_file
, root_omp_region
, 0);
9429 fprintf (dump_file
, "\n");
9432 remove_exit_barriers (root_omp_region
);
9433 expand_omp (root_omp_region
);
9435 free_omp_regions ();
9438 /* Scan the CFG and build a tree of OMP regions. Return the root of
9439 the OMP region tree. */
9442 build_omp_regions (void)
9444 gcc_assert (root_omp_region
== NULL
);
9445 calculate_dominance_info (CDI_DOMINATORS
);
9446 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
9449 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9452 execute_expand_omp (void)
9454 build_omp_regions ();
9456 if (!root_omp_region
)
9461 fprintf (dump_file
, "\nOMP region tree\n\n");
9462 dump_omp_region (dump_file
, root_omp_region
, 0);
9463 fprintf (dump_file
, "\n");
9466 remove_exit_barriers (root_omp_region
);
9468 expand_omp (root_omp_region
);
9470 cleanup_tree_cfg ();
9472 free_omp_regions ();
9477 /* OMP expansion -- the default pass, run before creation of SSA form. */
9481 const pass_data pass_data_expand_omp
=
9483 GIMPLE_PASS
, /* type */
9484 "ompexp", /* name */
9485 OPTGROUP_NONE
, /* optinfo_flags */
9486 TV_NONE
, /* tv_id */
9487 PROP_gimple_any
, /* properties_required */
9488 PROP_gimple_eomp
, /* properties_provided */
9489 0, /* properties_destroyed */
9490 0, /* todo_flags_start */
9491 0, /* todo_flags_finish */
9494 class pass_expand_omp
: public gimple_opt_pass
9497 pass_expand_omp (gcc::context
*ctxt
)
9498 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
9501 /* opt_pass methods: */
9502 virtual unsigned int execute (function
*)
9504 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
9505 || flag_openmp_simd
!= 0)
9508 /* This pass always runs, to provide PROP_gimple_eomp.
9509 But often, there is nothing to do. */
9513 return execute_expand_omp ();
9516 }; // class pass_expand_omp
9521 make_pass_expand_omp (gcc::context
*ctxt
)
9523 return new pass_expand_omp (ctxt
);
9528 const pass_data pass_data_expand_omp_ssa
=
9530 GIMPLE_PASS
, /* type */
9531 "ompexpssa", /* name */
9532 OPTGROUP_NONE
, /* optinfo_flags */
9533 TV_NONE
, /* tv_id */
9534 PROP_cfg
| PROP_ssa
, /* properties_required */
9535 PROP_gimple_eomp
, /* properties_provided */
9536 0, /* properties_destroyed */
9537 0, /* todo_flags_start */
9538 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
9541 class pass_expand_omp_ssa
: public gimple_opt_pass
9544 pass_expand_omp_ssa (gcc::context
*ctxt
)
9545 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
9548 /* opt_pass methods: */
9549 virtual bool gate (function
*fun
)
9551 return !(fun
->curr_properties
& PROP_gimple_eomp
);
9553 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
9555 }; // class pass_expand_omp_ssa
9560 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
9562 return new pass_expand_omp_ssa (ctxt
);
9565 /* Routines to lower OMP directives into OMP-GIMPLE. */
9567 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9568 convert it to gimple. */
9570 oacc_gimple_assign (tree dest
, tree_code op
, tree src
, gimple_seq
*seq
)
9574 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
9576 stmt
= gimple_build_assign (dest
, op
, dest
, src
);
9577 gimple_seq_add_stmt (seq
, stmt
);
9581 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9582 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9583 gimplify_assign (t
, rdest
, seq
);
9586 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9587 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9588 gimplify_assign (t
, idest
, seq
);
9591 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9592 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9593 gimplify_assign (t
, rsrc
, seq
);
9596 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9597 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9598 gimplify_assign (t
, isrc
, seq
);
9601 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9602 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9605 if (op
== PLUS_EXPR
)
9607 stmt
= gimple_build_assign (r
, op
, rdest
, rsrc
);
9608 gimple_seq_add_stmt (seq
, stmt
);
9610 stmt
= gimple_build_assign (i
, op
, idest
, isrc
);
9611 gimple_seq_add_stmt (seq
, stmt
);
9613 else if (op
== MULT_EXPR
)
9615 /* Let x = a + ib = dest, y = c + id = src.
9616 x * y = (ac - bd) + i(ad + bc) */
9617 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9618 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9619 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9620 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9622 stmt
= gimple_build_assign (ac
, MULT_EXPR
, rdest
, rsrc
);
9623 gimple_seq_add_stmt (seq
, stmt
);
9625 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, isrc
);
9626 gimple_seq_add_stmt (seq
, stmt
);
9628 stmt
= gimple_build_assign (r
, MINUS_EXPR
, ac
, bd
);
9629 gimple_seq_add_stmt (seq
, stmt
);
9631 stmt
= gimple_build_assign (ad
, MULT_EXPR
, rdest
, isrc
);
9632 gimple_seq_add_stmt (seq
, stmt
);
9634 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, rsrc
);
9635 gimple_seq_add_stmt (seq
, stmt
);
9637 stmt
= gimple_build_assign (i
, PLUS_EXPR
, ad
, bc
);
9638 gimple_seq_add_stmt (seq
, stmt
);
9643 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
9644 gimplify_assign (dest
, result
, seq
);
9647 /* Helper function to initialize local data for the reduction arrays.
9648 The reduction arrays need to be placed inside the calling function
9649 for accelerators, or else the host won't be able to preform the final
9653 oacc_initialize_reduction_data (tree clauses
, tree nthreads
,
9654 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9660 /* Find the innermost OpenACC parallel context. */
9661 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
9662 && (gimple_omp_target_kind (ctx
->stmt
)
9663 == GF_OMP_TARGET_KIND_OACC_PARALLEL
))
9667 gcc_checking_assert (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
9668 && (gimple_omp_target_kind (octx
->stmt
)
9669 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
9671 /* Extract the clauses. */
9672 oc
= gimple_omp_target_clauses (octx
->stmt
);
9674 /* Find the last outer clause. */
9675 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
9678 /* Allocate arrays for each reduction variable. */
9679 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9681 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9684 tree var
= OMP_CLAUSE_DECL (c
);
9685 tree type
= get_base_type (var
);
9686 tree array
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
),
9690 /* Calculate size of the reduction array. */
9691 t
= create_tmp_var (TREE_TYPE (nthreads
));
9692 stmt
= gimple_build_assign (t
, MULT_EXPR
, nthreads
,
9693 fold_convert (TREE_TYPE (nthreads
),
9694 TYPE_SIZE_UNIT (type
)));
9695 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9697 size
= create_tmp_var (sizetype
);
9698 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
9700 /* Now allocate memory for it. */
9701 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
9702 stmt
= gimple_build_call (call
, 1, size
);
9703 gimple_call_set_lhs (stmt
, array
);
9704 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9706 /* Map this array into the accelerator. */
9708 /* Add the reduction array to the list of clauses. */
9710 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
9711 OMP_CLAUSE_MAP_KIND (t
) = OMP_CLAUSE_MAP_FORCE_FROM
;
9712 OMP_CLAUSE_DECL (t
) = x
;
9713 OMP_CLAUSE_CHAIN (t
) = NULL
;
9715 OMP_CLAUSE_CHAIN (oc
) = t
;
9717 gimple_omp_target_set_clauses (as_a
<gomp_target
*> (octx
->stmt
), t
);
9718 OMP_CLAUSE_SIZE (t
) = size
;
9723 /* Helper function to process the array of partial reductions. Nthreads
9724 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9725 cannot be used here, because nthreads on the host may be different than
9726 on the accelerator. */
9729 oacc_finalize_reduction_data (tree clauses
, tree nthreads
,
9730 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9732 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
9737 let var = the original reduction variable
9738 let array = reduction variable array
9740 for (i = 0; i < nthreads; i++)
9744 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
9745 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
9746 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
9748 /* Create and initialize an index variable. */
9749 tree ix
= create_tmp_var (sizetype
);
9750 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
9753 /* Insert the loop header label here. */
9754 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
9756 /* Exit loop if ix >= nthreads. */
9757 x
= create_tmp_var (sizetype
);
9758 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
9759 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
9760 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9762 /* Insert the loop body label here. */
9763 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
9765 /* Collapse each reduction array, one element at a time. */
9766 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9768 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9771 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
9773 /* reduction(-:var) sums up the partial results, so it acts
9774 identically to reduction(+:var). */
9775 if (reduction_code
== MINUS_EXPR
)
9776 reduction_code
= PLUS_EXPR
;
9778 /* Set up reduction variable var. */
9779 var
= OMP_CLAUSE_DECL (c
);
9780 type
= get_base_type (var
);
9781 array
= lookup_oacc_reduction (oacc_get_reduction_array_id
9782 (OMP_CLAUSE_DECL (c
)), ctx
);
9784 /* Calculate the array offset. */
9785 tree offset
= create_tmp_var (sizetype
);
9786 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
9787 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, ix
);
9788 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9790 tree ptr
= create_tmp_var (TREE_TYPE (array
));
9791 stmt
= gimple_build_assign (ptr
, POINTER_PLUS_EXPR
, array
, offset
);
9792 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9794 /* Extract array[ix] into mem. */
9795 tree mem
= create_tmp_var (type
);
9796 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
9798 /* Find the original reduction variable. */
9799 if (is_reference (var
))
9800 var
= build_simple_mem_ref (var
);
9802 tree t
= create_tmp_var (type
);
9804 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
9805 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9807 /* var = var op mem */
9808 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
9810 case TRUTH_ANDIF_EXPR
:
9811 case TRUTH_ORIF_EXPR
:
9812 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
9814 gimplify_and_add (t
, stmt_seqp
);
9817 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9818 oacc_gimple_assign (t
, OMP_CLAUSE_REDUCTION_CODE (c
), mem
,
9822 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
9823 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
9824 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9827 /* Increment the induction variable. */
9828 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
9829 stmt
= gimple_build_assign (ix
, PLUS_EXPR
, ix
, one
);
9830 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9832 /* Go back to the top of the loop. */
9833 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
9835 /* Place the loop exit label here. */
9836 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
9839 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9840 scan that for reductions. */
9843 oacc_process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
9844 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
9846 gimple_stmt_iterator gsi
;
9847 gimple_seq inner
= NULL
;
9849 /* A collapse clause may have inserted a new bind block. */
9850 gsi
= gsi_start (*body
);
9851 while (!gsi_end_p (gsi
))
9853 gimple stmt
= gsi_stmt (gsi
);
9854 if (gbind
*bind_stmt
= dyn_cast
<gbind
*> (stmt
))
9856 inner
= gimple_bind_body (bind_stmt
);
9858 gsi
= gsi_start (*body
);
9860 else if (dyn_cast
<gomp_for
*> (stmt
))
9866 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9868 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
9870 bool reduction_found
= false;
9872 gimple stmt
= gsi_stmt (gsi
);
9874 switch (gimple_code (stmt
))
9876 case GIMPLE_OMP_FOR
:
9877 clauses
= gimple_omp_for_clauses (stmt
);
9879 /* Search for a reduction clause. */
9880 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9881 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
9883 reduction_found
= true;
9887 if (!reduction_found
)
9890 ctx
= maybe_lookup_ctx (stmt
);
9893 /* Extract the number of threads. */
9894 nthreads
= create_tmp_var (sizetype
);
9895 t
= oacc_max_threads (ctx
);
9896 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
9898 /* Determine if this is kernel will be executed on the host. */
9899 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
9900 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
9901 stmt
= gimple_build_call (call
, 0);
9902 gimple_call_set_lhs (stmt
, acc_device
);
9903 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9905 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9906 acc_device_host
= create_tmp_var (integer_type_node
,
9907 ".acc_device_host");
9908 gimplify_assign (acc_device_host
,
9909 build_int_cst (integer_type_node
,
9913 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9914 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9916 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9918 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9919 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9920 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9923 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9925 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9926 gimplify_assign (acc_device_host
,
9927 build_int_cst (integer_type_node
,
9928 GOMP_DEVICE_HOST_NONSHM
),
9931 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9932 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9934 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9936 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9937 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9938 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9941 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9943 oacc_initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
,
9945 oacc_finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
9948 // Scan for other directives which support reduction here.
9954 /* If ctx is a worksharing context inside of a cancellable parallel
9955 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
9956 and conditional branch to parallel's cancel_label to handle
9957 cancellation in the implicit barrier. */
9960 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
9962 gimple omp_return
= gimple_seq_last_stmt (*body
);
9963 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
9964 if (gimple_omp_return_nowait_p (omp_return
))
9967 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
9968 && ctx
->outer
->cancellable
)
9970 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
9971 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
9972 tree lhs
= create_tmp_var (c_bool_type
);
9973 gimple_omp_return_set_lhs (omp_return
, lhs
);
9974 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
9975 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
9976 fold_convert (c_bool_type
,
9977 boolean_false_node
),
9978 ctx
->outer
->cancel_label
, fallthru_label
);
9979 gimple_seq_add_stmt (body
, g
);
9980 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
9984 /* Lower the OpenMP sections directive in the current statement in GSI_P.
9985 CTX is the enclosing OMP context for the current statement. */
9988 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9990 tree block
, control
;
9991 gimple_stmt_iterator tgsi
;
9992 gomp_sections
*stmt
;
9994 gbind
*new_stmt
, *bind
;
9995 gimple_seq ilist
, dlist
, olist
, new_body
;
9997 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
9999 push_gimplify_context ();
10003 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
10004 &ilist
, &dlist
, ctx
, NULL
);
10006 new_body
= gimple_omp_body (stmt
);
10007 gimple_omp_set_body (stmt
, NULL
);
10008 tgsi
= gsi_start (new_body
);
10009 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
10014 sec_start
= gsi_stmt (tgsi
);
10015 sctx
= maybe_lookup_ctx (sec_start
);
10018 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
10019 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
10020 GSI_CONTINUE_LINKING
);
10021 gimple_omp_set_body (sec_start
, NULL
);
10023 if (gsi_one_before_end_p (tgsi
))
10025 gimple_seq l
= NULL
;
10026 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
10028 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
10029 gimple_omp_section_set_last (sec_start
);
10032 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
10033 GSI_CONTINUE_LINKING
);
10036 block
= make_node (BLOCK
);
10037 bind
= gimple_build_bind (NULL
, new_body
, block
);
10040 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
10042 block
= make_node (BLOCK
);
10043 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10044 gsi_replace (gsi_p
, new_stmt
, true);
10046 pop_gimplify_context (new_stmt
);
10047 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10048 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10049 if (BLOCK_VARS (block
))
10050 TREE_USED (block
) = 1;
10053 gimple_seq_add_seq (&new_body
, ilist
);
10054 gimple_seq_add_stmt (&new_body
, stmt
);
10055 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
10056 gimple_seq_add_stmt (&new_body
, bind
);
10058 control
= create_tmp_var (unsigned_type_node
, ".section");
10059 t
= gimple_build_omp_continue (control
, control
);
10060 gimple_omp_sections_set_control (stmt
, control
);
10061 gimple_seq_add_stmt (&new_body
, t
);
10063 gimple_seq_add_seq (&new_body
, olist
);
10064 if (ctx
->cancellable
)
10065 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10066 gimple_seq_add_seq (&new_body
, dlist
);
10068 new_body
= maybe_catch_exception (new_body
);
10070 t
= gimple_build_omp_return
10071 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
10072 OMP_CLAUSE_NOWAIT
));
10073 gimple_seq_add_stmt (&new_body
, t
);
10074 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
10076 gimple_bind_set_body (new_stmt
, new_body
);
10080 /* A subroutine of lower_omp_single. Expand the simple form of
10081 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10083 if (GOMP_single_start ())
10085 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10087 FIXME. It may be better to delay expanding the logic of this until
10088 pass_expand_omp. The expanded logic may make the job more difficult
10089 to a synchronization analysis pass. */
10092 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
10094 location_t loc
= gimple_location (single_stmt
);
10095 tree tlabel
= create_artificial_label (loc
);
10096 tree flabel
= create_artificial_label (loc
);
10100 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
10101 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
10102 call
= gimple_build_call (decl
, 0);
10103 gimple_call_set_lhs (call
, lhs
);
10104 gimple_seq_add_stmt (pre_p
, call
);
10106 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
10107 fold_convert_loc (loc
, TREE_TYPE (lhs
),
10108 boolean_true_node
),
10110 gimple_seq_add_stmt (pre_p
, cond
);
10111 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
10112 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10113 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
10117 /* A subroutine of lower_omp_single. Expand the simple form of
10118 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10120 #pragma omp single copyprivate (a, b, c)
10122 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10125 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10131 GOMP_single_copy_end (©out);
10142 FIXME. It may be better to delay expanding the logic of this until
10143 pass_expand_omp. The expanded logic may make the job more difficult
10144 to a synchronization analysis pass. */
10147 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
10150 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
10151 gimple_seq copyin_seq
;
10152 location_t loc
= gimple_location (single_stmt
);
10154 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
10156 ptr_type
= build_pointer_type (ctx
->record_type
);
10157 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
10159 l0
= create_artificial_label (loc
);
10160 l1
= create_artificial_label (loc
);
10161 l2
= create_artificial_label (loc
);
10163 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
10164 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
10165 t
= fold_convert_loc (loc
, ptr_type
, t
);
10166 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
10168 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
10169 build_int_cst (ptr_type
, 0));
10170 t
= build3 (COND_EXPR
, void_type_node
, t
,
10171 build_and_jump (&l0
), build_and_jump (&l1
));
10172 gimplify_and_add (t
, pre_p
);
10174 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
10176 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10179 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
10182 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10183 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
10184 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
10185 gimplify_and_add (t
, pre_p
);
10187 t
= build_and_jump (&l2
);
10188 gimplify_and_add (t
, pre_p
);
10190 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
10192 gimple_seq_add_seq (pre_p
, copyin_seq
);
10194 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
10198 /* Expand code for an OpenMP single directive. */
10201 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10205 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
10207 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
10209 push_gimplify_context ();
10211 block
= make_node (BLOCK
);
10212 bind
= gimple_build_bind (NULL
, NULL
, block
);
10213 gsi_replace (gsi_p
, bind
, true);
10216 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
10217 &bind_body
, &dlist
, ctx
, NULL
);
10218 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
10220 gimple_seq_add_stmt (&bind_body
, single_stmt
);
10222 if (ctx
->record_type
)
10223 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
10225 lower_omp_single_simple (single_stmt
, &bind_body
);
10227 gimple_omp_set_body (single_stmt
, NULL
);
10229 gimple_seq_add_seq (&bind_body
, dlist
);
10231 bind_body
= maybe_catch_exception (bind_body
);
10233 t
= gimple_build_omp_return
10234 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
10235 OMP_CLAUSE_NOWAIT
));
10236 gimple_seq_add_stmt (&bind_body_tail
, t
);
10237 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
10238 if (ctx
->record_type
)
10240 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
10241 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10242 TREE_THIS_VOLATILE (clobber
) = 1;
10243 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
10244 clobber
), GSI_SAME_STMT
);
10246 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
10247 gimple_bind_set_body (bind
, bind_body
);
10249 pop_gimplify_context (bind
);
10251 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10252 BLOCK_VARS (block
) = ctx
->block_vars
;
10253 if (BLOCK_VARS (block
))
10254 TREE_USED (block
) = 1;
10258 /* Expand code for an OpenMP master directive. */
10261 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10263 tree block
, lab
= NULL
, x
, bfn_decl
;
10264 gimple stmt
= gsi_stmt (*gsi_p
);
10266 location_t loc
= gimple_location (stmt
);
10269 push_gimplify_context ();
10271 block
= make_node (BLOCK
);
10272 bind
= gimple_build_bind (NULL
, NULL
, block
);
10273 gsi_replace (gsi_p
, bind
, true);
10274 gimple_bind_add_stmt (bind
, stmt
);
10276 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
10277 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
10278 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
10279 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
10281 gimplify_and_add (x
, &tseq
);
10282 gimple_bind_add_seq (bind
, tseq
);
10284 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10285 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10286 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10287 gimple_omp_set_body (stmt
, NULL
);
10289 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
10291 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10293 pop_gimplify_context (bind
);
10295 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10296 BLOCK_VARS (block
) = ctx
->block_vars
;
10300 /* Expand code for an OpenMP taskgroup directive. */
10303 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10305 gimple stmt
= gsi_stmt (*gsi_p
);
10308 tree block
= make_node (BLOCK
);
10310 bind
= gimple_build_bind (NULL
, NULL
, block
);
10311 gsi_replace (gsi_p
, bind
, true);
10312 gimple_bind_add_stmt (bind
, stmt
);
10314 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
10316 gimple_bind_add_stmt (bind
, x
);
10318 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10319 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10320 gimple_omp_set_body (stmt
, NULL
);
10322 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10324 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10325 BLOCK_VARS (block
) = ctx
->block_vars
;
10329 /* Expand code for an OpenMP ordered directive. */
10332 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10335 gimple stmt
= gsi_stmt (*gsi_p
);
10339 push_gimplify_context ();
10341 block
= make_node (BLOCK
);
10342 bind
= gimple_build_bind (NULL
, NULL
, block
);
10343 gsi_replace (gsi_p
, bind
, true);
10344 gimple_bind_add_stmt (bind
, stmt
);
10346 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
10348 gimple_bind_add_stmt (bind
, x
);
10350 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10351 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10352 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10353 gimple_omp_set_body (stmt
, NULL
);
10355 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
10356 gimple_bind_add_stmt (bind
, x
);
10358 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10360 pop_gimplify_context (bind
);
10362 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10363 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10367 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10368 substitution of a couple of function calls. But in the NAMED case,
10369 requires that languages coordinate a symbol name. It is therefore
10370 best put here in common code. */
10372 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
10375 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10378 tree name
, lock
, unlock
;
10379 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
10381 location_t loc
= gimple_location (stmt
);
10384 name
= gimple_omp_critical_name (stmt
);
10389 if (!critical_name_mutexes
)
10390 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
10392 tree
*n
= critical_name_mutexes
->get (name
);
10397 decl
= create_tmp_var_raw (ptr_type_node
);
10399 new_str
= ACONCAT ((".gomp_critical_user_",
10400 IDENTIFIER_POINTER (name
), NULL
));
10401 DECL_NAME (decl
) = get_identifier (new_str
);
10402 TREE_PUBLIC (decl
) = 1;
10403 TREE_STATIC (decl
) = 1;
10404 DECL_COMMON (decl
) = 1;
10405 DECL_ARTIFICIAL (decl
) = 1;
10406 DECL_IGNORED_P (decl
) = 1;
10408 varpool_node::finalize_decl (decl
);
10410 critical_name_mutexes
->put (name
, decl
);
10415 /* If '#pragma omp critical' is inside offloaded region or
10416 inside function marked as offloadable, the symbol must be
10417 marked as offloadable too. */
10419 if (cgraph_node::get (current_function_decl
)->offloadable
)
10420 varpool_node::get_create (decl
)->offloadable
= 1;
10422 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
10423 if (is_gimple_omp_offloaded (octx
->stmt
))
10425 varpool_node::get_create (decl
)->offloadable
= 1;
10429 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
10430 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
10432 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
10433 unlock
= build_call_expr_loc (loc
, unlock
, 1,
10434 build_fold_addr_expr_loc (loc
, decl
));
10438 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
10439 lock
= build_call_expr_loc (loc
, lock
, 0);
10441 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
10442 unlock
= build_call_expr_loc (loc
, unlock
, 0);
10445 push_gimplify_context ();
10447 block
= make_node (BLOCK
);
10448 bind
= gimple_build_bind (NULL
, NULL
, block
);
10449 gsi_replace (gsi_p
, bind
, true);
10450 gimple_bind_add_stmt (bind
, stmt
);
10452 tbody
= gimple_bind_body (bind
);
10453 gimplify_and_add (lock
, &tbody
);
10454 gimple_bind_set_body (bind
, tbody
);
10456 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10457 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10458 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10459 gimple_omp_set_body (stmt
, NULL
);
10461 tbody
= gimple_bind_body (bind
);
10462 gimplify_and_add (unlock
, &tbody
);
10463 gimple_bind_set_body (bind
, tbody
);
10465 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10467 pop_gimplify_context (bind
);
10468 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10469 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10473 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10474 for a lastprivate clause. Given a loop control predicate of (V
10475 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10476 is appended to *DLIST, iterator initialization is appended to
10480 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
10481 gimple_seq
*dlist
, struct omp_context
*ctx
)
10483 tree clauses
, cond
, vinit
;
10484 enum tree_code cond_code
;
10487 cond_code
= fd
->loop
.cond_code
;
10488 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
10490 /* When possible, use a strict equality expression. This can let VRP
10491 type optimizations deduce the value and remove a copy. */
10492 if (tree_fits_shwi_p (fd
->loop
.step
))
10494 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
10495 if (step
== 1 || step
== -1)
10496 cond_code
= EQ_EXPR
;
10499 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
10501 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
10503 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
10504 if (!gimple_seq_empty_p (stmts
))
10506 gimple_seq_add_seq (&stmts
, *dlist
);
10509 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10510 vinit
= fd
->loop
.n1
;
10511 if (cond_code
== EQ_EXPR
10512 && tree_fits_shwi_p (fd
->loop
.n2
)
10513 && ! integer_zerop (fd
->loop
.n2
))
10514 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
10516 vinit
= unshare_expr (vinit
);
10518 /* Initialize the iterator variable, so that threads that don't execute
10519 any iterations don't execute the lastprivate clauses by accident. */
10520 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
10525 /* Lower code for an OMP loop directive. */
10528 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10530 tree
*rhs_p
, block
;
10531 struct omp_for_data fd
, *fdp
= NULL
;
10532 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
10534 gimple_seq omp_for_body
, body
, dlist
;
10537 push_gimplify_context ();
10539 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
10541 block
= make_node (BLOCK
);
10542 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10543 /* Replace at gsi right away, so that 'stmt' is no member
10544 of a sequence anymore as we're going to add to to a different
10546 gsi_replace (gsi_p
, new_stmt
, true);
10548 /* Move declaration of temporaries in the loop body before we make
10550 omp_for_body
= gimple_omp_body (stmt
);
10551 if (!gimple_seq_empty_p (omp_for_body
)
10552 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
10555 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
10556 tree vars
= gimple_bind_vars (inner_bind
);
10557 gimple_bind_append_vars (new_stmt
, vars
);
10558 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10559 keep them on the inner_bind and it's block. */
10560 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
10561 if (gimple_bind_block (inner_bind
))
10562 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
10565 if (gimple_omp_for_combined_into_p (stmt
))
10567 extract_omp_for_data (stmt
, &fd
, NULL
);
10570 /* We need two temporaries with fd.loop.v type (istart/iend)
10571 and then (fd.collapse - 1) temporaries with the same
10572 type for count2 ... countN-1 vars if not constant. */
10574 tree type
= fd
.iter_type
;
10575 if (fd
.collapse
> 1
10576 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
10577 count
+= fd
.collapse
- 1;
10578 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
10579 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
10580 tree clauses
= *pc
;
10583 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
10584 OMP_CLAUSE__LOOPTEMP_
);
10585 for (i
= 0; i
< count
; i
++)
10590 gcc_assert (outerc
);
10591 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
10592 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
10593 OMP_CLAUSE__LOOPTEMP_
);
10597 temp
= create_tmp_var (type
);
10598 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
10600 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
10601 OMP_CLAUSE_DECL (*pc
) = temp
;
10602 pc
= &OMP_CLAUSE_CHAIN (*pc
);
10607 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10610 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
10612 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
10614 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10616 /* Lower the header expressions. At this point, we can assume that
10617 the header is of the form:
10619 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10621 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10622 using the .omp_data_s mapping, if needed. */
10623 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
10625 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
10626 if (!is_gimple_min_invariant (*rhs_p
))
10627 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10629 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
10630 if (!is_gimple_min_invariant (*rhs_p
))
10631 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10633 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
10634 if (!is_gimple_min_invariant (*rhs_p
))
10635 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10638 /* Once lowered, extract the bounds and clauses. */
10639 extract_omp_for_data (stmt
, &fd
, NULL
);
10641 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
10643 gimple_seq_add_stmt (&body
, stmt
);
10644 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
10646 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
10649 /* After the loop, add exit clauses. */
10650 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
10652 if (ctx
->cancellable
)
10653 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
10655 gimple_seq_add_seq (&body
, dlist
);
10657 body
= maybe_catch_exception (body
);
10659 /* Region exit marker goes at the end of the loop body. */
10660 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
10661 maybe_add_implicit_barrier_cancel (ctx
, &body
);
10662 pop_gimplify_context (new_stmt
);
10664 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10665 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
10666 if (BLOCK_VARS (block
))
10667 TREE_USED (block
) = 1;
10669 gimple_bind_set_body (new_stmt
, body
);
10670 gimple_omp_set_body (stmt
, NULL
);
10671 gimple_omp_for_set_pre_body (stmt
, NULL
);
10674 /* Callback for walk_stmts. Check if the current statement only contains
10675 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10678 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
10679 bool *handled_ops_p
,
10680 struct walk_stmt_info
*wi
)
10682 int *info
= (int *) wi
->info
;
10683 gimple stmt
= gsi_stmt (*gsi_p
);
10685 *handled_ops_p
= true;
10686 switch (gimple_code (stmt
))
10690 case GIMPLE_OMP_FOR
:
10691 case GIMPLE_OMP_SECTIONS
:
10692 *info
= *info
== 0 ? 1 : -1;
10701 struct omp_taskcopy_context
10703 /* This field must be at the beginning, as we do "inheritance": Some
10704 callback functions for tree-inline.c (e.g., omp_copy_decl)
10705 receive a copy_body_data pointer that is up-casted to an
10706 omp_context pointer. */
10712 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
10714 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
10716 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
10717 return create_tmp_var (TREE_TYPE (var
));
10723 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
10725 tree name
, new_fields
= NULL
, type
, f
;
10727 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
10728 name
= DECL_NAME (TYPE_NAME (orig_type
));
10729 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
10730 TYPE_DECL
, name
, type
);
10731 TYPE_NAME (type
) = name
;
10733 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
10735 tree new_f
= copy_node (f
);
10736 DECL_CONTEXT (new_f
) = type
;
10737 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
10738 TREE_CHAIN (new_f
) = new_fields
;
10739 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10740 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10741 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
10743 new_fields
= new_f
;
10744 tcctx
->cb
.decl_map
->put (f
, new_f
);
10746 TYPE_FIELDS (type
) = nreverse (new_fields
);
10747 layout_type (type
);
10751 /* Create task copyfn. */
10754 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
10756 struct function
*child_cfun
;
10757 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
10758 tree record_type
, srecord_type
, bind
, list
;
10759 bool record_needs_remap
= false, srecord_needs_remap
= false;
10761 struct omp_taskcopy_context tcctx
;
10762 location_t loc
= gimple_location (task_stmt
);
10764 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
10765 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
10766 gcc_assert (child_cfun
->cfg
== NULL
);
10767 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
10769 /* Reset DECL_CONTEXT on function arguments. */
10770 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
10771 DECL_CONTEXT (t
) = child_fn
;
10773 /* Populate the function. */
10774 push_gimplify_context ();
10775 push_cfun (child_cfun
);
10777 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
10778 TREE_SIDE_EFFECTS (bind
) = 1;
10780 DECL_SAVED_TREE (child_fn
) = bind
;
10781 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
10783 /* Remap src and dst argument types if needed. */
10784 record_type
= ctx
->record_type
;
10785 srecord_type
= ctx
->srecord_type
;
10786 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
10787 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10789 record_needs_remap
= true;
10792 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
10793 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10795 srecord_needs_remap
= true;
10799 if (record_needs_remap
|| srecord_needs_remap
)
10801 memset (&tcctx
, '\0', sizeof (tcctx
));
10802 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
10803 tcctx
.cb
.dst_fn
= child_fn
;
10804 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
10805 gcc_checking_assert (tcctx
.cb
.src_node
);
10806 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
10807 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
10808 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
10809 tcctx
.cb
.eh_lp_nr
= 0;
10810 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
10811 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
10814 if (record_needs_remap
)
10815 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
10816 if (srecord_needs_remap
)
10817 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
10820 tcctx
.cb
.decl_map
= NULL
;
10822 arg
= DECL_ARGUMENTS (child_fn
);
10823 TREE_TYPE (arg
) = build_pointer_type (record_type
);
10824 sarg
= DECL_CHAIN (arg
);
10825 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
10827 /* First pass: initialize temporaries used in record_type and srecord_type
10828 sizes and field offsets. */
10829 if (tcctx
.cb
.decl_map
)
10830 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10831 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10835 decl
= OMP_CLAUSE_DECL (c
);
10836 p
= tcctx
.cb
.decl_map
->get (decl
);
10839 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10840 sf
= (tree
) n
->value
;
10841 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10842 src
= build_simple_mem_ref_loc (loc
, sarg
);
10843 src
= omp_build_component_ref (src
, sf
);
10844 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
10845 append_to_statement_list (t
, &list
);
10848 /* Second pass: copy shared var pointers and copy construct non-VLA
10849 firstprivate vars. */
10850 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10851 switch (OMP_CLAUSE_CODE (c
))
10853 case OMP_CLAUSE_SHARED
:
10854 decl
= OMP_CLAUSE_DECL (c
);
10855 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10858 f
= (tree
) n
->value
;
10859 if (tcctx
.cb
.decl_map
)
10860 f
= *tcctx
.cb
.decl_map
->get (f
);
10861 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10862 sf
= (tree
) n
->value
;
10863 if (tcctx
.cb
.decl_map
)
10864 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10865 src
= build_simple_mem_ref_loc (loc
, sarg
);
10866 src
= omp_build_component_ref (src
, sf
);
10867 dst
= build_simple_mem_ref_loc (loc
, arg
);
10868 dst
= omp_build_component_ref (dst
, f
);
10869 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10870 append_to_statement_list (t
, &list
);
10872 case OMP_CLAUSE_FIRSTPRIVATE
:
10873 decl
= OMP_CLAUSE_DECL (c
);
10874 if (is_variable_sized (decl
))
10876 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10879 f
= (tree
) n
->value
;
10880 if (tcctx
.cb
.decl_map
)
10881 f
= *tcctx
.cb
.decl_map
->get (f
);
10882 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10885 sf
= (tree
) n
->value
;
10886 if (tcctx
.cb
.decl_map
)
10887 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10888 src
= build_simple_mem_ref_loc (loc
, sarg
);
10889 src
= omp_build_component_ref (src
, sf
);
10890 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
10891 src
= build_simple_mem_ref_loc (loc
, src
);
10895 dst
= build_simple_mem_ref_loc (loc
, arg
);
10896 dst
= omp_build_component_ref (dst
, f
);
10897 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10898 append_to_statement_list (t
, &list
);
10900 case OMP_CLAUSE_PRIVATE
:
10901 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
10903 decl
= OMP_CLAUSE_DECL (c
);
10904 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10905 f
= (tree
) n
->value
;
10906 if (tcctx
.cb
.decl_map
)
10907 f
= *tcctx
.cb
.decl_map
->get (f
);
10908 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10911 sf
= (tree
) n
->value
;
10912 if (tcctx
.cb
.decl_map
)
10913 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10914 src
= build_simple_mem_ref_loc (loc
, sarg
);
10915 src
= omp_build_component_ref (src
, sf
);
10916 if (use_pointer_for_field (decl
, NULL
))
10917 src
= build_simple_mem_ref_loc (loc
, src
);
10921 dst
= build_simple_mem_ref_loc (loc
, arg
);
10922 dst
= omp_build_component_ref (dst
, f
);
10923 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10924 append_to_statement_list (t
, &list
);
10930 /* Last pass: handle VLA firstprivates. */
10931 if (tcctx
.cb
.decl_map
)
10932 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10933 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10937 decl
= OMP_CLAUSE_DECL (c
);
10938 if (!is_variable_sized (decl
))
10940 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10943 f
= (tree
) n
->value
;
10944 f
= *tcctx
.cb
.decl_map
->get (f
);
10945 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
10946 ind
= DECL_VALUE_EXPR (decl
);
10947 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
10948 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
10949 n
= splay_tree_lookup (ctx
->sfield_map
,
10950 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10951 sf
= (tree
) n
->value
;
10952 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10953 src
= build_simple_mem_ref_loc (loc
, sarg
);
10954 src
= omp_build_component_ref (src
, sf
);
10955 src
= build_simple_mem_ref_loc (loc
, src
);
10956 dst
= build_simple_mem_ref_loc (loc
, arg
);
10957 dst
= omp_build_component_ref (dst
, f
);
10958 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10959 append_to_statement_list (t
, &list
);
10960 n
= splay_tree_lookup (ctx
->field_map
,
10961 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10962 df
= (tree
) n
->value
;
10963 df
= *tcctx
.cb
.decl_map
->get (df
);
10964 ptr
= build_simple_mem_ref_loc (loc
, arg
);
10965 ptr
= omp_build_component_ref (ptr
, df
);
10966 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
10967 build_fold_addr_expr_loc (loc
, dst
));
10968 append_to_statement_list (t
, &list
);
10971 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
10972 append_to_statement_list (t
, &list
);
10974 if (tcctx
.cb
.decl_map
)
10975 delete tcctx
.cb
.decl_map
;
10976 pop_gimplify_context (NULL
);
10977 BIND_EXPR_BODY (bind
) = list
;
10982 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
10986 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
10988 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
10989 OMP_CLAUSE_DEPEND
);
10990 gcc_assert (clauses
);
10991 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10992 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
10993 switch (OMP_CLAUSE_DEPEND_KIND (c
))
10995 case OMP_CLAUSE_DEPEND_IN
:
10998 case OMP_CLAUSE_DEPEND_OUT
:
10999 case OMP_CLAUSE_DEPEND_INOUT
:
11003 gcc_unreachable ();
11005 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
11006 tree array
= create_tmp_var (type
);
11007 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
11009 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
11010 gimple_seq_add_stmt (iseq
, g
);
11011 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
11013 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
11014 gimple_seq_add_stmt (iseq
, g
);
11015 for (i
= 0; i
< 2; i
++)
11017 if ((i
? n_in
: n_out
) == 0)
11019 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11020 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
11021 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
11023 tree t
= OMP_CLAUSE_DECL (c
);
11024 t
= fold_convert (ptr_type_node
, t
);
11025 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
11026 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
11027 NULL_TREE
, NULL_TREE
);
11028 g
= gimple_build_assign (r
, t
);
11029 gimple_seq_add_stmt (iseq
, g
);
11032 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
11033 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
11034 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
11035 OMP_CLAUSE_CHAIN (c
) = *p
;
11037 tree clobber
= build_constructor (type
, NULL
);
11038 TREE_THIS_VOLATILE (clobber
) = 1;
11039 g
= gimple_build_assign (array
, clobber
);
11040 gimple_seq_add_stmt (oseq
, g
);
11043 /* Lower the OpenMP parallel or task directive in the current statement
11044 in GSI_P. CTX holds context information for the directive. */
11047 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11051 gimple stmt
= gsi_stmt (*gsi_p
);
11052 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
11053 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
11054 location_t loc
= gimple_location (stmt
);
11056 clauses
= gimple_omp_taskreg_clauses (stmt
);
11058 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
11059 par_body
= gimple_bind_body (par_bind
);
11060 child_fn
= ctx
->cb
.dst_fn
;
11061 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11062 && !gimple_omp_parallel_combined_p (stmt
))
11064 struct walk_stmt_info wi
;
11067 memset (&wi
, 0, sizeof (wi
));
11069 wi
.val_only
= true;
11070 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
11072 gimple_omp_parallel_set_combined_p (stmt
, true);
11074 gimple_seq dep_ilist
= NULL
;
11075 gimple_seq dep_olist
= NULL
;
11076 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11077 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
11079 push_gimplify_context ();
11080 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11081 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
11084 if (ctx
->srecord_type
)
11085 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
11087 push_gimplify_context ();
11092 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
11093 lower_omp (&par_body
, ctx
);
11094 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
11095 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
11097 /* Declare all the variables created by mapping and the variables
11098 declared in the scope of the parallel body. */
11099 record_vars_into (ctx
->block_vars
, child_fn
);
11100 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
11102 if (ctx
->record_type
)
11105 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
11106 : ctx
->record_type
, ".omp_data_o");
11107 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11108 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11109 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
11114 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
11115 lower_send_shared_vars (&ilist
, &olist
, ctx
);
11117 if (ctx
->record_type
)
11119 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
11120 TREE_THIS_VOLATILE (clobber
) = 1;
11121 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11125 /* Once all the expansions are done, sequence all the different
11126 fragments inside gimple_omp_body. */
11130 if (ctx
->record_type
)
11132 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11133 /* fixup_child_record_type might have changed receiver_decl's type. */
11134 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11135 gimple_seq_add_stmt (&new_body
,
11136 gimple_build_assign (ctx
->receiver_decl
, t
));
11139 gimple_seq_add_seq (&new_body
, par_ilist
);
11140 gimple_seq_add_seq (&new_body
, par_body
);
11141 gimple_seq_add_seq (&new_body
, par_rlist
);
11142 if (ctx
->cancellable
)
11143 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
11144 gimple_seq_add_seq (&new_body
, par_olist
);
11145 new_body
= maybe_catch_exception (new_body
);
11146 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11147 gimple_omp_set_body (stmt
, new_body
);
11149 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
11150 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
11151 gimple_bind_add_seq (bind
, ilist
);
11152 gimple_bind_add_stmt (bind
, stmt
);
11153 gimple_bind_add_seq (bind
, olist
);
11155 pop_gimplify_context (NULL
);
11159 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11160 gimple_bind_add_stmt (dep_bind
, bind
);
11161 gimple_bind_add_seq (dep_bind
, dep_olist
);
11162 pop_gimplify_context (dep_bind
);
11166 /* Lower the GIMPLE_OMP_TARGET in the current statement
11167 in GSI_P. CTX holds context information for the directive. */
11170 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11173 tree child_fn
, t
, c
;
11174 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
11175 gbind
*tgt_bind
, *bind
;
11176 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
11177 location_t loc
= gimple_location (stmt
);
11178 bool offloaded
, data_region
;
11179 unsigned int map_cnt
= 0;
11181 offloaded
= is_gimple_omp_offloaded (stmt
);
11182 switch (gimple_omp_target_kind (stmt
))
11184 case GF_OMP_TARGET_KIND_REGION
:
11185 case GF_OMP_TARGET_KIND_UPDATE
:
11186 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11187 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11188 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11189 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11190 data_region
= false;
11192 case GF_OMP_TARGET_KIND_DATA
:
11193 case GF_OMP_TARGET_KIND_OACC_DATA
:
11194 data_region
= true;
11197 gcc_unreachable ();
11200 clauses
= gimple_omp_target_clauses (stmt
);
11206 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
11207 tgt_body
= gimple_bind_body (tgt_bind
);
11209 else if (data_region
)
11210 tgt_body
= gimple_omp_body (stmt
);
11211 child_fn
= ctx
->cb
.dst_fn
;
11213 push_gimplify_context ();
11218 && is_gimple_omp_oacc (stmt
))
11219 oacc_process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
11221 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11222 switch (OMP_CLAUSE_CODE (c
))
11228 case OMP_CLAUSE_MAP
:
11229 #ifdef ENABLE_CHECKING
11230 /* First check what we're prepared to handle in the following. */
11231 switch (OMP_CLAUSE_MAP_KIND (c
))
11233 case OMP_CLAUSE_MAP_ALLOC
:
11234 case OMP_CLAUSE_MAP_TO
:
11235 case OMP_CLAUSE_MAP_FROM
:
11236 case OMP_CLAUSE_MAP_TOFROM
:
11237 case OMP_CLAUSE_MAP_POINTER
:
11238 case OMP_CLAUSE_MAP_TO_PSET
:
11240 case OMP_CLAUSE_MAP_FORCE_ALLOC
:
11241 case OMP_CLAUSE_MAP_FORCE_TO
:
11242 case OMP_CLAUSE_MAP_FORCE_FROM
:
11243 case OMP_CLAUSE_MAP_FORCE_TOFROM
:
11244 case OMP_CLAUSE_MAP_FORCE_PRESENT
:
11245 case OMP_CLAUSE_MAP_FORCE_DEALLOC
:
11246 case OMP_CLAUSE_MAP_FORCE_DEVICEPTR
:
11247 gcc_assert (is_gimple_omp_oacc (stmt
));
11250 gcc_unreachable ();
11254 case OMP_CLAUSE_TO
:
11255 case OMP_CLAUSE_FROM
:
11256 var
= OMP_CLAUSE_DECL (c
);
11259 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11260 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11265 if (DECL_SIZE (var
)
11266 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
11268 tree var2
= DECL_VALUE_EXPR (var
);
11269 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
11270 var2
= TREE_OPERAND (var2
, 0);
11271 gcc_assert (DECL_P (var2
));
11275 if (!maybe_lookup_field (var
, ctx
))
11280 x
= build_receiver_ref (var
, true, ctx
);
11281 tree new_var
= lookup_decl (var
, ctx
);
11282 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
11283 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11284 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11285 x
= build_simple_mem_ref (x
);
11286 SET_DECL_VALUE_EXPR (new_var
, x
);
11287 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11294 target_nesting_level
++;
11295 lower_omp (&tgt_body
, ctx
);
11296 target_nesting_level
--;
11298 else if (data_region
)
11299 lower_omp (&tgt_body
, ctx
);
11303 /* Declare all the variables created by mapping and the variables
11304 declared in the scope of the target body. */
11305 record_vars_into (ctx
->block_vars
, child_fn
);
11306 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
11311 if (ctx
->record_type
)
11314 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
11315 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11316 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11317 t
= make_tree_vec (3);
11318 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
11319 TREE_VEC_ELT (t
, 1)
11320 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
11321 ".omp_data_sizes");
11322 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
11323 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
11324 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
11327 if (is_gimple_omp_oacc (stmt
))
11329 tkind_type
= short_unsigned_type_node
;
11334 tkind_type
= unsigned_char_type_node
;
11337 TREE_VEC_ELT (t
, 2)
11338 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
11339 ".omp_data_kinds");
11340 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
11341 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
11342 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
11343 gimple_omp_target_set_data_arg (stmt
, t
);
11345 vec
<constructor_elt
, va_gc
> *vsize
;
11346 vec
<constructor_elt
, va_gc
> *vkind
;
11347 vec_alloc (vsize
, map_cnt
);
11348 vec_alloc (vkind
, map_cnt
);
11349 unsigned int map_idx
= 0;
11351 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11352 switch (OMP_CLAUSE_CODE (c
))
11358 case OMP_CLAUSE_MAP
:
11359 case OMP_CLAUSE_TO
:
11360 case OMP_CLAUSE_FROM
:
11362 ovar
= OMP_CLAUSE_DECL (c
);
11363 if (!DECL_P (ovar
))
11365 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11366 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11368 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
11369 == get_base_address (ovar
));
11370 nc
= OMP_CLAUSE_CHAIN (c
);
11371 ovar
= OMP_CLAUSE_DECL (nc
);
11375 tree x
= build_sender_ref (ovar
, ctx
);
11377 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
11378 gimplify_assign (x
, v
, &ilist
);
11384 if (DECL_SIZE (ovar
)
11385 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
11387 tree ovar2
= DECL_VALUE_EXPR (ovar
);
11388 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
11389 ovar2
= TREE_OPERAND (ovar2
, 0);
11390 gcc_assert (DECL_P (ovar2
));
11393 if (!maybe_lookup_field (ovar
, ctx
))
11397 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
11398 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
11399 talign
= DECL_ALIGN_UNIT (ovar
);
11402 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11403 tree x
= build_sender_ref (ovar
, ctx
);
11404 if (maybe_lookup_oacc_reduction (var
, ctx
))
11406 gcc_checking_assert (offloaded
11407 && is_gimple_omp_oacc (stmt
));
11408 gimplify_assign (x
, var
, &ilist
);
11410 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11411 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
11412 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11413 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
11415 gcc_assert (offloaded
);
11417 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
11418 mark_addressable (avar
);
11419 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
11420 talign
= DECL_ALIGN_UNIT (avar
);
11421 avar
= build_fold_addr_expr (avar
);
11422 gimplify_assign (x
, avar
, &ilist
);
11424 else if (is_gimple_reg (var
))
11426 gcc_assert (offloaded
);
11427 tree avar
= create_tmp_var (TREE_TYPE (var
));
11428 mark_addressable (avar
);
11429 enum omp_clause_map_kind map_kind
11430 = OMP_CLAUSE_MAP_KIND (c
);
11431 if (GOMP_MAP_COPY_TO_P (map_kind
)
11432 || map_kind
== OMP_CLAUSE_MAP_POINTER
11433 || map_kind
== OMP_CLAUSE_MAP_TO_PSET
11434 || map_kind
== OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
11435 gimplify_assign (avar
, var
, &ilist
);
11436 avar
= build_fold_addr_expr (avar
);
11437 gimplify_assign (x
, avar
, &ilist
);
11438 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
11439 || map_kind
== OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
11440 && !TYPE_READONLY (TREE_TYPE (var
)))
11442 x
= build_sender_ref (ovar
, ctx
);
11443 x
= build_simple_mem_ref (x
);
11444 gimplify_assign (var
, x
, &olist
);
11449 var
= build_fold_addr_expr (var
);
11450 gimplify_assign (x
, var
, &ilist
);
11453 tree s
= OMP_CLAUSE_SIZE (c
);
11454 if (s
== NULL_TREE
)
11455 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11456 s
= fold_convert (size_type_node
, s
);
11457 tree purpose
= size_int (map_idx
++);
11458 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11459 if (TREE_CODE (s
) != INTEGER_CST
)
11460 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11462 unsigned HOST_WIDE_INT tkind
;
11463 switch (OMP_CLAUSE_CODE (c
))
11465 case OMP_CLAUSE_MAP
:
11466 tkind
= OMP_CLAUSE_MAP_KIND (c
);
11468 case OMP_CLAUSE_TO
:
11469 tkind
= OMP_CLAUSE_MAP_TO
;
11471 case OMP_CLAUSE_FROM
:
11472 tkind
= OMP_CLAUSE_MAP_FROM
;
11475 gcc_unreachable ();
11477 gcc_checking_assert (tkind
11478 < (HOST_WIDE_INT_C (1U) << talign_shift
));
11479 talign
= ceil_log2 (talign
);
11480 tkind
|= talign
<< talign_shift
;
11481 gcc_checking_assert (tkind
11482 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11483 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
11484 build_int_cstu (tkind_type
, tkind
));
11489 gcc_assert (map_idx
== map_cnt
);
11491 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
11492 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
11493 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
11494 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
11495 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
11497 gimple_seq initlist
= NULL
;
11498 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
11499 TREE_VEC_ELT (t
, 1)),
11500 &initlist
, true, NULL_TREE
);
11501 gimple_seq_add_seq (&ilist
, initlist
);
11503 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
11505 TREE_THIS_VOLATILE (clobber
) = 1;
11506 gimple_seq_add_stmt (&olist
,
11507 gimple_build_assign (TREE_VEC_ELT (t
, 1),
11511 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
11512 TREE_THIS_VOLATILE (clobber
) = 1;
11513 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11517 /* Once all the expansions are done, sequence all the different
11518 fragments inside gimple_omp_body. */
11523 && ctx
->record_type
)
11525 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11526 /* fixup_child_record_type might have changed receiver_decl's type. */
11527 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11528 gimple_seq_add_stmt (&new_body
,
11529 gimple_build_assign (ctx
->receiver_decl
, t
));
11534 gimple_seq_add_seq (&new_body
, tgt_body
);
11535 new_body
= maybe_catch_exception (new_body
);
11537 else if (data_region
)
11538 new_body
= tgt_body
;
11539 if (offloaded
|| data_region
)
11541 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11542 gimple_omp_set_body (stmt
, new_body
);
11545 bind
= gimple_build_bind (NULL
, NULL
,
11546 tgt_bind
? gimple_bind_block (tgt_bind
)
11548 gsi_replace (gsi_p
, bind
, true);
11549 gimple_bind_add_seq (bind
, irlist
);
11550 gimple_bind_add_seq (bind
, ilist
);
11551 gimple_bind_add_stmt (bind
, stmt
);
11552 gimple_bind_add_seq (bind
, olist
);
11553 gimple_bind_add_seq (bind
, orlist
);
11555 pop_gimplify_context (NULL
);
11558 /* Expand code for an OpenMP teams directive. */
11561 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11563 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
11564 push_gimplify_context ();
11566 tree block
= make_node (BLOCK
);
11567 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
11568 gsi_replace (gsi_p
, bind
, true);
11569 gimple_seq bind_body
= NULL
;
11570 gimple_seq dlist
= NULL
;
11571 gimple_seq olist
= NULL
;
11573 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11574 OMP_CLAUSE_NUM_TEAMS
);
11575 if (num_teams
== NULL_TREE
)
11576 num_teams
= build_int_cst (unsigned_type_node
, 0);
11579 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
11580 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
11581 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
11583 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11584 OMP_CLAUSE_THREAD_LIMIT
);
11585 if (thread_limit
== NULL_TREE
)
11586 thread_limit
= build_int_cst (unsigned_type_node
, 0);
11589 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
11590 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
11591 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
11595 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
11596 &bind_body
, &dlist
, ctx
, NULL
);
11597 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
11598 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
11599 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
11601 location_t loc
= gimple_location (teams_stmt
);
11602 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
11603 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
11604 gimple_set_location (call
, loc
);
11605 gimple_seq_add_stmt (&bind_body
, call
);
11607 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
11608 gimple_omp_set_body (teams_stmt
, NULL
);
11609 gimple_seq_add_seq (&bind_body
, olist
);
11610 gimple_seq_add_seq (&bind_body
, dlist
);
11611 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
11612 gimple_bind_set_body (bind
, bind_body
);
11614 pop_gimplify_context (bind
);
11616 gimple_bind_append_vars (bind
, ctx
->block_vars
);
11617 BLOCK_VARS (block
) = ctx
->block_vars
;
11618 if (BLOCK_VARS (block
))
11619 TREE_USED (block
) = 1;
11623 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11624 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11625 of OMP context, but with task_shared_vars set. */
11628 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
11633 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11634 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
11637 if (task_shared_vars
11639 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
11642 /* If a global variable has been privatized, TREE_CONSTANT on
11643 ADDR_EXPR might be wrong. */
11644 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
11645 recompute_tree_invariant_for_addr_expr (t
);
11647 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
11652 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11654 gimple stmt
= gsi_stmt (*gsi_p
);
11655 struct walk_stmt_info wi
;
11658 if (gimple_has_location (stmt
))
11659 input_location
= gimple_location (stmt
);
11661 if (task_shared_vars
)
11662 memset (&wi
, '\0', sizeof (wi
));
11664 /* If we have issued syntax errors, avoid doing any heavy lifting.
11665 Just replace the OMP directives with a NOP to avoid
11666 confusing RTL expansion. */
11667 if (seen_error () && is_gimple_omp (stmt
))
11669 gsi_replace (gsi_p
, gimple_build_nop (), true);
11673 switch (gimple_code (stmt
))
11677 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
11678 if ((ctx
|| task_shared_vars
)
11679 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
11680 lower_omp_regimplify_p
,
11681 ctx
? NULL
: &wi
, NULL
)
11682 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
11683 lower_omp_regimplify_p
,
11684 ctx
? NULL
: &wi
, NULL
)))
11685 gimple_regimplify_operands (cond_stmt
, gsi_p
);
11689 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
11691 case GIMPLE_EH_FILTER
:
11692 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
11695 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
11696 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
11698 case GIMPLE_TRANSACTION
:
11699 lower_omp (gimple_transaction_body_ptr (
11700 as_a
<gtransaction
*> (stmt
)),
11704 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
11706 case GIMPLE_OMP_PARALLEL
:
11707 case GIMPLE_OMP_TASK
:
11708 ctx
= maybe_lookup_ctx (stmt
);
11710 if (ctx
->cancellable
)
11711 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11712 lower_omp_taskreg (gsi_p
, ctx
);
11714 case GIMPLE_OMP_FOR
:
11715 ctx
= maybe_lookup_ctx (stmt
);
11717 if (ctx
->cancellable
)
11718 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11719 lower_omp_for (gsi_p
, ctx
);
11721 case GIMPLE_OMP_SECTIONS
:
11722 ctx
= maybe_lookup_ctx (stmt
);
11724 if (ctx
->cancellable
)
11725 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11726 lower_omp_sections (gsi_p
, ctx
);
11728 case GIMPLE_OMP_SINGLE
:
11729 ctx
= maybe_lookup_ctx (stmt
);
11731 lower_omp_single (gsi_p
, ctx
);
11733 case GIMPLE_OMP_MASTER
:
11734 ctx
= maybe_lookup_ctx (stmt
);
11736 lower_omp_master (gsi_p
, ctx
);
11738 case GIMPLE_OMP_TASKGROUP
:
11739 ctx
= maybe_lookup_ctx (stmt
);
11741 lower_omp_taskgroup (gsi_p
, ctx
);
11743 case GIMPLE_OMP_ORDERED
:
11744 ctx
= maybe_lookup_ctx (stmt
);
11746 lower_omp_ordered (gsi_p
, ctx
);
11748 case GIMPLE_OMP_CRITICAL
:
11749 ctx
= maybe_lookup_ctx (stmt
);
11751 lower_omp_critical (gsi_p
, ctx
);
11753 case GIMPLE_OMP_ATOMIC_LOAD
:
11754 if ((ctx
|| task_shared_vars
)
11755 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
11756 as_a
<gomp_atomic_load
*> (stmt
)),
11757 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
11758 gimple_regimplify_operands (stmt
, gsi_p
);
11760 case GIMPLE_OMP_TARGET
:
11761 ctx
= maybe_lookup_ctx (stmt
);
11763 lower_omp_target (gsi_p
, ctx
);
11765 case GIMPLE_OMP_TEAMS
:
11766 ctx
= maybe_lookup_ctx (stmt
);
11768 lower_omp_teams (gsi_p
, ctx
);
11772 call_stmt
= as_a
<gcall
*> (stmt
);
11773 fndecl
= gimple_call_fndecl (call_stmt
);
11775 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
11776 switch (DECL_FUNCTION_CODE (fndecl
))
11778 case BUILT_IN_GOMP_BARRIER
:
11782 case BUILT_IN_GOMP_CANCEL
:
11783 case BUILT_IN_GOMP_CANCELLATION_POINT
:
11786 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
11787 cctx
= cctx
->outer
;
11788 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
11789 if (!cctx
->cancellable
)
11791 if (DECL_FUNCTION_CODE (fndecl
)
11792 == BUILT_IN_GOMP_CANCELLATION_POINT
)
11794 stmt
= gimple_build_nop ();
11795 gsi_replace (gsi_p
, stmt
, false);
11799 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
11801 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
11802 gimple_call_set_fndecl (call_stmt
, fndecl
);
11803 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
11806 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
11807 gimple_call_set_lhs (call_stmt
, lhs
);
11808 tree fallthru_label
;
11809 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
11811 g
= gimple_build_label (fallthru_label
);
11812 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11813 g
= gimple_build_cond (NE_EXPR
, lhs
,
11814 fold_convert (TREE_TYPE (lhs
),
11815 boolean_false_node
),
11816 cctx
->cancel_label
, fallthru_label
);
11817 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11824 if ((ctx
|| task_shared_vars
)
11825 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
11828 /* Just remove clobbers, this should happen only if we have
11829 "privatized" local addressable variables in SIMD regions,
11830 the clobber isn't needed in that case and gimplifying address
11831 of the ARRAY_REF into a pointer and creating MEM_REF based
11832 clobber would create worse code than we get with the clobber
11834 if (gimple_clobber_p (stmt
))
11836 gsi_replace (gsi_p
, gimple_build_nop (), true);
11839 gimple_regimplify_operands (stmt
, gsi_p
);
11846 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
11848 location_t saved_location
= input_location
;
11849 gimple_stmt_iterator gsi
;
11850 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11851 lower_omp_1 (&gsi
, ctx
);
11852 /* During gimplification, we haven't folded statments inside offloading
11853 regions (gimplify.c:maybe_fold_stmt); do that now. */
11854 if (target_nesting_level
)
11855 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11857 input_location
= saved_location
;
11860 /* Main entry point. */
11862 static unsigned int
11863 execute_lower_omp (void)
11869 /* This pass always runs, to provide PROP_gimple_lomp.
11870 But often, there is nothing to do. */
11871 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
11872 && flag_openmp_simd
== 0)
11875 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
11876 delete_omp_context
);
11878 body
= gimple_body (current_function_decl
);
11879 scan_omp (&body
, NULL
);
11880 gcc_assert (taskreg_nesting_level
== 0);
11881 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
11882 finish_taskreg_scan (ctx
);
11883 taskreg_contexts
.release ();
11885 if (all_contexts
->root
)
11887 if (task_shared_vars
)
11888 push_gimplify_context ();
11889 lower_omp (&body
, NULL
);
11890 if (task_shared_vars
)
11891 pop_gimplify_context (NULL
);
11896 splay_tree_delete (all_contexts
);
11897 all_contexts
= NULL
;
11899 BITMAP_FREE (task_shared_vars
);
11905 const pass_data pass_data_lower_omp
=
11907 GIMPLE_PASS
, /* type */
11908 "omplower", /* name */
11909 OPTGROUP_NONE
, /* optinfo_flags */
11910 TV_NONE
, /* tv_id */
11911 PROP_gimple_any
, /* properties_required */
11912 PROP_gimple_lomp
, /* properties_provided */
11913 0, /* properties_destroyed */
11914 0, /* todo_flags_start */
11915 0, /* todo_flags_finish */
11918 class pass_lower_omp
: public gimple_opt_pass
11921 pass_lower_omp (gcc::context
*ctxt
)
11922 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
11925 /* opt_pass methods: */
11926 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
11928 }; // class pass_lower_omp
11930 } // anon namespace
11933 make_pass_lower_omp (gcc::context
*ctxt
)
11935 return new pass_lower_omp (ctxt
);
11938 /* The following is a utility to diagnose structured block violations.
11939 It is not part of the "omplower" pass, as that's invoked too late. It
11940 should be invoked by the respective front ends after gimplification. */
11942 static splay_tree all_labels
;
11944 /* Check for mismatched contexts and generate an error if needed. Return
11945 true if an error is detected. */
11948 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
11949 gimple branch_ctx
, gimple label_ctx
)
11951 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
11952 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
11954 if (label_ctx
== branch_ctx
)
11957 const char* kind
= NULL
;
11962 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
11963 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
11965 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
11966 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
11967 kind
= "Cilk Plus";
11971 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
11972 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
11974 gcc_checking_assert (kind
== NULL
);
11980 gcc_checking_assert (flag_openmp
);
11985 Previously we kept track of the label's entire context in diagnose_sb_[12]
11986 so we could traverse it and issue a correct "exit" or "enter" error
11987 message upon a structured block violation.
11989 We built the context by building a list with tree_cons'ing, but there is
11990 no easy counterpart in gimple tuples. It seems like far too much work
11991 for issuing exit/enter error messages. If someone really misses the
11992 distinct error message... patches welcome.
11996 /* Try to avoid confusing the user by producing and error message
11997 with correct "exit" or "enter" verbiage. We prefer "exit"
11998 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
11999 if (branch_ctx
== NULL
)
12005 if (TREE_VALUE (label_ctx
) == branch_ctx
)
12010 label_ctx
= TREE_CHAIN (label_ctx
);
12015 error ("invalid exit from %s structured block", kind
);
12017 error ("invalid entry to %s structured block", kind
);
12020 /* If it's obvious we have an invalid entry, be specific about the error. */
12021 if (branch_ctx
== NULL
)
12022 error ("invalid entry to %s structured block", kind
);
12025 /* Otherwise, be vague and lazy, but efficient. */
12026 error ("invalid branch to/from %s structured block", kind
);
12029 gsi_replace (gsi_p
, gimple_build_nop (), false);
12033 /* Pass 1: Create a minimal tree of structured blocks, and record
12034 where each label is found. */
12037 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12038 struct walk_stmt_info
*wi
)
12040 gimple context
= (gimple
) wi
->info
;
12041 gimple inner_context
;
12042 gimple stmt
= gsi_stmt (*gsi_p
);
12044 *handled_ops_p
= true;
12046 switch (gimple_code (stmt
))
12050 case GIMPLE_OMP_PARALLEL
:
12051 case GIMPLE_OMP_TASK
:
12052 case GIMPLE_OMP_SECTIONS
:
12053 case GIMPLE_OMP_SINGLE
:
12054 case GIMPLE_OMP_SECTION
:
12055 case GIMPLE_OMP_MASTER
:
12056 case GIMPLE_OMP_ORDERED
:
12057 case GIMPLE_OMP_CRITICAL
:
12058 case GIMPLE_OMP_TARGET
:
12059 case GIMPLE_OMP_TEAMS
:
12060 case GIMPLE_OMP_TASKGROUP
:
12061 /* The minimal context here is just the current OMP construct. */
12062 inner_context
= stmt
;
12063 wi
->info
= inner_context
;
12064 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12065 wi
->info
= context
;
12068 case GIMPLE_OMP_FOR
:
12069 inner_context
= stmt
;
12070 wi
->info
= inner_context
;
12071 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12073 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
12074 diagnose_sb_1
, NULL
, wi
);
12075 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12076 wi
->info
= context
;
12080 splay_tree_insert (all_labels
,
12081 (splay_tree_key
) gimple_label_label (
12082 as_a
<glabel
*> (stmt
)),
12083 (splay_tree_value
) context
);
12093 /* Pass 2: Check each branch and see if its context differs from that of
12094 the destination label's context. */
12097 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12098 struct walk_stmt_info
*wi
)
12100 gimple context
= (gimple
) wi
->info
;
12102 gimple stmt
= gsi_stmt (*gsi_p
);
12104 *handled_ops_p
= true;
12106 switch (gimple_code (stmt
))
12110 case GIMPLE_OMP_PARALLEL
:
12111 case GIMPLE_OMP_TASK
:
12112 case GIMPLE_OMP_SECTIONS
:
12113 case GIMPLE_OMP_SINGLE
:
12114 case GIMPLE_OMP_SECTION
:
12115 case GIMPLE_OMP_MASTER
:
12116 case GIMPLE_OMP_ORDERED
:
12117 case GIMPLE_OMP_CRITICAL
:
12118 case GIMPLE_OMP_TARGET
:
12119 case GIMPLE_OMP_TEAMS
:
12120 case GIMPLE_OMP_TASKGROUP
:
12122 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12123 wi
->info
= context
;
12126 case GIMPLE_OMP_FOR
:
12128 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12130 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
12131 diagnose_sb_2
, NULL
, wi
);
12132 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12133 wi
->info
= context
;
12138 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
12139 tree lab
= gimple_cond_true_label (cond_stmt
);
12142 n
= splay_tree_lookup (all_labels
,
12143 (splay_tree_key
) lab
);
12144 diagnose_sb_0 (gsi_p
, context
,
12145 n
? (gimple
) n
->value
: NULL
);
12147 lab
= gimple_cond_false_label (cond_stmt
);
12150 n
= splay_tree_lookup (all_labels
,
12151 (splay_tree_key
) lab
);
12152 diagnose_sb_0 (gsi_p
, context
,
12153 n
? (gimple
) n
->value
: NULL
);
12160 tree lab
= gimple_goto_dest (stmt
);
12161 if (TREE_CODE (lab
) != LABEL_DECL
)
12164 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12165 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
12169 case GIMPLE_SWITCH
:
12171 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
12173 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
12175 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
12176 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12177 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
12183 case GIMPLE_RETURN
:
12184 diagnose_sb_0 (gsi_p
, context
, NULL
);
12194 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12197 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
12200 gimple last
= last_stmt (bb
);
12201 enum gimple_code code
= gimple_code (last
);
12202 struct omp_region
*cur_region
= *region
;
12203 bool fallthru
= false;
12207 case GIMPLE_OMP_PARALLEL
:
12208 case GIMPLE_OMP_TASK
:
12209 case GIMPLE_OMP_FOR
:
12210 case GIMPLE_OMP_SINGLE
:
12211 case GIMPLE_OMP_TEAMS
:
12212 case GIMPLE_OMP_MASTER
:
12213 case GIMPLE_OMP_TASKGROUP
:
12214 case GIMPLE_OMP_ORDERED
:
12215 case GIMPLE_OMP_CRITICAL
:
12216 case GIMPLE_OMP_SECTION
:
12217 cur_region
= new_omp_region (bb
, code
, cur_region
);
12221 case GIMPLE_OMP_TARGET
:
12222 cur_region
= new_omp_region (bb
, code
, cur_region
);
12224 switch (gimple_omp_target_kind (last
))
12226 case GF_OMP_TARGET_KIND_REGION
:
12227 case GF_OMP_TARGET_KIND_DATA
:
12228 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12229 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12230 case GF_OMP_TARGET_KIND_OACC_DATA
:
12232 case GF_OMP_TARGET_KIND_UPDATE
:
12233 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12234 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12235 cur_region
= cur_region
->outer
;
12238 gcc_unreachable ();
12242 case GIMPLE_OMP_SECTIONS
:
12243 cur_region
= new_omp_region (bb
, code
, cur_region
);
12247 case GIMPLE_OMP_SECTIONS_SWITCH
:
12251 case GIMPLE_OMP_ATOMIC_LOAD
:
12252 case GIMPLE_OMP_ATOMIC_STORE
:
12256 case GIMPLE_OMP_RETURN
:
12257 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12258 somewhere other than the next block. This will be
12260 cur_region
->exit
= bb
;
12261 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
12262 cur_region
= cur_region
->outer
;
12265 case GIMPLE_OMP_CONTINUE
:
12266 cur_region
->cont
= bb
;
12267 switch (cur_region
->type
)
12269 case GIMPLE_OMP_FOR
:
12270 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12271 succs edges as abnormal to prevent splitting
12273 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
12274 /* Make the loopback edge. */
12275 make_edge (bb
, single_succ (cur_region
->entry
),
12278 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12279 corresponds to the case that the body of the loop
12280 is not executed at all. */
12281 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
12282 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
12286 case GIMPLE_OMP_SECTIONS
:
12287 /* Wire up the edges into and out of the nested sections. */
12289 basic_block switch_bb
= single_succ (cur_region
->entry
);
12291 struct omp_region
*i
;
12292 for (i
= cur_region
->inner
; i
; i
= i
->next
)
12294 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
12295 make_edge (switch_bb
, i
->entry
, 0);
12296 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
12299 /* Make the loopback edge to the block with
12300 GIMPLE_OMP_SECTIONS_SWITCH. */
12301 make_edge (bb
, switch_bb
, 0);
12303 /* Make the edge from the switch to exit. */
12304 make_edge (switch_bb
, bb
->next_bb
, 0);
12310 gcc_unreachable ();
12315 gcc_unreachable ();
12318 if (*region
!= cur_region
)
12320 *region
= cur_region
;
12322 *region_idx
= cur_region
->entry
->index
;
12330 static unsigned int
12331 diagnose_omp_structured_block_errors (void)
12333 struct walk_stmt_info wi
;
12334 gimple_seq body
= gimple_body (current_function_decl
);
12336 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
12338 memset (&wi
, 0, sizeof (wi
));
12339 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
12341 memset (&wi
, 0, sizeof (wi
));
12342 wi
.want_locations
= true;
12343 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
12345 gimple_set_body (current_function_decl
, body
);
12347 splay_tree_delete (all_labels
);
12355 const pass_data pass_data_diagnose_omp_blocks
=
12357 GIMPLE_PASS
, /* type */
12358 "*diagnose_omp_blocks", /* name */
12359 OPTGROUP_NONE
, /* optinfo_flags */
12360 TV_NONE
, /* tv_id */
12361 PROP_gimple_any
, /* properties_required */
12362 0, /* properties_provided */
12363 0, /* properties_destroyed */
12364 0, /* todo_flags_start */
12365 0, /* todo_flags_finish */
12368 class pass_diagnose_omp_blocks
: public gimple_opt_pass
12371 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12372 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
12375 /* opt_pass methods: */
12376 virtual bool gate (function
*)
12378 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
12380 virtual unsigned int execute (function
*)
12382 return diagnose_omp_structured_block_errors ();
12385 }; // class pass_diagnose_omp_blocks
12387 } // anon namespace
12390 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12392 return new pass_diagnose_omp_blocks (ctxt
);
12395 /* SIMD clone supporting code. */
12397 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12398 of arguments to reserve space for. */
12400 static struct cgraph_simd_clone
*
12401 simd_clone_struct_alloc (int nargs
)
12403 struct cgraph_simd_clone
*clone_info
;
12404 size_t len
= (sizeof (struct cgraph_simd_clone
)
12405 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
12406 clone_info
= (struct cgraph_simd_clone
*)
12407 ggc_internal_cleared_alloc (len
);
12411 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12414 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
12415 struct cgraph_simd_clone
*from
)
12417 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
12418 + ((from
->nargs
- from
->inbranch
)
12419 * sizeof (struct cgraph_simd_clone_arg
))));
12422 /* Return vector of parameter types of function FNDECL. This uses
12423 TYPE_ARG_TYPES if available, otherwise falls back to types of
12424 DECL_ARGUMENTS types. */
12427 simd_clone_vector_of_formal_parm_types (tree fndecl
)
12429 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
12430 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
12431 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
12434 FOR_EACH_VEC_ELT (args
, i
, arg
)
12435 args
[i
] = TREE_TYPE (args
[i
]);
12439 /* Given a simd function in NODE, extract the simd specific
12440 information from the OMP clauses passed in CLAUSES, and return
12441 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12442 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12443 otherwise set to FALSE. */
12445 static struct cgraph_simd_clone
*
12446 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
12447 bool *inbranch_specified
)
12449 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12452 *inbranch_specified
= false;
12454 n
= args
.length ();
12455 if (n
> 0 && args
.last () == void_type_node
)
12458 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12459 be cloned have a distinctive artificial label in addition to "omp
12463 && lookup_attribute ("cilk simd function",
12464 DECL_ATTRIBUTES (node
->decl
)));
12466 /* Allocate one more than needed just in case this is an in-branch
12467 clone which will require a mask argument. */
12468 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
12469 clone_info
->nargs
= n
;
12470 clone_info
->cilk_elemental
= cilk_clone
;
12477 clauses
= TREE_VALUE (clauses
);
12478 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
12481 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
12483 switch (OMP_CLAUSE_CODE (t
))
12485 case OMP_CLAUSE_INBRANCH
:
12486 clone_info
->inbranch
= 1;
12487 *inbranch_specified
= true;
12489 case OMP_CLAUSE_NOTINBRANCH
:
12490 clone_info
->inbranch
= 0;
12491 *inbranch_specified
= true;
12493 case OMP_CLAUSE_SIMDLEN
:
12494 clone_info
->simdlen
12495 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
12497 case OMP_CLAUSE_LINEAR
:
12499 tree decl
= OMP_CLAUSE_DECL (t
);
12500 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
12501 int argno
= TREE_INT_CST_LOW (decl
);
12502 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
12504 clone_info
->args
[argno
].arg_type
12505 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
12506 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12507 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
12508 && clone_info
->args
[argno
].linear_step
< n
);
12512 if (POINTER_TYPE_P (args
[argno
]))
12513 step
= fold_convert (ssizetype
, step
);
12514 if (!tree_fits_shwi_p (step
))
12516 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12517 "ignoring large linear step");
12521 else if (integer_zerop (step
))
12523 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12524 "ignoring zero linear step");
12530 clone_info
->args
[argno
].arg_type
12531 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
12532 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12537 case OMP_CLAUSE_UNIFORM
:
12539 tree decl
= OMP_CLAUSE_DECL (t
);
12540 int argno
= tree_to_uhwi (decl
);
12541 clone_info
->args
[argno
].arg_type
12542 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
12545 case OMP_CLAUSE_ALIGNED
:
12547 tree decl
= OMP_CLAUSE_DECL (t
);
12548 int argno
= tree_to_uhwi (decl
);
12549 clone_info
->args
[argno
].alignment
12550 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
12561 /* Given a SIMD clone in NODE, calculate the characteristic data
12562 type and return the coresponding type. The characteristic data
12563 type is computed as described in the Intel Vector ABI. */
12566 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
12567 struct cgraph_simd_clone
*clone_info
)
12569 tree type
= integer_type_node
;
12570 tree fndecl
= node
->decl
;
12572 /* a) For non-void function, the characteristic data type is the
12574 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
12575 type
= TREE_TYPE (TREE_TYPE (fndecl
));
12577 /* b) If the function has any non-uniform, non-linear parameters,
12578 then the characteristic data type is the type of the first
12582 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
12583 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
12584 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
12592 /* c) If the characteristic data type determined by a) or b) above
12593 is struct, union, or class type which is pass-by-value (except
12594 for the type that maps to the built-in complex data type), the
12595 characteristic data type is int. */
12596 if (RECORD_OR_UNION_TYPE_P (type
)
12597 && !aggregate_value_p (type
, NULL
)
12598 && TREE_CODE (type
) != COMPLEX_TYPE
)
12599 return integer_type_node
;
12601 /* d) If none of the above three classes is applicable, the
12602 characteristic data type is int. */
12606 /* e) For Intel Xeon Phi native and offload compilation, if the
12607 resulting characteristic data type is 8-bit or 16-bit integer
12608 data type, the characteristic data type is int. */
12609 /* Well, we don't handle Xeon Phi yet. */
12613 simd_clone_mangle (struct cgraph_node
*node
,
12614 struct cgraph_simd_clone
*clone_info
)
12616 char vecsize_mangle
= clone_info
->vecsize_mangle
;
12617 char mask
= clone_info
->inbranch
? 'M' : 'N';
12618 unsigned int simdlen
= clone_info
->simdlen
;
12622 gcc_assert (vecsize_mangle
&& simdlen
);
12624 pp_string (&pp
, "_ZGV");
12625 pp_character (&pp
, vecsize_mangle
);
12626 pp_character (&pp
, mask
);
12627 pp_decimal_int (&pp
, simdlen
);
12629 for (n
= 0; n
< clone_info
->nargs
; ++n
)
12631 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
12633 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
12634 pp_character (&pp
, 'u');
12635 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12637 gcc_assert (arg
.linear_step
!= 0);
12638 pp_character (&pp
, 'l');
12639 if (arg
.linear_step
> 1)
12640 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12641 else if (arg
.linear_step
< 0)
12643 pp_character (&pp
, 'n');
12644 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
12648 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
12650 pp_character (&pp
, 's');
12651 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12654 pp_character (&pp
, 'v');
12657 pp_character (&pp
, 'a');
12658 pp_decimal_int (&pp
, arg
.alignment
);
12662 pp_underscore (&pp
);
12664 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
12665 const char *str
= pp_formatted_text (&pp
);
12667 /* If there already is a SIMD clone with the same mangled name, don't
12668 add another one. This can happen e.g. for
12669 #pragma omp declare simd
12670 #pragma omp declare simd simdlen(8)
12671 int foo (int, int);
12672 if the simdlen is assumed to be 8 for the first one, etc. */
12673 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
12674 clone
= clone
->simdclone
->next_clone
)
12675 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
12679 return get_identifier (str
);
12682 /* Create a simd clone of OLD_NODE and return it. */
12684 static struct cgraph_node
*
12685 simd_clone_create (struct cgraph_node
*old_node
)
12687 struct cgraph_node
*new_node
;
12688 if (old_node
->definition
)
12690 if (!old_node
->has_gimple_body_p ())
12692 old_node
->get_body ();
12693 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
12699 tree old_decl
= old_node
->decl
;
12700 tree new_decl
= copy_node (old_node
->decl
);
12701 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
12702 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
12703 SET_DECL_RTL (new_decl
, NULL
);
12704 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
12705 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
12706 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
12707 symtab
->call_cgraph_insertion_hooks (new_node
);
12709 if (new_node
== NULL
)
12712 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
12714 /* The function cgraph_function_versioning () will force the new
12715 symbol local. Undo this, and inherit external visability from
12717 new_node
->local
.local
= old_node
->local
.local
;
12718 new_node
->externally_visible
= old_node
->externally_visible
;
12723 /* Adjust the return type of the given function to its appropriate
12724 vector counterpart. Returns a simd array to be used throughout the
12725 function as a return value. */
12728 simd_clone_adjust_return_type (struct cgraph_node
*node
)
12730 tree fndecl
= node
->decl
;
12731 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
12732 unsigned int veclen
;
12735 /* Adjust the function return type. */
12736 if (orig_rettype
== void_type_node
)
12738 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
12739 t
= TREE_TYPE (TREE_TYPE (fndecl
));
12740 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
12741 veclen
= node
->simdclone
->vecsize_int
;
12743 veclen
= node
->simdclone
->vecsize_float
;
12744 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
12745 if (veclen
> node
->simdclone
->simdlen
)
12746 veclen
= node
->simdclone
->simdlen
;
12747 if (POINTER_TYPE_P (t
))
12748 t
= pointer_sized_int_node
;
12749 if (veclen
== node
->simdclone
->simdlen
)
12750 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
12753 t
= build_vector_type (t
, veclen
);
12754 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
12756 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
12757 if (!node
->definition
)
12760 t
= DECL_RESULT (fndecl
);
12761 /* Adjust the DECL_RESULT. */
12762 gcc_assert (TREE_TYPE (t
) != void_type_node
);
12763 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
12766 tree atype
= build_array_type_nelts (orig_rettype
,
12767 node
->simdclone
->simdlen
);
12768 if (veclen
!= node
->simdclone
->simdlen
)
12769 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
12771 /* Set up a SIMD array to use as the return value. */
12772 tree retval
= create_tmp_var_raw (atype
, "retval");
12773 gimple_add_tmp_var (retval
);
12777 /* Each vector argument has a corresponding array to be used locally
12778 as part of the eventual loop. Create such temporary array and
12781 PREFIX is the prefix to be used for the temporary.
12783 TYPE is the inner element type.
12785 SIMDLEN is the number of elements. */
12788 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
12790 tree atype
= build_array_type_nelts (type
, simdlen
);
12791 tree avar
= create_tmp_var_raw (atype
, prefix
);
12792 gimple_add_tmp_var (avar
);
12796 /* Modify the function argument types to their corresponding vector
12797 counterparts if appropriate. Also, create one array for each simd
12798 argument to be used locally when using the function arguments as
12801 NODE is the function whose arguments are to be adjusted.
12803 Returns an adjustment vector that will be filled describing how the
12804 argument types will be adjusted. */
12806 static ipa_parm_adjustment_vec
12807 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
12810 ipa_parm_adjustment_vec adjustments
;
12812 if (node
->definition
)
12813 args
= ipa_get_vector_of_formal_parms (node
->decl
);
12815 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12816 adjustments
.create (args
.length ());
12817 unsigned i
, j
, veclen
;
12818 struct ipa_parm_adjustment adj
;
12819 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
12821 memset (&adj
, 0, sizeof (adj
));
12822 tree parm
= args
[i
];
12823 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
12824 adj
.base_index
= i
;
12827 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
12828 node
->simdclone
->args
[i
].orig_type
= parm_type
;
12830 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
12832 /* No adjustment necessary for scalar arguments. */
12833 adj
.op
= IPA_PARM_OP_COPY
;
12837 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
12838 veclen
= node
->simdclone
->vecsize_int
;
12840 veclen
= node
->simdclone
->vecsize_float
;
12841 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
12842 if (veclen
> node
->simdclone
->simdlen
)
12843 veclen
= node
->simdclone
->simdlen
;
12844 adj
.arg_prefix
= "simd";
12845 if (POINTER_TYPE_P (parm_type
))
12846 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12848 adj
.type
= build_vector_type (parm_type
, veclen
);
12849 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
12850 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12852 adjustments
.safe_push (adj
);
12855 memset (&adj
, 0, sizeof (adj
));
12856 adj
.op
= IPA_PARM_OP_NEW
;
12857 adj
.arg_prefix
= "simd";
12858 adj
.base_index
= i
;
12859 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
12863 if (node
->definition
)
12864 node
->simdclone
->args
[i
].simd_array
12865 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
12866 parm_type
, node
->simdclone
->simdlen
);
12868 adjustments
.safe_push (adj
);
12871 if (node
->simdclone
->inbranch
)
12874 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
12877 memset (&adj
, 0, sizeof (adj
));
12878 adj
.op
= IPA_PARM_OP_NEW
;
12879 adj
.arg_prefix
= "mask";
12881 adj
.base_index
= i
;
12882 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
12883 veclen
= node
->simdclone
->vecsize_int
;
12885 veclen
= node
->simdclone
->vecsize_float
;
12886 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
12887 if (veclen
> node
->simdclone
->simdlen
)
12888 veclen
= node
->simdclone
->simdlen
;
12889 if (POINTER_TYPE_P (base_type
))
12890 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12892 adj
.type
= build_vector_type (base_type
, veclen
);
12893 adjustments
.safe_push (adj
);
12895 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12896 adjustments
.safe_push (adj
);
12898 /* We have previously allocated one extra entry for the mask. Use
12900 struct cgraph_simd_clone
*sc
= node
->simdclone
;
12902 if (node
->definition
)
12904 sc
->args
[i
].orig_arg
12905 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
12906 sc
->args
[i
].simd_array
12907 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
12909 sc
->args
[i
].orig_type
= base_type
;
12910 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
12913 if (node
->definition
)
12914 ipa_modify_formal_parameters (node
->decl
, adjustments
);
12917 tree new_arg_types
= NULL_TREE
, new_reversed
;
12918 bool last_parm_void
= false;
12919 if (args
.length () > 0 && args
.last () == void_type_node
)
12920 last_parm_void
= true;
12922 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
12923 j
= adjustments
.length ();
12924 for (i
= 0; i
< j
; i
++)
12926 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
12928 if (adj
->op
== IPA_PARM_OP_COPY
)
12929 ptype
= args
[adj
->base_index
];
12932 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
12934 new_reversed
= nreverse (new_arg_types
);
12935 if (last_parm_void
)
12938 TREE_CHAIN (new_arg_types
) = void_list_node
;
12940 new_reversed
= void_list_node
;
12943 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
12944 TYPE_ARG_TYPES (new_type
) = new_reversed
;
12945 TREE_TYPE (node
->decl
) = new_type
;
12947 adjustments
.release ();
12950 return adjustments
;
12953 /* Initialize and copy the function arguments in NODE to their
12954 corresponding local simd arrays. Returns a fresh gimple_seq with
12955 the instruction sequence generated. */
12958 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
12959 ipa_parm_adjustment_vec adjustments
)
12961 gimple_seq seq
= NULL
;
12962 unsigned i
= 0, j
= 0, k
;
12964 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
12966 arg
= DECL_CHAIN (arg
), i
++, j
++)
12968 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
12971 node
->simdclone
->args
[i
].vector_arg
= arg
;
12973 tree array
= node
->simdclone
->args
[i
].simd_array
;
12974 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
12976 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12977 tree ptr
= build_fold_addr_expr (array
);
12978 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
12979 build_int_cst (ptype
, 0));
12980 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
12981 gimplify_and_add (t
, &seq
);
12985 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
12986 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12987 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
12989 tree ptr
= build_fold_addr_expr (array
);
12993 arg
= DECL_CHAIN (arg
);
12997 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
12998 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
12999 build_int_cst (ptype
, k
* elemsize
));
13000 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
13001 gimplify_and_add (t
, &seq
);
13008 /* Callback info for ipa_simd_modify_stmt_ops below. */
13010 struct modify_stmt_info
{
13011 ipa_parm_adjustment_vec adjustments
;
13013 /* True if the parent statement was modified by
13014 ipa_simd_modify_stmt_ops. */
13018 /* Callback for walk_gimple_op.
13020 Adjust operands from a given statement as specified in the
13021 adjustments vector in the callback data. */
13024 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
13026 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13027 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
13028 tree
*orig_tp
= tp
;
13029 if (TREE_CODE (*tp
) == ADDR_EXPR
)
13030 tp
= &TREE_OPERAND (*tp
, 0);
13031 struct ipa_parm_adjustment
*cand
= NULL
;
13032 if (TREE_CODE (*tp
) == PARM_DECL
)
13033 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
13037 *walk_subtrees
= 0;
13040 tree repl
= NULL_TREE
;
13042 repl
= unshare_expr (cand
->new_decl
);
13047 *walk_subtrees
= 0;
13048 bool modified
= info
->modified
;
13049 info
->modified
= false;
13050 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
13051 if (!info
->modified
)
13053 info
->modified
= modified
;
13056 info
->modified
= modified
;
13065 repl
= build_fold_addr_expr (repl
);
13067 if (is_gimple_debug (info
->stmt
))
13069 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
13070 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
13071 DECL_ARTIFICIAL (vexpr
) = 1;
13072 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
13073 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
13078 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
13079 repl
= gimple_assign_lhs (stmt
);
13081 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
13082 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13085 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
13087 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
13093 info
->modified
= true;
13097 /* Traverse the function body and perform all modifications as
13098 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13099 modified such that the replacement/reduction value will now be an
13100 offset into the corresponding simd_array.
13102 This function will replace all function argument uses with their
13103 corresponding simd array elements, and ajust the return values
13107 ipa_simd_modify_function_body (struct cgraph_node
*node
,
13108 ipa_parm_adjustment_vec adjustments
,
13109 tree retval_array
, tree iter
)
13112 unsigned int i
, j
, l
;
13114 /* Re-use the adjustments array, but this time use it to replace
13115 every function argument use to an offset into the corresponding
13117 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
13119 if (!node
->simdclone
->args
[i
].vector_arg
)
13122 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
13123 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
13124 adjustments
[j
].new_decl
13125 = build4 (ARRAY_REF
,
13127 node
->simdclone
->args
[i
].simd_array
,
13129 NULL_TREE
, NULL_TREE
);
13130 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
13131 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
13132 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
13135 l
= adjustments
.length ();
13136 for (i
= 1; i
< num_ssa_names
; i
++)
13138 tree name
= ssa_name (i
);
13140 && SSA_NAME_VAR (name
)
13141 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
13143 for (j
= 0; j
< l
; j
++)
13144 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
13145 && adjustments
[j
].new_decl
)
13148 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
13151 = copy_var_decl (adjustments
[j
].base
,
13152 DECL_NAME (adjustments
[j
].base
),
13153 TREE_TYPE (adjustments
[j
].base
));
13154 adjustments
[j
].new_ssa_base
= base_var
;
13157 base_var
= adjustments
[j
].new_ssa_base
;
13158 if (SSA_NAME_IS_DEFAULT_DEF (name
))
13160 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13161 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
13162 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
13163 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
13164 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13165 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
13166 gimple stmt
= gimple_build_assign (name
, new_decl
);
13167 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13170 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13175 struct modify_stmt_info info
;
13176 info
.adjustments
= adjustments
;
13178 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
13180 gimple_stmt_iterator gsi
;
13182 gsi
= gsi_start_bb (bb
);
13183 while (!gsi_end_p (gsi
))
13185 gimple stmt
= gsi_stmt (gsi
);
13187 struct walk_stmt_info wi
;
13189 memset (&wi
, 0, sizeof (wi
));
13190 info
.modified
= false;
13192 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
13194 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
13196 tree retval
= gimple_return_retval (return_stmt
);
13199 gsi_remove (&gsi
, true);
13203 /* Replace `return foo' with `retval_array[iter] = foo'. */
13204 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
13205 retval_array
, iter
, NULL
, NULL
);
13206 stmt
= gimple_build_assign (ref
, retval
);
13207 gsi_replace (&gsi
, stmt
, true);
13208 info
.modified
= true;
13213 update_stmt (stmt
);
13214 if (maybe_clean_eh_stmt (stmt
))
13215 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
13222 /* Adjust the argument types in NODE to their appropriate vector
13226 simd_clone_adjust (struct cgraph_node
*node
)
13228 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
13230 targetm
.simd_clone
.adjust (node
);
13232 tree retval
= simd_clone_adjust_return_type (node
);
13233 ipa_parm_adjustment_vec adjustments
13234 = simd_clone_adjust_argument_types (node
);
13236 push_gimplify_context ();
13238 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
13240 /* Adjust all uses of vector arguments accordingly. Adjust all
13241 return values accordingly. */
13242 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
13243 tree iter1
= make_ssa_name (iter
);
13244 tree iter2
= make_ssa_name (iter
);
13245 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
13247 /* Initialize the iteration variable. */
13248 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13249 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
13250 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
13251 /* Insert the SIMD array and iv initialization at function
13253 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
13255 pop_gimplify_context (NULL
);
13257 /* Create a new BB right before the original exit BB, to hold the
13258 iteration increment and the condition/branch. */
13259 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
13260 basic_block incr_bb
= create_empty_bb (orig_exit
);
13261 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
13262 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13263 flag. Set it now to be a FALLTHRU_EDGE. */
13264 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
13265 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
13266 for (unsigned i
= 0;
13267 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
13269 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
13270 redirect_edge_succ (e
, incr_bb
);
13272 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
13273 e
->probability
= REG_BR_PROB_BASE
;
13274 gsi
= gsi_last_bb (incr_bb
);
13275 gimple g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
13276 build_int_cst (unsigned_type_node
, 1));
13277 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13279 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13280 struct loop
*loop
= alloc_loop ();
13281 cfun
->has_force_vectorize_loops
= true;
13282 loop
->safelen
= node
->simdclone
->simdlen
;
13283 loop
->force_vectorize
= true;
13284 loop
->header
= body_bb
;
13286 /* Branch around the body if the mask applies. */
13287 if (node
->simdclone
->inbranch
)
13289 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
13291 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
13292 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
13293 tree aref
= build4 (ARRAY_REF
,
13294 TREE_TYPE (TREE_TYPE (mask_array
)),
13297 g
= gimple_build_assign (mask
, aref
);
13298 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13299 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
13300 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
13302 aref
= build1 (VIEW_CONVERT_EXPR
,
13303 build_nonstandard_integer_type (bitsize
, 0), mask
);
13304 mask
= make_ssa_name (TREE_TYPE (aref
));
13305 g
= gimple_build_assign (mask
, aref
);
13306 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13309 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
13311 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13312 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
13313 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
13316 /* Generate the condition. */
13317 g
= gimple_build_cond (LT_EXPR
,
13319 build_int_cst (unsigned_type_node
,
13320 node
->simdclone
->simdlen
),
13322 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13323 e
= split_block (incr_bb
, gsi_stmt (gsi
));
13324 basic_block latch_bb
= e
->dest
;
13325 basic_block new_exit_bb
;
13326 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
13327 loop
->latch
= latch_bb
;
13329 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
13331 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
13332 /* The successor of incr_bb is already pointing to latch_bb; just
13334 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13335 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
13337 gphi
*phi
= create_phi_node (iter1
, body_bb
);
13338 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
13339 edge latch_edge
= single_succ_edge (latch_bb
);
13340 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
13342 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13344 /* Generate the new return. */
13345 gsi
= gsi_last_bb (new_exit_bb
);
13347 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
13348 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
13349 retval
= TREE_OPERAND (retval
, 0);
13352 retval
= build1 (VIEW_CONVERT_EXPR
,
13353 TREE_TYPE (TREE_TYPE (node
->decl
)),
13355 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
13356 false, GSI_CONTINUE_LINKING
);
13358 g
= gimple_build_return (retval
);
13359 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13361 /* Handle aligned clauses by replacing default defs of the aligned
13362 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13363 lhs. Handle linear by adding PHIs. */
13364 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
13365 if (node
->simdclone
->args
[i
].alignment
13366 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13367 && (node
->simdclone
->args
[i
].alignment
13368 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
13369 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13372 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
13373 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13374 tree def
= ssa_default_def (cfun
, orig_arg
);
13375 if (def
&& !has_zero_uses (def
))
13377 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
13378 gimple_seq seq
= NULL
;
13379 bool need_cvt
= false;
13381 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
13383 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
13386 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
13387 gimple_call_set_lhs (g
, t
);
13388 gimple_seq_add_stmt_without_update (&seq
, g
);
13391 t
= make_ssa_name (orig_arg
);
13392 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
13393 gimple_seq_add_stmt_without_update (&seq
, g
);
13395 gsi_insert_seq_on_edge_immediate
13396 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
13398 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13399 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
13401 node
->create_edge (cgraph_node::get_create (fn
),
13402 call
, entry_bb
->count
, freq
);
13404 imm_use_iterator iter
;
13405 use_operand_p use_p
;
13407 tree repl
= gimple_get_lhs (g
);
13408 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13409 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
13412 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13413 SET_USE (use_p
, repl
);
13416 else if (node
->simdclone
->args
[i
].arg_type
13417 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
13419 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13420 tree def
= ssa_default_def (cfun
, orig_arg
);
13421 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13422 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
13423 if (def
&& !has_zero_uses (def
))
13425 iter1
= make_ssa_name (orig_arg
);
13426 iter2
= make_ssa_name (orig_arg
);
13427 phi
= create_phi_node (iter1
, body_bb
);
13428 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
13429 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13430 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13431 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
13432 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13433 ? TREE_TYPE (orig_arg
) : sizetype
;
13435 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
13436 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
13437 gsi
= gsi_last_bb (incr_bb
);
13438 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13440 imm_use_iterator iter
;
13441 use_operand_p use_p
;
13443 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13444 if (use_stmt
== phi
)
13447 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13448 SET_USE (use_p
, iter1
);
13452 calculate_dominance_info (CDI_DOMINATORS
);
13453 add_loop (loop
, loop
->header
->loop_father
);
13454 update_ssa (TODO_update_ssa
);
13459 /* If the function in NODE is tagged as an elemental SIMD function,
13460 create the appropriate SIMD clones. */
13463 expand_simd_clones (struct cgraph_node
*node
)
13465 tree attr
= lookup_attribute ("omp declare simd",
13466 DECL_ATTRIBUTES (node
->decl
));
13467 if (attr
== NULL_TREE
13468 || node
->global
.inlined_to
13469 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
13473 #pragma omp declare simd
13475 in C, there we don't know the argument types at all. */
13476 if (!node
->definition
13477 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
13482 /* Start with parsing the "omp declare simd" attribute(s). */
13483 bool inbranch_clause_specified
;
13484 struct cgraph_simd_clone
*clone_info
13485 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
13486 &inbranch_clause_specified
);
13487 if (clone_info
== NULL
)
13490 int orig_simdlen
= clone_info
->simdlen
;
13491 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
13492 /* The target can return 0 (no simd clones should be created),
13493 1 (just one ISA of simd clones should be created) or higher
13494 count of ISA variants. In that case, clone_info is initialized
13495 for the first ISA variant. */
13497 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
13502 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13503 also create one inbranch and one !inbranch clone of it. */
13504 for (int i
= 0; i
< count
* 2; i
++)
13506 struct cgraph_simd_clone
*clone
= clone_info
;
13507 if (inbranch_clause_specified
&& (i
& 1) != 0)
13512 clone
= simd_clone_struct_alloc (clone_info
->nargs
13514 simd_clone_struct_copy (clone
, clone_info
);
13515 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13516 and simd_clone_adjust_argument_types did to the first
13518 clone
->nargs
-= clone_info
->inbranch
;
13519 clone
->simdlen
= orig_simdlen
;
13520 /* And call the target hook again to get the right ISA. */
13521 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
13525 clone
->inbranch
= 1;
13528 /* simd_clone_mangle might fail if such a clone has been created
13530 tree id
= simd_clone_mangle (node
, clone
);
13531 if (id
== NULL_TREE
)
13534 /* Only when we are sure we want to create the clone actually
13535 clone the function (or definitions) or create another
13536 extern FUNCTION_DECL (for prototypes without definitions). */
13537 struct cgraph_node
*n
= simd_clone_create (node
);
13541 n
->simdclone
= clone
;
13542 clone
->origin
= node
;
13543 clone
->next_clone
= NULL
;
13544 if (node
->simd_clones
== NULL
)
13546 clone
->prev_clone
= n
;
13547 node
->simd_clones
= n
;
13551 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
13552 clone
->prev_clone
->simdclone
->next_clone
= n
;
13553 node
->simd_clones
->simdclone
->prev_clone
= n
;
13555 symtab
->change_decl_assembler_name (n
->decl
, id
);
13556 /* And finally adjust the return type, parameters and for
13557 definitions also function body. */
13558 if (node
->definition
)
13559 simd_clone_adjust (n
);
13562 simd_clone_adjust_return_type (n
);
13563 simd_clone_adjust_argument_types (n
);
13567 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
13570 /* Entry point for IPA simd clone creation pass. */
13572 static unsigned int
13573 ipa_omp_simd_clone (void)
13575 struct cgraph_node
*node
;
13576 FOR_EACH_FUNCTION (node
)
13577 expand_simd_clones (node
);
13583 const pass_data pass_data_omp_simd_clone
=
13585 SIMPLE_IPA_PASS
, /* type */
13586 "simdclone", /* name */
13587 OPTGROUP_NONE
, /* optinfo_flags */
13588 TV_NONE
, /* tv_id */
13589 ( PROP_ssa
| PROP_cfg
), /* properties_required */
13590 0, /* properties_provided */
13591 0, /* properties_destroyed */
13592 0, /* todo_flags_start */
13593 0, /* todo_flags_finish */
13596 class pass_omp_simd_clone
: public simple_ipa_opt_pass
13599 pass_omp_simd_clone(gcc::context
*ctxt
)
13600 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
13603 /* opt_pass methods: */
13604 virtual bool gate (function
*);
13605 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
13609 pass_omp_simd_clone::gate (function
*)
13611 return ((flag_openmp
|| flag_openmp_simd
13613 || (in_lto_p
&& !flag_wpa
))
13614 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
13617 } // anon namespace
13619 simple_ipa_opt_pass
*
13620 make_pass_omp_simd_clone (gcc::context
*ctxt
)
13622 return new pass_omp_simd_clone (ctxt
);
13625 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
13626 adds their addresses and sizes to constructor-vector V_CTOR. */
13628 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
13629 vec
<constructor_elt
, va_gc
> *v_ctor
)
13631 unsigned len
= vec_safe_length (v_decls
);
13632 for (unsigned i
= 0; i
< len
; i
++)
13634 tree it
= (*v_decls
)[i
];
13635 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
13637 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
13639 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
13640 fold_convert (const_ptr_type_node
,
13641 DECL_SIZE_UNIT (it
)));
13645 /* Create new symbols containing (address, size) pairs for global variables,
13646 marked with "omp declare target" attribute, as well as addresses for the
13647 functions, which are outlined offloading regions. */
13649 omp_finish_file (void)
13651 unsigned num_funcs
= vec_safe_length (offload_funcs
);
13652 unsigned num_vars
= vec_safe_length (offload_vars
);
13654 if (num_funcs
== 0 && num_vars
== 0)
13657 if (targetm_common
.have_named_sections
)
13659 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
13660 vec_alloc (v_f
, num_funcs
);
13661 vec_alloc (v_v
, num_vars
* 2);
13663 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
13664 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
13666 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13668 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13670 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13671 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13672 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
13673 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
13674 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
13675 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
13676 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13677 get_identifier (".offload_func_table"),
13679 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13680 get_identifier (".offload_var_table"),
13682 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
13683 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
13684 otherwise a joint table in a binary will contain padding between
13685 tables from multiple object files. */
13686 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
13687 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
13688 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
13689 DECL_INITIAL (funcs_decl
) = ctor_f
;
13690 DECL_INITIAL (vars_decl
) = ctor_v
;
13691 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
13692 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
13694 varpool_node::finalize_decl (vars_decl
);
13695 varpool_node::finalize_decl (funcs_decl
);
13699 for (unsigned i
= 0; i
< num_funcs
; i
++)
13701 tree it
= (*offload_funcs
)[i
];
13702 targetm
.record_offload_symbol (it
);
13704 for (unsigned i
= 0; i
< num_vars
; i
++)
13706 tree it
= (*offload_vars
)[i
];
13707 targetm
.record_offload_symbol (it
);
13712 #include "gt-omp-low.h"