1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
37 #include "hard-reg-set.h"
40 #include "dominance.h"
43 #include "basic-block.h"
44 #include "tree-ssa-alias.h"
45 #include "internal-fn.h"
46 #include "gimple-fold.h"
47 #include "gimple-expr.h"
51 #include "gimple-iterator.h"
52 #include "gimplify-me.h"
53 #include "gimple-walk.h"
54 #include "tree-iterator.h"
55 #include "tree-inline.h"
56 #include "langhooks.h"
57 #include "diagnostic-core.h"
58 #include "gimple-ssa.h"
60 #include "plugin-api.h"
64 #include "tree-phinodes.h"
65 #include "ssa-iterators.h"
66 #include "tree-ssanames.h"
67 #include "tree-into-ssa.h"
73 #include "tree-pass.h"
75 #include "splay-tree.h"
80 #include "gimple-low.h"
81 #include "tree-cfgcleanup.h"
82 #include "pretty-print.h"
83 #include "alloc-pool.h"
85 #include "tree-nested.h"
90 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
91 phases. The first phase scans the function looking for OMP statements
92 and then for variables that must be replaced to satisfy data sharing
93 clauses. The second phase expands code for the constructs, as well as
94 re-gimplifying things when variables have been replaced with complex
97 Final code generation is done by pass_expand_omp. The flowgraph is
98 scanned for parallel regions which are then moved to a new
99 function, to be invoked by the thread library. */
101 /* Parallel region information. Every parallel and workshare
102 directive is enclosed between two markers, the OMP_* directive
103 and a corresponding OMP_RETURN statement. */
107 /* The enclosing region. */
108 struct omp_region
*outer
;
110 /* First child region. */
111 struct omp_region
*inner
;
113 /* Next peer region. */
114 struct omp_region
*next
;
116 /* Block containing the omp directive as its last stmt. */
119 /* Block containing the OMP_RETURN as its last stmt. */
122 /* Block containing the OMP_CONTINUE as its last stmt. */
125 /* If this is a combined parallel+workshare region, this is a list
126 of additional arguments needed by the combined parallel+workshare
128 vec
<tree
, va_gc
> *ws_args
;
130 /* The code for the omp directive of this region. */
131 enum gimple_code type
;
133 /* Schedule kind, only used for OMP_FOR type regions. */
134 enum omp_clause_schedule_kind sched_kind
;
136 /* True if this is a combined parallel+workshare region. */
137 bool is_combined_parallel
;
140 /* Context structure. Used to store information about each parallel
141 directive in the code. */
143 typedef struct omp_context
145 /* This field must be at the beginning, as we do "inheritance": Some
146 callback functions for tree-inline.c (e.g., omp_copy_decl)
147 receive a copy_body_data pointer that is up-casted to an
148 omp_context pointer. */
151 /* The tree of contexts corresponding to the encountered constructs. */
152 struct omp_context
*outer
;
155 /* Map variables to fields in a structure that allows communication
156 between sending and receiving threads. */
157 splay_tree field_map
;
162 /* These are used just by task contexts, if task firstprivate fn is
163 needed. srecord_type is used to communicate from the thread
164 that encountered the task construct to task firstprivate fn,
165 record_type is allocated by GOMP_task, initialized by task firstprivate
166 fn and passed to the task body fn. */
167 splay_tree sfield_map
;
170 /* A chain of variables to add to the top-level block surrounding the
171 construct. In the case of a parallel, this is in the child function. */
174 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
175 barriers should jump to during omplower pass. */
178 /* What to do with variables with implicitly determined sharing
180 enum omp_clause_default_kind default_kind
;
182 /* Nesting depth of this context. Used to beautify error messages re
183 invalid gotos. The outermost ctx is depth 1, with depth 0 being
184 reserved for the main body of the function. */
187 /* True if this parallel directive is nested within another. */
190 /* True if this construct can be cancelled. */
195 struct omp_for_data_loop
197 tree v
, n1
, n2
, step
;
198 enum tree_code cond_code
;
201 /* A structure describing the main elements of a parallel loop. */
205 struct omp_for_data_loop loop
;
210 bool have_nowait
, have_ordered
;
211 enum omp_clause_schedule_kind sched_kind
;
212 struct omp_for_data_loop
*loops
;
216 static splay_tree all_contexts
;
217 static int taskreg_nesting_level
;
218 static int target_nesting_level
;
219 static struct omp_region
*root_omp_region
;
220 static bitmap task_shared_vars
;
221 static vec
<omp_context
*> taskreg_contexts
;
223 static void scan_omp (gimple_seq
*, omp_context
*);
224 static tree
scan_omp_1_op (tree
*, int *, void *);
226 #define WALK_SUBSTMTS \
230 case GIMPLE_EH_FILTER: \
231 case GIMPLE_TRANSACTION: \
232 /* The sub-statements for these should be walked. */ \
233 *handled_ops_p = false; \
236 /* Convenience function for calling scan_omp_1_op on tree operands. */
239 scan_omp_op (tree
*tp
, omp_context
*ctx
)
241 struct walk_stmt_info wi
;
243 memset (&wi
, 0, sizeof (wi
));
245 wi
.want_locations
= true;
247 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
250 static void lower_omp (gimple_seq
*, omp_context
*);
251 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
252 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
254 /* Find an OpenMP clause of type KIND within CLAUSES. */
257 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
259 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
260 if (OMP_CLAUSE_CODE (clauses
) == kind
)
266 /* Return true if CTX is for an omp parallel. */
269 is_parallel_ctx (omp_context
*ctx
)
271 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
275 /* Return true if CTX is for an omp task. */
278 is_task_ctx (omp_context
*ctx
)
280 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
284 /* Return true if CTX is for an omp parallel or omp task. */
287 is_taskreg_ctx (omp_context
*ctx
)
289 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
290 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
294 /* Return true if REGION is a combined parallel+workshare region. */
297 is_combined_parallel (struct omp_region
*region
)
299 return region
->is_combined_parallel
;
303 /* Extract the header elements of parallel loop FOR_STMT and store
307 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
308 struct omp_for_data_loop
*loops
)
310 tree t
, var
, *collapse_iter
, *collapse_count
;
311 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
312 struct omp_for_data_loop
*loop
;
314 struct omp_for_data_loop dummy_loop
;
315 location_t loc
= gimple_location (for_stmt
);
316 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
317 bool distribute
= gimple_omp_for_kind (for_stmt
)
318 == GF_OMP_FOR_KIND_DISTRIBUTE
;
320 fd
->for_stmt
= for_stmt
;
322 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
323 if (fd
->collapse
> 1)
326 fd
->loops
= &fd
->loop
;
328 fd
->have_nowait
= distribute
|| simd
;
329 fd
->have_ordered
= false;
330 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
331 fd
->chunk_size
= NULL_TREE
;
332 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
333 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
334 collapse_iter
= NULL
;
335 collapse_count
= NULL
;
337 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
338 switch (OMP_CLAUSE_CODE (t
))
340 case OMP_CLAUSE_NOWAIT
:
341 fd
->have_nowait
= true;
343 case OMP_CLAUSE_ORDERED
:
344 fd
->have_ordered
= true;
346 case OMP_CLAUSE_SCHEDULE
:
347 gcc_assert (!distribute
);
348 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
349 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
351 case OMP_CLAUSE_DIST_SCHEDULE
:
352 gcc_assert (distribute
);
353 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
355 case OMP_CLAUSE_COLLAPSE
:
356 if (fd
->collapse
> 1)
358 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
359 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
366 /* FIXME: for now map schedule(auto) to schedule(static).
367 There should be analysis to determine whether all iterations
368 are approximately the same amount of work (then schedule(static)
369 is best) or if it varies (then schedule(dynamic,N) is better). */
370 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
372 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
373 gcc_assert (fd
->chunk_size
== NULL
);
375 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
376 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
377 gcc_assert (fd
->chunk_size
== NULL
);
378 else if (fd
->chunk_size
== NULL
)
380 /* We only need to compute a default chunk size for ordered
381 static loops and dynamic loops. */
382 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
384 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
385 ? integer_zero_node
: integer_one_node
;
388 for (i
= 0; i
< fd
->collapse
; i
++)
390 if (fd
->collapse
== 1)
392 else if (loops
!= NULL
)
397 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
398 gcc_assert (SSA_VAR_P (loop
->v
));
399 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
400 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
401 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
402 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
404 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
405 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
406 switch (loop
->cond_code
)
412 gcc_assert (gimple_omp_for_kind (for_stmt
)
413 == GF_OMP_FOR_KIND_CILKSIMD
414 || (gimple_omp_for_kind (for_stmt
)
415 == GF_OMP_FOR_KIND_CILKFOR
));
418 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
419 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
421 loop
->n2
= fold_build2_loc (loc
,
422 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
423 build_int_cst (TREE_TYPE (loop
->n2
), 1));
424 loop
->cond_code
= LT_EXPR
;
427 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
428 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
430 loop
->n2
= fold_build2_loc (loc
,
431 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
432 build_int_cst (TREE_TYPE (loop
->n2
), 1));
433 loop
->cond_code
= GT_EXPR
;
439 t
= gimple_omp_for_incr (for_stmt
, i
);
440 gcc_assert (TREE_OPERAND (t
, 0) == var
);
441 switch (TREE_CODE (t
))
444 loop
->step
= TREE_OPERAND (t
, 1);
446 case POINTER_PLUS_EXPR
:
447 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
450 loop
->step
= TREE_OPERAND (t
, 1);
451 loop
->step
= fold_build1_loc (loc
,
452 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
460 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
461 && !fd
->have_ordered
))
463 if (fd
->collapse
== 1)
464 iter_type
= TREE_TYPE (loop
->v
);
466 || TYPE_PRECISION (iter_type
)
467 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
469 = build_nonstandard_integer_type
470 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
472 else if (iter_type
!= long_long_unsigned_type_node
)
474 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
475 iter_type
= long_long_unsigned_type_node
;
476 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
477 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
478 >= TYPE_PRECISION (iter_type
))
482 if (loop
->cond_code
== LT_EXPR
)
483 n
= fold_build2_loc (loc
,
484 PLUS_EXPR
, TREE_TYPE (loop
->v
),
485 loop
->n2
, loop
->step
);
488 if (TREE_CODE (n
) != INTEGER_CST
489 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
490 iter_type
= long_long_unsigned_type_node
;
492 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
493 > TYPE_PRECISION (iter_type
))
497 if (loop
->cond_code
== LT_EXPR
)
500 n2
= fold_build2_loc (loc
,
501 PLUS_EXPR
, TREE_TYPE (loop
->v
),
502 loop
->n2
, loop
->step
);
506 n1
= fold_build2_loc (loc
,
507 MINUS_EXPR
, TREE_TYPE (loop
->v
),
508 loop
->n2
, loop
->step
);
511 if (TREE_CODE (n1
) != INTEGER_CST
512 || TREE_CODE (n2
) != INTEGER_CST
513 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
514 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
515 iter_type
= long_long_unsigned_type_node
;
519 if (collapse_count
&& *collapse_count
== NULL
)
521 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
522 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
523 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
524 if (t
&& integer_zerop (t
))
525 count
= build_zero_cst (long_long_unsigned_type_node
);
526 else if ((i
== 0 || count
!= NULL_TREE
)
527 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
528 && TREE_CONSTANT (loop
->n1
)
529 && TREE_CONSTANT (loop
->n2
)
530 && TREE_CODE (loop
->step
) == INTEGER_CST
)
532 tree itype
= TREE_TYPE (loop
->v
);
534 if (POINTER_TYPE_P (itype
))
535 itype
= signed_type_for (itype
);
536 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
537 t
= fold_build2_loc (loc
,
539 fold_convert_loc (loc
, itype
, loop
->step
), t
);
540 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
541 fold_convert_loc (loc
, itype
, loop
->n2
));
542 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
543 fold_convert_loc (loc
, itype
, loop
->n1
));
544 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
545 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
546 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
547 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
548 fold_convert_loc (loc
, itype
,
551 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
552 fold_convert_loc (loc
, itype
, loop
->step
));
553 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
554 if (count
!= NULL_TREE
)
555 count
= fold_build2_loc (loc
,
556 MULT_EXPR
, long_long_unsigned_type_node
,
560 if (TREE_CODE (count
) != INTEGER_CST
)
563 else if (count
&& !integer_zerop (count
))
570 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
571 || fd
->have_ordered
))
573 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
574 iter_type
= long_long_unsigned_type_node
;
576 iter_type
= long_integer_type_node
;
578 else if (collapse_iter
&& *collapse_iter
!= NULL
)
579 iter_type
= TREE_TYPE (*collapse_iter
);
580 fd
->iter_type
= iter_type
;
581 if (collapse_iter
&& *collapse_iter
== NULL
)
582 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
583 if (collapse_count
&& *collapse_count
== NULL
)
586 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
588 *collapse_count
= create_tmp_var (iter_type
, ".count");
591 if (fd
->collapse
> 1)
593 fd
->loop
.v
= *collapse_iter
;
594 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
595 fd
->loop
.n2
= *collapse_count
;
596 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
597 fd
->loop
.cond_code
= LT_EXPR
;
602 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
603 is the immediate dominator of PAR_ENTRY_BB, return true if there
604 are no data dependencies that would prevent expanding the parallel
605 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
607 When expanding a combined parallel+workshare region, the call to
608 the child function may need additional arguments in the case of
609 GIMPLE_OMP_FOR regions. In some cases, these arguments are
610 computed out of variables passed in from the parent to the child
611 via 'struct .omp_data_s'. For instance:
613 #pragma omp parallel for schedule (guided, i * 4)
618 # BLOCK 2 (PAR_ENTRY_BB)
620 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
622 # BLOCK 3 (WS_ENTRY_BB)
623 .omp_data_i = &.omp_data_o;
624 D.1667 = .omp_data_i->i;
626 #pragma omp for schedule (guided, D.1598)
628 When we outline the parallel region, the call to the child function
629 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
630 that value is computed *after* the call site. So, in principle we
631 cannot do the transformation.
633 To see whether the code in WS_ENTRY_BB blocks the combined
634 parallel+workshare call, we collect all the variables used in the
635 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
636 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
639 FIXME. If we had the SSA form built at this point, we could merely
640 hoist the code in block 3 into block 2 and be done with it. But at
641 this point we don't have dataflow information and though we could
642 hack something up here, it is really not worth the aggravation. */
645 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
647 struct omp_for_data fd
;
648 gimple ws_stmt
= last_stmt (ws_entry_bb
);
650 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
653 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
655 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
657 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
659 if (fd
.iter_type
!= long_integer_type_node
)
662 /* FIXME. We give up too easily here. If any of these arguments
663 are not constants, they will likely involve variables that have
664 been mapped into fields of .omp_data_s for sharing with the child
665 function. With appropriate data flow, it would be possible to
667 if (!is_gimple_min_invariant (fd
.loop
.n1
)
668 || !is_gimple_min_invariant (fd
.loop
.n2
)
669 || !is_gimple_min_invariant (fd
.loop
.step
)
670 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
677 /* Collect additional arguments needed to emit a combined
678 parallel+workshare call. WS_STMT is the workshare directive being
681 static vec
<tree
, va_gc
> *
682 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
685 location_t loc
= gimple_location (ws_stmt
);
686 vec
<tree
, va_gc
> *ws_args
;
688 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
690 struct omp_for_data fd
;
693 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
697 if (gimple_omp_for_combined_into_p (ws_stmt
))
700 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
701 OMP_CLAUSE__LOOPTEMP_
);
703 n1
= OMP_CLAUSE_DECL (innerc
);
704 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
705 OMP_CLAUSE__LOOPTEMP_
);
707 n2
= OMP_CLAUSE_DECL (innerc
);
710 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
712 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
713 ws_args
->quick_push (t
);
715 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
716 ws_args
->quick_push (t
);
718 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
719 ws_args
->quick_push (t
);
723 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
724 ws_args
->quick_push (t
);
729 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
731 /* Number of sections is equal to the number of edges from the
732 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
733 the exit of the sections region. */
734 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
735 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
736 vec_alloc (ws_args
, 1);
737 ws_args
->quick_push (t
);
745 /* Discover whether REGION is a combined parallel+workshare region. */
748 determine_parallel_type (struct omp_region
*region
)
750 basic_block par_entry_bb
, par_exit_bb
;
751 basic_block ws_entry_bb
, ws_exit_bb
;
753 if (region
== NULL
|| region
->inner
== NULL
754 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
755 || region
->inner
->cont
== NULL
)
758 /* We only support parallel+for and parallel+sections. */
759 if (region
->type
!= GIMPLE_OMP_PARALLEL
760 || (region
->inner
->type
!= GIMPLE_OMP_FOR
761 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
764 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
765 WS_EXIT_BB -> PAR_EXIT_BB. */
766 par_entry_bb
= region
->entry
;
767 par_exit_bb
= region
->exit
;
768 ws_entry_bb
= region
->inner
->entry
;
769 ws_exit_bb
= region
->inner
->exit
;
771 if (single_succ (par_entry_bb
) == ws_entry_bb
772 && single_succ (ws_exit_bb
) == par_exit_bb
773 && workshare_safe_to_combine_p (ws_entry_bb
)
774 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
775 || (last_and_only_stmt (ws_entry_bb
)
776 && last_and_only_stmt (par_exit_bb
))))
778 gimple par_stmt
= last_stmt (par_entry_bb
);
779 gimple ws_stmt
= last_stmt (ws_entry_bb
);
781 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
783 /* If this is a combined parallel loop, we need to determine
784 whether or not to use the combined library calls. There
785 are two cases where we do not apply the transformation:
786 static loops and any kind of ordered loop. In the first
787 case, we already open code the loop so there is no need
788 to do anything else. In the latter case, the combined
789 parallel loop call would still need extra synchronization
790 to implement ordered semantics, so there would not be any
791 gain in using the combined call. */
792 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
793 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
795 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
796 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
798 region
->is_combined_parallel
= false;
799 region
->inner
->is_combined_parallel
= false;
804 region
->is_combined_parallel
= true;
805 region
->inner
->is_combined_parallel
= true;
806 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
811 /* Return true if EXPR is variable sized. */
814 is_variable_sized (const_tree expr
)
816 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
819 /* Return true if DECL is a reference type. */
822 is_reference (tree decl
)
824 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
827 /* Lookup variables in the decl or field splay trees. The "maybe" form
828 allows for the variable form to not have been entered, otherwise we
829 assert that the variable must have been entered. */
832 lookup_decl (tree var
, omp_context
*ctx
)
834 tree
*n
= ctx
->cb
.decl_map
->get (var
);
839 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
841 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
842 return n
? *n
: NULL_TREE
;
846 lookup_field (tree var
, omp_context
*ctx
)
849 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
850 return (tree
) n
->value
;
854 lookup_sfield (tree var
, omp_context
*ctx
)
857 n
= splay_tree_lookup (ctx
->sfield_map
858 ? ctx
->sfield_map
: ctx
->field_map
,
859 (splay_tree_key
) var
);
860 return (tree
) n
->value
;
864 maybe_lookup_field (tree var
, omp_context
*ctx
)
867 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
868 return n
? (tree
) n
->value
: NULL_TREE
;
871 /* Return true if DECL should be copied by pointer. SHARED_CTX is
872 the parallel context if DECL is to be shared. */
875 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
877 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
880 /* We can only use copy-in/copy-out semantics for shared variables
881 when we know the value is not accessible from an outer scope. */
884 /* ??? Trivially accessible from anywhere. But why would we even
885 be passing an address in this case? Should we simply assert
886 this to be false, or should we have a cleanup pass that removes
887 these from the list of mappings? */
888 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
891 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
892 without analyzing the expression whether or not its location
893 is accessible to anyone else. In the case of nested parallel
894 regions it certainly may be. */
895 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
898 /* Do not use copy-in/copy-out for variables that have their
900 if (TREE_ADDRESSABLE (decl
))
903 /* lower_send_shared_vars only uses copy-in, but not copy-out
905 if (TREE_READONLY (decl
)
906 || ((TREE_CODE (decl
) == RESULT_DECL
907 || TREE_CODE (decl
) == PARM_DECL
)
908 && DECL_BY_REFERENCE (decl
)))
911 /* Disallow copy-in/out in nested parallel if
912 decl is shared in outer parallel, otherwise
913 each thread could store the shared variable
914 in its own copy-in location, making the
915 variable no longer really shared. */
916 if (shared_ctx
->is_nested
)
920 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
921 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
928 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
929 c
; c
= OMP_CLAUSE_CHAIN (c
))
930 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
931 && OMP_CLAUSE_DECL (c
) == decl
)
935 goto maybe_mark_addressable_and_ret
;
939 /* For tasks avoid using copy-in/out. As tasks can be
940 deferred or executed in different thread, when GOMP_task
941 returns, the task hasn't necessarily terminated. */
942 if (is_task_ctx (shared_ctx
))
945 maybe_mark_addressable_and_ret
:
946 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
947 if (is_gimple_reg (outer
))
949 /* Taking address of OUTER in lower_send_shared_vars
950 might need regimplification of everything that uses the
952 if (!task_shared_vars
)
953 task_shared_vars
= BITMAP_ALLOC (NULL
);
954 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
955 TREE_ADDRESSABLE (outer
) = 1;
964 /* Construct a new automatic decl similar to VAR. */
967 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
969 tree copy
= copy_var_decl (var
, name
, type
);
971 DECL_CONTEXT (copy
) = current_function_decl
;
972 DECL_CHAIN (copy
) = ctx
->block_vars
;
973 ctx
->block_vars
= copy
;
979 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
981 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
984 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
987 omp_build_component_ref (tree obj
, tree field
)
989 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
990 if (TREE_THIS_VOLATILE (field
))
991 TREE_THIS_VOLATILE (ret
) |= 1;
992 if (TREE_READONLY (field
))
993 TREE_READONLY (ret
) |= 1;
997 /* Build tree nodes to access the field for VAR on the receiver side. */
1000 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1002 tree x
, field
= lookup_field (var
, ctx
);
1004 /* If the receiver record type was remapped in the child function,
1005 remap the field into the new record type. */
1006 x
= maybe_lookup_field (field
, ctx
);
1010 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1011 x
= omp_build_component_ref (x
, field
);
1013 x
= build_simple_mem_ref (x
);
1018 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1019 of a parallel, this is a component reference; for workshare constructs
1020 this is some variable. */
1023 build_outer_var_ref (tree var
, omp_context
*ctx
)
1027 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1029 else if (is_variable_sized (var
))
1031 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1032 x
= build_outer_var_ref (x
, ctx
);
1033 x
= build_simple_mem_ref (x
);
1035 else if (is_taskreg_ctx (ctx
))
1037 bool by_ref
= use_pointer_for_field (var
, NULL
);
1038 x
= build_receiver_ref (var
, by_ref
, ctx
);
1040 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1041 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1043 /* #pragma omp simd isn't a worksharing construct, and can reference even
1044 private vars in its linear etc. clauses. */
1046 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1047 x
= lookup_decl (var
, ctx
->outer
);
1048 else if (ctx
->outer
)
1049 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1053 else if (ctx
->outer
)
1054 x
= lookup_decl (var
, ctx
->outer
);
1055 else if (is_reference (var
))
1056 /* This can happen with orphaned constructs. If var is reference, it is
1057 possible it is shared and as such valid. */
1062 if (is_reference (var
))
1063 x
= build_simple_mem_ref (x
);
1068 /* Build tree nodes to access the field for VAR on the sender side. */
1071 build_sender_ref (tree var
, omp_context
*ctx
)
1073 tree field
= lookup_sfield (var
, ctx
);
1074 return omp_build_component_ref (ctx
->sender_decl
, field
);
1077 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1080 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1082 tree field
, type
, sfield
= NULL_TREE
;
1084 gcc_assert ((mask
& 1) == 0
1085 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1086 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1087 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1089 type
= TREE_TYPE (var
);
1092 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1093 type
= build_pointer_type (build_pointer_type (type
));
1096 type
= build_pointer_type (type
);
1097 else if ((mask
& 3) == 1 && is_reference (var
))
1098 type
= TREE_TYPE (type
);
1100 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1101 FIELD_DECL
, DECL_NAME (var
), type
);
1103 /* Remember what variable this field was created for. This does have a
1104 side effect of making dwarf2out ignore this member, so for helpful
1105 debugging we clear it later in delete_omp_context. */
1106 DECL_ABSTRACT_ORIGIN (field
) = var
;
1107 if (type
== TREE_TYPE (var
))
1109 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1110 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1111 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1114 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1116 if ((mask
& 3) == 3)
1118 insert_field_into_struct (ctx
->record_type
, field
);
1119 if (ctx
->srecord_type
)
1121 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1122 FIELD_DECL
, DECL_NAME (var
), type
);
1123 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1124 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1125 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1126 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1127 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1132 if (ctx
->srecord_type
== NULL_TREE
)
1136 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1137 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1138 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1140 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1141 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1142 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1143 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1144 splay_tree_insert (ctx
->sfield_map
,
1145 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1146 (splay_tree_value
) sfield
);
1150 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1151 : ctx
->srecord_type
, field
);
1155 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1156 (splay_tree_value
) field
);
1157 if ((mask
& 2) && ctx
->sfield_map
)
1158 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1159 (splay_tree_value
) sfield
);
1163 install_var_local (tree var
, omp_context
*ctx
)
1165 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1166 insert_decl_map (&ctx
->cb
, var
, new_var
);
1170 /* Adjust the replacement for DECL in CTX for the new context. This means
1171 copying the DECL_VALUE_EXPR, and fixing up the type. */
1174 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1176 tree new_decl
, size
;
1178 new_decl
= lookup_decl (decl
, ctx
);
1180 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1182 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1183 && DECL_HAS_VALUE_EXPR_P (decl
))
1185 tree ve
= DECL_VALUE_EXPR (decl
);
1186 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1187 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1188 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1191 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1193 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1194 if (size
== error_mark_node
)
1195 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1196 DECL_SIZE (new_decl
) = size
;
1198 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1199 if (size
== error_mark_node
)
1200 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1201 DECL_SIZE_UNIT (new_decl
) = size
;
1205 /* The callback for remap_decl. Search all containing contexts for a
1206 mapping of the variable; this avoids having to duplicate the splay
1207 tree ahead of time. We know a mapping doesn't already exist in the
1208 given context. Create new mappings to implement default semantics. */
1211 omp_copy_decl (tree var
, copy_body_data
*cb
)
1213 omp_context
*ctx
= (omp_context
*) cb
;
1216 if (TREE_CODE (var
) == LABEL_DECL
)
1218 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1219 DECL_CONTEXT (new_var
) = current_function_decl
;
1220 insert_decl_map (&ctx
->cb
, var
, new_var
);
1224 while (!is_taskreg_ctx (ctx
))
1229 new_var
= maybe_lookup_decl (var
, ctx
);
1234 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1237 return error_mark_node
;
1241 /* Debugging dumps for parallel regions. */
1242 void dump_omp_region (FILE *, struct omp_region
*, int);
1243 void debug_omp_region (struct omp_region
*);
1244 void debug_all_omp_regions (void);
1246 /* Dump the parallel region tree rooted at REGION. */
1249 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1251 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1252 gimple_code_name
[region
->type
]);
1255 dump_omp_region (file
, region
->inner
, indent
+ 4);
1259 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1260 region
->cont
->index
);
1264 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1265 region
->exit
->index
);
1267 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1270 dump_omp_region (file
, region
->next
, indent
);
1274 debug_omp_region (struct omp_region
*region
)
1276 dump_omp_region (stderr
, region
, 0);
1280 debug_all_omp_regions (void)
1282 dump_omp_region (stderr
, root_omp_region
, 0);
1286 /* Create a new parallel region starting at STMT inside region PARENT. */
1288 static struct omp_region
*
1289 new_omp_region (basic_block bb
, enum gimple_code type
,
1290 struct omp_region
*parent
)
1292 struct omp_region
*region
= XCNEW (struct omp_region
);
1294 region
->outer
= parent
;
1296 region
->type
= type
;
1300 /* This is a nested region. Add it to the list of inner
1301 regions in PARENT. */
1302 region
->next
= parent
->inner
;
1303 parent
->inner
= region
;
1307 /* This is a toplevel region. Add it to the list of toplevel
1308 regions in ROOT_OMP_REGION. */
1309 region
->next
= root_omp_region
;
1310 root_omp_region
= region
;
1316 /* Release the memory associated with the region tree rooted at REGION. */
1319 free_omp_region_1 (struct omp_region
*region
)
1321 struct omp_region
*i
, *n
;
1323 for (i
= region
->inner
; i
; i
= n
)
1326 free_omp_region_1 (i
);
1332 /* Release the memory for the entire omp region tree. */
1335 free_omp_regions (void)
1337 struct omp_region
*r
, *n
;
1338 for (r
= root_omp_region
; r
; r
= n
)
1341 free_omp_region_1 (r
);
1343 root_omp_region
= NULL
;
1347 /* Create a new context, with OUTER_CTX being the surrounding context. */
1349 static omp_context
*
1350 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1352 omp_context
*ctx
= XCNEW (omp_context
);
1354 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1355 (splay_tree_value
) ctx
);
1360 ctx
->outer
= outer_ctx
;
1361 ctx
->cb
= outer_ctx
->cb
;
1362 ctx
->cb
.block
= NULL
;
1363 ctx
->depth
= outer_ctx
->depth
+ 1;
1367 ctx
->cb
.src_fn
= current_function_decl
;
1368 ctx
->cb
.dst_fn
= current_function_decl
;
1369 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1370 gcc_checking_assert (ctx
->cb
.src_node
);
1371 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1372 ctx
->cb
.src_cfun
= cfun
;
1373 ctx
->cb
.copy_decl
= omp_copy_decl
;
1374 ctx
->cb
.eh_lp_nr
= 0;
1375 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1379 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1384 static gimple_seq
maybe_catch_exception (gimple_seq
);
1386 /* Finalize task copyfn. */
1389 finalize_task_copyfn (gimple task_stmt
)
1391 struct function
*child_cfun
;
1393 gimple_seq seq
= NULL
, new_seq
;
1396 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1397 if (child_fn
== NULL_TREE
)
1400 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1401 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1403 push_cfun (child_cfun
);
1404 bind
= gimplify_body (child_fn
, false);
1405 gimple_seq_add_stmt (&seq
, bind
);
1406 new_seq
= maybe_catch_exception (seq
);
1409 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1411 gimple_seq_add_stmt (&seq
, bind
);
1413 gimple_set_body (child_fn
, seq
);
1416 /* Inform the callgraph about the new function. */
1417 cgraph_node::add_new_function (child_fn
, false);
1420 /* Destroy a omp_context data structures. Called through the splay tree
1421 value delete callback. */
1424 delete_omp_context (splay_tree_value value
)
1426 omp_context
*ctx
= (omp_context
*) value
;
1428 delete ctx
->cb
.decl_map
;
1431 splay_tree_delete (ctx
->field_map
);
1432 if (ctx
->sfield_map
)
1433 splay_tree_delete (ctx
->sfield_map
);
1435 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1436 it produces corrupt debug information. */
1437 if (ctx
->record_type
)
1440 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1441 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1443 if (ctx
->srecord_type
)
1446 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1447 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1450 if (is_task_ctx (ctx
))
1451 finalize_task_copyfn (ctx
->stmt
);
1456 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1460 fixup_child_record_type (omp_context
*ctx
)
1462 tree f
, type
= ctx
->record_type
;
1464 /* ??? It isn't sufficient to just call remap_type here, because
1465 variably_modified_type_p doesn't work the way we expect for
1466 record types. Testing each field for whether it needs remapping
1467 and creating a new record by hand works, however. */
1468 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1469 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1473 tree name
, new_fields
= NULL
;
1475 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1476 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1477 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1478 TYPE_DECL
, name
, type
);
1479 TYPE_NAME (type
) = name
;
1481 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1483 tree new_f
= copy_node (f
);
1484 DECL_CONTEXT (new_f
) = type
;
1485 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1486 DECL_CHAIN (new_f
) = new_fields
;
1487 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1488 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1490 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1494 /* Arrange to be able to look up the receiver field
1495 given the sender field. */
1496 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1497 (splay_tree_value
) new_f
);
1499 TYPE_FIELDS (type
) = nreverse (new_fields
);
1503 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1506 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1507 specified by CLAUSES. */
1510 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1513 bool scan_array_reductions
= false;
1515 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1519 switch (OMP_CLAUSE_CODE (c
))
1521 case OMP_CLAUSE_PRIVATE
:
1522 decl
= OMP_CLAUSE_DECL (c
);
1523 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1525 else if (!is_variable_sized (decl
))
1526 install_var_local (decl
, ctx
);
1529 case OMP_CLAUSE_SHARED
:
1530 decl
= OMP_CLAUSE_DECL (c
);
1531 /* Ignore shared directives in teams construct. */
1532 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1534 /* Global variables don't need to be copied,
1535 the receiver side will use them directly. */
1536 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1537 if (is_global_var (odecl
))
1539 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1542 gcc_assert (is_taskreg_ctx (ctx
));
1543 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1544 || !is_variable_sized (decl
));
1545 /* Global variables don't need to be copied,
1546 the receiver side will use them directly. */
1547 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1549 by_ref
= use_pointer_for_field (decl
, ctx
);
1550 if (! TREE_READONLY (decl
)
1551 || TREE_ADDRESSABLE (decl
)
1553 || is_reference (decl
))
1555 install_var_field (decl
, by_ref
, 3, ctx
);
1556 install_var_local (decl
, ctx
);
1559 /* We don't need to copy const scalar vars back. */
1560 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1563 case OMP_CLAUSE_LASTPRIVATE
:
1564 /* Let the corresponding firstprivate clause create
1566 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1570 case OMP_CLAUSE_FIRSTPRIVATE
:
1571 case OMP_CLAUSE_REDUCTION
:
1572 case OMP_CLAUSE_LINEAR
:
1573 decl
= OMP_CLAUSE_DECL (c
);
1575 if (is_variable_sized (decl
))
1577 if (is_task_ctx (ctx
))
1578 install_var_field (decl
, false, 1, ctx
);
1581 else if (is_taskreg_ctx (ctx
))
1584 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1585 by_ref
= use_pointer_for_field (decl
, NULL
);
1587 if (is_task_ctx (ctx
)
1588 && (global
|| by_ref
|| is_reference (decl
)))
1590 install_var_field (decl
, false, 1, ctx
);
1592 install_var_field (decl
, by_ref
, 2, ctx
);
1595 install_var_field (decl
, by_ref
, 3, ctx
);
1597 install_var_local (decl
, ctx
);
1600 case OMP_CLAUSE__LOOPTEMP_
:
1601 gcc_assert (is_parallel_ctx (ctx
));
1602 decl
= OMP_CLAUSE_DECL (c
);
1603 install_var_field (decl
, false, 3, ctx
);
1604 install_var_local (decl
, ctx
);
1607 case OMP_CLAUSE_COPYPRIVATE
:
1608 case OMP_CLAUSE_COPYIN
:
1609 decl
= OMP_CLAUSE_DECL (c
);
1610 by_ref
= use_pointer_for_field (decl
, NULL
);
1611 install_var_field (decl
, by_ref
, 3, ctx
);
1614 case OMP_CLAUSE_DEFAULT
:
1615 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1618 case OMP_CLAUSE_FINAL
:
1620 case OMP_CLAUSE_NUM_THREADS
:
1621 case OMP_CLAUSE_NUM_TEAMS
:
1622 case OMP_CLAUSE_THREAD_LIMIT
:
1623 case OMP_CLAUSE_DEVICE
:
1624 case OMP_CLAUSE_SCHEDULE
:
1625 case OMP_CLAUSE_DIST_SCHEDULE
:
1626 case OMP_CLAUSE_DEPEND
:
1627 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1629 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1633 case OMP_CLAUSE_FROM
:
1634 case OMP_CLAUSE_MAP
:
1636 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1637 decl
= OMP_CLAUSE_DECL (c
);
1638 /* Global variables with "omp declare target" attribute
1639 don't need to be copied, the receiver side will use them
1641 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1643 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1644 && lookup_attribute ("omp declare target",
1645 DECL_ATTRIBUTES (decl
)))
1647 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1648 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1650 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1651 #pragma omp target data, there is nothing to map for
1653 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1654 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1659 if (DECL_SIZE (decl
)
1660 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1662 tree decl2
= DECL_VALUE_EXPR (decl
);
1663 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1664 decl2
= TREE_OPERAND (decl2
, 0);
1665 gcc_assert (DECL_P (decl2
));
1666 install_var_field (decl2
, true, 3, ctx
);
1667 install_var_local (decl2
, ctx
);
1668 install_var_local (decl
, ctx
);
1672 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1673 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1674 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1675 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1676 install_var_field (decl
, true, 7, ctx
);
1678 install_var_field (decl
, true, 3, ctx
);
1679 if (gimple_omp_target_kind (ctx
->stmt
)
1680 == GF_OMP_TARGET_KIND_REGION
)
1681 install_var_local (decl
, ctx
);
1686 tree base
= get_base_address (decl
);
1687 tree nc
= OMP_CLAUSE_CHAIN (c
);
1690 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1691 && OMP_CLAUSE_DECL (nc
) == base
1692 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1693 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1695 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1696 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1702 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1703 decl
= OMP_CLAUSE_DECL (c
);
1705 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1706 (splay_tree_key
) decl
));
1708 = build_decl (OMP_CLAUSE_LOCATION (c
),
1709 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1710 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1711 insert_field_into_struct (ctx
->record_type
, field
);
1712 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1713 (splay_tree_value
) field
);
1718 case OMP_CLAUSE_NOWAIT
:
1719 case OMP_CLAUSE_ORDERED
:
1720 case OMP_CLAUSE_COLLAPSE
:
1721 case OMP_CLAUSE_UNTIED
:
1722 case OMP_CLAUSE_MERGEABLE
:
1723 case OMP_CLAUSE_PROC_BIND
:
1724 case OMP_CLAUSE_SAFELEN
:
1727 case OMP_CLAUSE_ALIGNED
:
1728 decl
= OMP_CLAUSE_DECL (c
);
1729 if (is_global_var (decl
)
1730 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1731 install_var_local (decl
, ctx
);
1739 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1741 switch (OMP_CLAUSE_CODE (c
))
1743 case OMP_CLAUSE_LASTPRIVATE
:
1744 /* Let the corresponding firstprivate clause create
1746 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1747 scan_array_reductions
= true;
1748 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1752 case OMP_CLAUSE_PRIVATE
:
1753 case OMP_CLAUSE_FIRSTPRIVATE
:
1754 case OMP_CLAUSE_REDUCTION
:
1755 case OMP_CLAUSE_LINEAR
:
1756 decl
= OMP_CLAUSE_DECL (c
);
1757 if (is_variable_sized (decl
))
1758 install_var_local (decl
, ctx
);
1759 fixup_remapped_decl (decl
, ctx
,
1760 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1761 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1762 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1763 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1764 scan_array_reductions
= true;
1765 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1766 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1767 scan_array_reductions
= true;
1770 case OMP_CLAUSE_SHARED
:
1771 /* Ignore shared directives in teams construct. */
1772 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1774 decl
= OMP_CLAUSE_DECL (c
);
1775 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1776 fixup_remapped_decl (decl
, ctx
, false);
1779 case OMP_CLAUSE_MAP
:
1780 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1782 decl
= OMP_CLAUSE_DECL (c
);
1784 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1785 && lookup_attribute ("omp declare target",
1786 DECL_ATTRIBUTES (decl
)))
1790 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1791 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1792 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1794 tree new_decl
= lookup_decl (decl
, ctx
);
1795 TREE_TYPE (new_decl
)
1796 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1798 else if (DECL_SIZE (decl
)
1799 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1801 tree decl2
= DECL_VALUE_EXPR (decl
);
1802 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1803 decl2
= TREE_OPERAND (decl2
, 0);
1804 gcc_assert (DECL_P (decl2
));
1805 fixup_remapped_decl (decl2
, ctx
, false);
1806 fixup_remapped_decl (decl
, ctx
, true);
1809 fixup_remapped_decl (decl
, ctx
, false);
1813 case OMP_CLAUSE_COPYPRIVATE
:
1814 case OMP_CLAUSE_COPYIN
:
1815 case OMP_CLAUSE_DEFAULT
:
1817 case OMP_CLAUSE_NUM_THREADS
:
1818 case OMP_CLAUSE_NUM_TEAMS
:
1819 case OMP_CLAUSE_THREAD_LIMIT
:
1820 case OMP_CLAUSE_DEVICE
:
1821 case OMP_CLAUSE_SCHEDULE
:
1822 case OMP_CLAUSE_DIST_SCHEDULE
:
1823 case OMP_CLAUSE_NOWAIT
:
1824 case OMP_CLAUSE_ORDERED
:
1825 case OMP_CLAUSE_COLLAPSE
:
1826 case OMP_CLAUSE_UNTIED
:
1827 case OMP_CLAUSE_FINAL
:
1828 case OMP_CLAUSE_MERGEABLE
:
1829 case OMP_CLAUSE_PROC_BIND
:
1830 case OMP_CLAUSE_SAFELEN
:
1831 case OMP_CLAUSE_ALIGNED
:
1832 case OMP_CLAUSE_DEPEND
:
1833 case OMP_CLAUSE__LOOPTEMP_
:
1835 case OMP_CLAUSE_FROM
:
1836 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1844 if (scan_array_reductions
)
1845 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1846 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1847 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1849 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1850 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1852 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1853 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1854 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1855 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1856 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1857 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
1860 /* Create a new name for omp child function. Returns an identifier. If
1861 IS_CILK_FOR is true then the suffix for the child function is
1865 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
1868 return clone_function_name (current_function_decl
, "_cilk_for_fn");
1869 return clone_function_name (current_function_decl
,
1870 task_copy
? "_omp_cpyfn" : "_omp_fn");
1873 /* Returns the type of the induction variable for the child function for
1874 _Cilk_for and the types for _high and _low variables based on TYPE. */
1877 cilk_for_check_loop_diff_type (tree type
)
1879 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
1881 if (TYPE_UNSIGNED (type
))
1882 return uint32_type_node
;
1884 return integer_type_node
;
1888 if (TYPE_UNSIGNED (type
))
1889 return uint64_type_node
;
1891 return long_long_integer_type_node
;
1895 /* Build a decl for the omp child function. It'll not contain a body
1896 yet, just the bare decl. */
1899 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1901 tree decl
, type
, name
, t
;
1904 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
1905 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
1906 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
1907 tree cilk_var_type
= NULL_TREE
;
1909 name
= create_omp_child_function_name (task_copy
,
1910 cilk_for_count
!= NULL_TREE
);
1912 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1913 ptr_type_node
, NULL_TREE
);
1914 else if (cilk_for_count
)
1916 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
1917 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
1918 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1919 cilk_var_type
, cilk_var_type
, NULL_TREE
);
1922 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1924 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
1927 ctx
->cb
.dst_fn
= decl
;
1929 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1931 TREE_STATIC (decl
) = 1;
1932 TREE_USED (decl
) = 1;
1933 DECL_ARTIFICIAL (decl
) = 1;
1934 DECL_IGNORED_P (decl
) = 0;
1935 TREE_PUBLIC (decl
) = 0;
1936 DECL_UNINLINABLE (decl
) = 1;
1937 DECL_EXTERNAL (decl
) = 0;
1938 DECL_CONTEXT (decl
) = NULL_TREE
;
1939 DECL_INITIAL (decl
) = make_node (BLOCK
);
1940 bool target_p
= false;
1941 if (lookup_attribute ("omp declare target",
1942 DECL_ATTRIBUTES (current_function_decl
)))
1947 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1948 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1949 && gimple_omp_target_kind (octx
->stmt
)
1950 == GF_OMP_TARGET_KIND_REGION
)
1957 DECL_ATTRIBUTES (decl
)
1958 = tree_cons (get_identifier ("omp declare target"),
1959 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1961 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1962 RESULT_DECL
, NULL_TREE
, void_type_node
);
1963 DECL_ARTIFICIAL (t
) = 1;
1964 DECL_IGNORED_P (t
) = 1;
1965 DECL_CONTEXT (t
) = decl
;
1966 DECL_RESULT (decl
) = t
;
1968 /* _Cilk_for's child function requires two extra parameters called
1969 __low and __high that are set the by Cilk runtime when it calls this
1973 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1974 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
1975 DECL_ARTIFICIAL (t
) = 1;
1976 DECL_NAMELESS (t
) = 1;
1977 DECL_ARG_TYPE (t
) = ptr_type_node
;
1978 DECL_CONTEXT (t
) = current_function_decl
;
1980 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1981 DECL_ARGUMENTS (decl
) = t
;
1983 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1984 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
1985 DECL_ARTIFICIAL (t
) = 1;
1986 DECL_NAMELESS (t
) = 1;
1987 DECL_ARG_TYPE (t
) = ptr_type_node
;
1988 DECL_CONTEXT (t
) = current_function_decl
;
1990 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1991 DECL_ARGUMENTS (decl
) = t
;
1994 tree data_name
= get_identifier (".omp_data_i");
1995 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
1997 DECL_ARTIFICIAL (t
) = 1;
1998 DECL_NAMELESS (t
) = 1;
1999 DECL_ARG_TYPE (t
) = ptr_type_node
;
2000 DECL_CONTEXT (t
) = current_function_decl
;
2003 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2004 DECL_ARGUMENTS (decl
) = t
;
2006 ctx
->receiver_decl
= t
;
2009 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2010 PARM_DECL
, get_identifier (".omp_data_o"),
2012 DECL_ARTIFICIAL (t
) = 1;
2013 DECL_NAMELESS (t
) = 1;
2014 DECL_ARG_TYPE (t
) = ptr_type_node
;
2015 DECL_CONTEXT (t
) = current_function_decl
;
2017 TREE_ADDRESSABLE (t
) = 1;
2018 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2019 DECL_ARGUMENTS (decl
) = t
;
2022 /* Allocate memory for the function structure. The call to
2023 allocate_struct_function clobbers CFUN, so we need to restore
2025 push_struct_function (decl
);
2026 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2030 /* Callback for walk_gimple_seq. Check if combined parallel
2031 contains gimple_omp_for_combined_into_p OMP_FOR. */
2034 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2035 bool *handled_ops_p
,
2036 struct walk_stmt_info
*wi
)
2038 gimple stmt
= gsi_stmt (*gsi_p
);
2040 *handled_ops_p
= true;
2041 switch (gimple_code (stmt
))
2045 case GIMPLE_OMP_FOR
:
2046 if (gimple_omp_for_combined_into_p (stmt
)
2047 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2050 return integer_zero_node
;
2059 /* Scan an OpenMP parallel directive. */
2062 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2066 gimple stmt
= gsi_stmt (*gsi
);
2068 /* Ignore parallel directives with empty bodies, unless there
2069 are copyin clauses. */
2071 && empty_body_p (gimple_omp_body (stmt
))
2072 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2073 OMP_CLAUSE_COPYIN
) == NULL
)
2075 gsi_replace (gsi
, gimple_build_nop (), false);
2079 if (gimple_omp_parallel_combined_p (stmt
))
2082 struct walk_stmt_info wi
;
2084 memset (&wi
, 0, sizeof (wi
));
2086 walk_gimple_seq (gimple_omp_body (stmt
),
2087 find_combined_for
, NULL
, &wi
);
2088 for_stmt
= (gimple
) wi
.info
;
2091 struct omp_for_data fd
;
2092 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2093 /* We need two temporaries with fd.loop.v type (istart/iend)
2094 and then (fd.collapse - 1) temporaries with the same
2095 type for count2 ... countN-1 vars if not constant. */
2096 size_t count
= 2, i
;
2097 tree type
= fd
.iter_type
;
2099 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2100 count
+= fd
.collapse
- 1;
2101 for (i
= 0; i
< count
; i
++)
2103 tree temp
= create_tmp_var (type
, NULL
);
2104 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2105 OMP_CLAUSE__LOOPTEMP_
);
2106 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2107 OMP_CLAUSE_DECL (c
) = temp
;
2108 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2109 gimple_omp_parallel_set_clauses (stmt
, c
);
2114 ctx
= new_omp_context (stmt
, outer_ctx
);
2115 taskreg_contexts
.safe_push (ctx
);
2116 if (taskreg_nesting_level
> 1)
2117 ctx
->is_nested
= true;
2118 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2119 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2120 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2121 name
= create_tmp_var_name (".omp_data_s");
2122 name
= build_decl (gimple_location (stmt
),
2123 TYPE_DECL
, name
, ctx
->record_type
);
2124 DECL_ARTIFICIAL (name
) = 1;
2125 DECL_NAMELESS (name
) = 1;
2126 TYPE_NAME (ctx
->record_type
) = name
;
2127 create_omp_child_function (ctx
, false);
2128 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2130 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2131 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2133 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2134 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2137 /* Scan an OpenMP task directive. */
2140 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2144 gimple stmt
= gsi_stmt (*gsi
);
2146 /* Ignore task directives with empty bodies. */
2148 && empty_body_p (gimple_omp_body (stmt
)))
2150 gsi_replace (gsi
, gimple_build_nop (), false);
2154 ctx
= new_omp_context (stmt
, outer_ctx
);
2155 taskreg_contexts
.safe_push (ctx
);
2156 if (taskreg_nesting_level
> 1)
2157 ctx
->is_nested
= true;
2158 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2159 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2160 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2161 name
= create_tmp_var_name (".omp_data_s");
2162 name
= build_decl (gimple_location (stmt
),
2163 TYPE_DECL
, name
, ctx
->record_type
);
2164 DECL_ARTIFICIAL (name
) = 1;
2165 DECL_NAMELESS (name
) = 1;
2166 TYPE_NAME (ctx
->record_type
) = name
;
2167 create_omp_child_function (ctx
, false);
2168 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2170 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2172 if (ctx
->srecord_type
)
2174 name
= create_tmp_var_name (".omp_data_a");
2175 name
= build_decl (gimple_location (stmt
),
2176 TYPE_DECL
, name
, ctx
->srecord_type
);
2177 DECL_ARTIFICIAL (name
) = 1;
2178 DECL_NAMELESS (name
) = 1;
2179 TYPE_NAME (ctx
->srecord_type
) = name
;
2180 create_omp_child_function (ctx
, true);
2183 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2185 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2187 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2188 t
= build_int_cst (long_integer_type_node
, 0);
2189 gimple_omp_task_set_arg_size (stmt
, t
);
2190 t
= build_int_cst (long_integer_type_node
, 1);
2191 gimple_omp_task_set_arg_align (stmt
, t
);
2196 /* If any decls have been made addressable during scan_omp,
2197 adjust their fields if needed, and layout record types
2198 of parallel/task constructs. */
2201 finish_taskreg_scan (omp_context
*ctx
)
2203 if (ctx
->record_type
== NULL_TREE
)
2206 /* If any task_shared_vars were needed, verify all
2207 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2208 statements if use_pointer_for_field hasn't changed
2209 because of that. If it did, update field types now. */
2210 if (task_shared_vars
)
2214 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2215 c
; c
= OMP_CLAUSE_CHAIN (c
))
2216 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2218 tree decl
= OMP_CLAUSE_DECL (c
);
2220 /* Global variables don't need to be copied,
2221 the receiver side will use them directly. */
2222 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2224 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2225 || !use_pointer_for_field (decl
, ctx
))
2227 tree field
= lookup_field (decl
, ctx
);
2228 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2229 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2231 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2232 TREE_THIS_VOLATILE (field
) = 0;
2233 DECL_USER_ALIGN (field
) = 0;
2234 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2235 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2236 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2237 if (ctx
->srecord_type
)
2239 tree sfield
= lookup_sfield (decl
, ctx
);
2240 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2241 TREE_THIS_VOLATILE (sfield
) = 0;
2242 DECL_USER_ALIGN (sfield
) = 0;
2243 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2244 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2245 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2250 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2252 layout_type (ctx
->record_type
);
2253 fixup_child_record_type (ctx
);
2257 location_t loc
= gimple_location (ctx
->stmt
);
2258 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2259 /* Move VLA fields to the end. */
2260 p
= &TYPE_FIELDS (ctx
->record_type
);
2262 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2263 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2266 *p
= TREE_CHAIN (*p
);
2267 TREE_CHAIN (*q
) = NULL_TREE
;
2268 q
= &TREE_CHAIN (*q
);
2271 p
= &DECL_CHAIN (*p
);
2273 layout_type (ctx
->record_type
);
2274 fixup_child_record_type (ctx
);
2275 if (ctx
->srecord_type
)
2276 layout_type (ctx
->srecord_type
);
2277 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2278 TYPE_SIZE_UNIT (ctx
->record_type
));
2279 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2280 t
= build_int_cst (long_integer_type_node
,
2281 TYPE_ALIGN_UNIT (ctx
->record_type
));
2282 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2287 /* Scan an OpenMP loop directive. */
2290 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2295 ctx
= new_omp_context (stmt
, outer_ctx
);
2297 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2299 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2300 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2302 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2303 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2304 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2305 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2307 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2310 /* Scan an OpenMP sections directive. */
2313 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2317 ctx
= new_omp_context (stmt
, outer_ctx
);
2318 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2319 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2322 /* Scan an OpenMP single directive. */
2325 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2330 ctx
= new_omp_context (stmt
, outer_ctx
);
2331 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2332 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2333 name
= create_tmp_var_name (".omp_copy_s");
2334 name
= build_decl (gimple_location (stmt
),
2335 TYPE_DECL
, name
, ctx
->record_type
);
2336 TYPE_NAME (ctx
->record_type
) = name
;
2338 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2339 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2341 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2342 ctx
->record_type
= NULL
;
2344 layout_type (ctx
->record_type
);
2347 /* Scan an OpenMP target{, data, update} directive. */
2350 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2354 int kind
= gimple_omp_target_kind (stmt
);
2356 ctx
= new_omp_context (stmt
, outer_ctx
);
2357 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2358 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2359 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2360 name
= create_tmp_var_name (".omp_data_t");
2361 name
= build_decl (gimple_location (stmt
),
2362 TYPE_DECL
, name
, ctx
->record_type
);
2363 DECL_ARTIFICIAL (name
) = 1;
2364 DECL_NAMELESS (name
) = 1;
2365 TYPE_NAME (ctx
->record_type
) = name
;
2366 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2368 create_omp_child_function (ctx
, false);
2369 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2372 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2373 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2375 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2376 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2379 TYPE_FIELDS (ctx
->record_type
)
2380 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2381 #ifdef ENABLE_CHECKING
2383 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2384 for (field
= TYPE_FIELDS (ctx
->record_type
);
2386 field
= DECL_CHAIN (field
))
2387 gcc_assert (DECL_ALIGN (field
) == align
);
2389 layout_type (ctx
->record_type
);
2390 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2391 fixup_child_record_type (ctx
);
2395 /* Scan an OpenMP teams directive. */
2398 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2400 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2401 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2402 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2405 /* Check OpenMP nesting restrictions. */
2407 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2411 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2412 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2414 error_at (gimple_location (stmt
),
2415 "OpenMP constructs may not be nested inside simd region");
2418 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2420 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2421 || (gimple_omp_for_kind (stmt
)
2422 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2423 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2425 error_at (gimple_location (stmt
),
2426 "only distribute or parallel constructs are allowed to "
2427 "be closely nested inside teams construct");
2432 switch (gimple_code (stmt
))
2434 case GIMPLE_OMP_FOR
:
2435 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2437 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2439 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2441 error_at (gimple_location (stmt
),
2442 "distribute construct must be closely nested inside "
2450 if (is_gimple_call (stmt
)
2451 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2452 == BUILT_IN_GOMP_CANCEL
2453 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2454 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2456 const char *bad
= NULL
;
2457 const char *kind
= NULL
;
2460 error_at (gimple_location (stmt
), "orphaned %qs construct",
2461 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2462 == BUILT_IN_GOMP_CANCEL
2463 ? "#pragma omp cancel"
2464 : "#pragma omp cancellation point");
2467 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2468 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2472 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2473 bad
= "#pragma omp parallel";
2474 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2475 == BUILT_IN_GOMP_CANCEL
2476 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2477 ctx
->cancellable
= true;
2481 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2482 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2483 bad
= "#pragma omp for";
2484 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2485 == BUILT_IN_GOMP_CANCEL
2486 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2488 ctx
->cancellable
= true;
2489 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2491 warning_at (gimple_location (stmt
), 0,
2492 "%<#pragma omp cancel for%> inside "
2493 "%<nowait%> for construct");
2494 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2495 OMP_CLAUSE_ORDERED
))
2496 warning_at (gimple_location (stmt
), 0,
2497 "%<#pragma omp cancel for%> inside "
2498 "%<ordered%> for construct");
2503 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2504 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2505 bad
= "#pragma omp sections";
2506 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2507 == BUILT_IN_GOMP_CANCEL
2508 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2510 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2512 ctx
->cancellable
= true;
2513 if (find_omp_clause (gimple_omp_sections_clauses
2516 warning_at (gimple_location (stmt
), 0,
2517 "%<#pragma omp cancel sections%> inside "
2518 "%<nowait%> sections construct");
2522 gcc_assert (ctx
->outer
2523 && gimple_code (ctx
->outer
->stmt
)
2524 == GIMPLE_OMP_SECTIONS
);
2525 ctx
->outer
->cancellable
= true;
2526 if (find_omp_clause (gimple_omp_sections_clauses
2529 warning_at (gimple_location (stmt
), 0,
2530 "%<#pragma omp cancel sections%> inside "
2531 "%<nowait%> sections construct");
2537 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2538 bad
= "#pragma omp task";
2540 ctx
->cancellable
= true;
2544 error_at (gimple_location (stmt
), "invalid arguments");
2549 error_at (gimple_location (stmt
),
2550 "%<%s %s%> construct not closely nested inside of %qs",
2551 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2552 == BUILT_IN_GOMP_CANCEL
2553 ? "#pragma omp cancel"
2554 : "#pragma omp cancellation point", kind
, bad
);
2559 case GIMPLE_OMP_SECTIONS
:
2560 case GIMPLE_OMP_SINGLE
:
2561 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2562 switch (gimple_code (ctx
->stmt
))
2564 case GIMPLE_OMP_FOR
:
2565 case GIMPLE_OMP_SECTIONS
:
2566 case GIMPLE_OMP_SINGLE
:
2567 case GIMPLE_OMP_ORDERED
:
2568 case GIMPLE_OMP_MASTER
:
2569 case GIMPLE_OMP_TASK
:
2570 case GIMPLE_OMP_CRITICAL
:
2571 if (is_gimple_call (stmt
))
2573 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2574 != BUILT_IN_GOMP_BARRIER
)
2576 error_at (gimple_location (stmt
),
2577 "barrier region may not be closely nested inside "
2578 "of work-sharing, critical, ordered, master or "
2579 "explicit task region");
2582 error_at (gimple_location (stmt
),
2583 "work-sharing region may not be closely nested inside "
2584 "of work-sharing, critical, ordered, master or explicit "
2587 case GIMPLE_OMP_PARALLEL
:
2593 case GIMPLE_OMP_MASTER
:
2594 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2595 switch (gimple_code (ctx
->stmt
))
2597 case GIMPLE_OMP_FOR
:
2598 case GIMPLE_OMP_SECTIONS
:
2599 case GIMPLE_OMP_SINGLE
:
2600 case GIMPLE_OMP_TASK
:
2601 error_at (gimple_location (stmt
),
2602 "master region may not be closely nested inside "
2603 "of work-sharing or explicit task region");
2605 case GIMPLE_OMP_PARALLEL
:
2611 case GIMPLE_OMP_ORDERED
:
2612 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2613 switch (gimple_code (ctx
->stmt
))
2615 case GIMPLE_OMP_CRITICAL
:
2616 case GIMPLE_OMP_TASK
:
2617 error_at (gimple_location (stmt
),
2618 "ordered region may not be closely nested inside "
2619 "of critical or explicit task region");
2621 case GIMPLE_OMP_FOR
:
2622 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2623 OMP_CLAUSE_ORDERED
) == NULL
)
2625 error_at (gimple_location (stmt
),
2626 "ordered region must be closely nested inside "
2627 "a loop region with an ordered clause");
2631 case GIMPLE_OMP_PARALLEL
:
2632 error_at (gimple_location (stmt
),
2633 "ordered region must be closely nested inside "
2634 "a loop region with an ordered clause");
2640 case GIMPLE_OMP_CRITICAL
:
2641 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2642 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2643 && (gimple_omp_critical_name (stmt
)
2644 == gimple_omp_critical_name (ctx
->stmt
)))
2646 error_at (gimple_location (stmt
),
2647 "critical region may not be nested inside a critical "
2648 "region with the same name");
2652 case GIMPLE_OMP_TEAMS
:
2654 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2655 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2657 error_at (gimple_location (stmt
),
2658 "teams construct not closely nested inside of target "
2663 case GIMPLE_OMP_TARGET
:
2664 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2665 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
2666 && gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_REGION
)
2669 switch (gimple_omp_target_kind (stmt
))
2671 case GF_OMP_TARGET_KIND_REGION
: name
= "target"; break;
2672 case GF_OMP_TARGET_KIND_DATA
: name
= "target data"; break;
2673 case GF_OMP_TARGET_KIND_UPDATE
: name
= "target update"; break;
2674 default: gcc_unreachable ();
2676 warning_at (gimple_location (stmt
), 0,
2677 "%s construct inside of target region", name
);
2687 /* Helper function scan_omp.
2689 Callback for walk_tree or operators in walk_gimple_stmt used to
2690 scan for OpenMP directives in TP. */
2693 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2695 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2696 omp_context
*ctx
= (omp_context
*) wi
->info
;
2699 switch (TREE_CODE (t
))
2706 *tp
= remap_decl (t
, &ctx
->cb
);
2710 if (ctx
&& TYPE_P (t
))
2711 *tp
= remap_type (t
, &ctx
->cb
);
2712 else if (!DECL_P (t
))
2717 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2718 if (tem
!= TREE_TYPE (t
))
2720 if (TREE_CODE (t
) == INTEGER_CST
)
2721 *tp
= wide_int_to_tree (tem
, t
);
2723 TREE_TYPE (t
) = tem
;
2733 /* Return true if FNDECL is a setjmp or a longjmp. */
2736 setjmp_or_longjmp_p (const_tree fndecl
)
2738 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2739 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2740 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2743 tree declname
= DECL_NAME (fndecl
);
2746 const char *name
= IDENTIFIER_POINTER (declname
);
2747 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2751 /* Helper function for scan_omp.
2753 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2754 the current statement in GSI. */
2757 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2758 struct walk_stmt_info
*wi
)
2760 gimple stmt
= gsi_stmt (*gsi
);
2761 omp_context
*ctx
= (omp_context
*) wi
->info
;
2763 if (gimple_has_location (stmt
))
2764 input_location
= gimple_location (stmt
);
2766 /* Check the OpenMP nesting restrictions. */
2767 bool remove
= false;
2768 if (is_gimple_omp (stmt
))
2769 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2770 else if (is_gimple_call (stmt
))
2772 tree fndecl
= gimple_call_fndecl (stmt
);
2775 if (setjmp_or_longjmp_p (fndecl
)
2777 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2778 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2781 error_at (gimple_location (stmt
),
2782 "setjmp/longjmp inside simd construct");
2784 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2785 switch (DECL_FUNCTION_CODE (fndecl
))
2787 case BUILT_IN_GOMP_BARRIER
:
2788 case BUILT_IN_GOMP_CANCEL
:
2789 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2790 case BUILT_IN_GOMP_TASKYIELD
:
2791 case BUILT_IN_GOMP_TASKWAIT
:
2792 case BUILT_IN_GOMP_TASKGROUP_START
:
2793 case BUILT_IN_GOMP_TASKGROUP_END
:
2794 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2803 stmt
= gimple_build_nop ();
2804 gsi_replace (gsi
, stmt
, false);
2807 *handled_ops_p
= true;
2809 switch (gimple_code (stmt
))
2811 case GIMPLE_OMP_PARALLEL
:
2812 taskreg_nesting_level
++;
2813 scan_omp_parallel (gsi
, ctx
);
2814 taskreg_nesting_level
--;
2817 case GIMPLE_OMP_TASK
:
2818 taskreg_nesting_level
++;
2819 scan_omp_task (gsi
, ctx
);
2820 taskreg_nesting_level
--;
2823 case GIMPLE_OMP_FOR
:
2824 scan_omp_for (stmt
, ctx
);
2827 case GIMPLE_OMP_SECTIONS
:
2828 scan_omp_sections (stmt
, ctx
);
2831 case GIMPLE_OMP_SINGLE
:
2832 scan_omp_single (stmt
, ctx
);
2835 case GIMPLE_OMP_SECTION
:
2836 case GIMPLE_OMP_MASTER
:
2837 case GIMPLE_OMP_TASKGROUP
:
2838 case GIMPLE_OMP_ORDERED
:
2839 case GIMPLE_OMP_CRITICAL
:
2840 ctx
= new_omp_context (stmt
, ctx
);
2841 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2844 case GIMPLE_OMP_TARGET
:
2845 scan_omp_target (stmt
, ctx
);
2848 case GIMPLE_OMP_TEAMS
:
2849 scan_omp_teams (stmt
, ctx
);
2856 *handled_ops_p
= false;
2858 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2859 insert_decl_map (&ctx
->cb
, var
, var
);
2863 *handled_ops_p
= false;
2871 /* Scan all the statements starting at the current statement. CTX
2872 contains context information about the OpenMP directives and
2873 clauses found during the scan. */
2876 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2878 location_t saved_location
;
2879 struct walk_stmt_info wi
;
2881 memset (&wi
, 0, sizeof (wi
));
2883 wi
.want_locations
= true;
2885 saved_location
= input_location
;
2886 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2887 input_location
= saved_location
;
2890 /* Re-gimplification and code generation routines. */
2892 /* Build a call to GOMP_barrier. */
2895 build_omp_barrier (tree lhs
)
2897 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2898 : BUILT_IN_GOMP_BARRIER
);
2899 gimple g
= gimple_build_call (fndecl
, 0);
2901 gimple_call_set_lhs (g
, lhs
);
2905 /* If a context was created for STMT when it was scanned, return it. */
2907 static omp_context
*
2908 maybe_lookup_ctx (gimple stmt
)
2911 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2912 return n
? (omp_context
*) n
->value
: NULL
;
2916 /* Find the mapping for DECL in CTX or the immediately enclosing
2917 context that has a mapping for DECL.
2919 If CTX is a nested parallel directive, we may have to use the decl
2920 mappings created in CTX's parent context. Suppose that we have the
2921 following parallel nesting (variable UIDs showed for clarity):
2924 #omp parallel shared(iD.1562) -> outer parallel
2925 iD.1562 = iD.1562 + 1;
2927 #omp parallel shared (iD.1562) -> inner parallel
2928 iD.1562 = iD.1562 - 1;
2930 Each parallel structure will create a distinct .omp_data_s structure
2931 for copying iD.1562 in/out of the directive:
2933 outer parallel .omp_data_s.1.i -> iD.1562
2934 inner parallel .omp_data_s.2.i -> iD.1562
2936 A shared variable mapping will produce a copy-out operation before
2937 the parallel directive and a copy-in operation after it. So, in
2938 this case we would have:
2941 .omp_data_o.1.i = iD.1562;
2942 #omp parallel shared(iD.1562) -> outer parallel
2943 .omp_data_i.1 = &.omp_data_o.1
2944 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2946 .omp_data_o.2.i = iD.1562; -> **
2947 #omp parallel shared(iD.1562) -> inner parallel
2948 .omp_data_i.2 = &.omp_data_o.2
2949 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2952 ** This is a problem. The symbol iD.1562 cannot be referenced
2953 inside the body of the outer parallel region. But since we are
2954 emitting this copy operation while expanding the inner parallel
2955 directive, we need to access the CTX structure of the outer
2956 parallel directive to get the correct mapping:
2958 .omp_data_o.2.i = .omp_data_i.1->i
2960 Since there may be other workshare or parallel directives enclosing
2961 the parallel directive, it may be necessary to walk up the context
2962 parent chain. This is not a problem in general because nested
2963 parallelism happens only rarely. */
2966 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2971 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2972 t
= maybe_lookup_decl (decl
, up
);
2974 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2976 return t
? t
: decl
;
2980 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2981 in outer contexts. */
2984 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2989 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2990 t
= maybe_lookup_decl (decl
, up
);
2992 return t
? t
: decl
;
2996 /* Construct the initialization value for reduction CLAUSE. */
2999 omp_reduction_init (tree clause
, tree type
)
3001 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3002 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3009 case TRUTH_ORIF_EXPR
:
3010 case TRUTH_XOR_EXPR
:
3012 return build_zero_cst (type
);
3015 case TRUTH_AND_EXPR
:
3016 case TRUTH_ANDIF_EXPR
:
3018 return fold_convert_loc (loc
, type
, integer_one_node
);
3021 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3024 if (SCALAR_FLOAT_TYPE_P (type
))
3026 REAL_VALUE_TYPE max
, min
;
3027 if (HONOR_INFINITIES (TYPE_MODE (type
)))
3030 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3033 real_maxval (&min
, 1, TYPE_MODE (type
));
3034 return build_real (type
, min
);
3038 gcc_assert (INTEGRAL_TYPE_P (type
));
3039 return TYPE_MIN_VALUE (type
);
3043 if (SCALAR_FLOAT_TYPE_P (type
))
3045 REAL_VALUE_TYPE max
;
3046 if (HONOR_INFINITIES (TYPE_MODE (type
)))
3049 real_maxval (&max
, 0, TYPE_MODE (type
));
3050 return build_real (type
, max
);
3054 gcc_assert (INTEGRAL_TYPE_P (type
));
3055 return TYPE_MAX_VALUE (type
);
3063 /* Return alignment to be assumed for var in CLAUSE, which should be
3064 OMP_CLAUSE_ALIGNED. */
3067 omp_clause_aligned_alignment (tree clause
)
3069 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3070 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3072 /* Otherwise return implementation defined alignment. */
3073 unsigned int al
= 1;
3074 machine_mode mode
, vmode
;
3075 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3077 vs
= 1 << floor_log2 (vs
);
3078 static enum mode_class classes
[]
3079 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3080 for (int i
= 0; i
< 4; i
+= 2)
3081 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3083 mode
= GET_MODE_WIDER_MODE (mode
))
3085 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3086 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3089 && GET_MODE_SIZE (vmode
) < vs
3090 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3091 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3093 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3094 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3096 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3097 / GET_MODE_SIZE (mode
));
3098 if (TYPE_MODE (type
) != vmode
)
3100 if (TYPE_ALIGN_UNIT (type
) > al
)
3101 al
= TYPE_ALIGN_UNIT (type
);
3103 return build_int_cst (integer_type_node
, al
);
3106 /* Return maximum possible vectorization factor for the target. */
3113 || !flag_tree_loop_optimize
3114 || (!flag_tree_loop_vectorize
3115 && (global_options_set
.x_flag_tree_loop_vectorize
3116 || global_options_set
.x_flag_tree_vectorize
)))
3119 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3122 vs
= 1 << floor_log2 (vs
);
3125 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3126 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3127 return GET_MODE_NUNITS (vqimode
);
3131 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3135 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3136 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3140 max_vf
= omp_max_vf ();
3143 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3144 OMP_CLAUSE_SAFELEN
);
3145 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3147 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3149 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3153 idx
= create_tmp_var (unsigned_type_node
, NULL
);
3154 lane
= create_tmp_var (unsigned_type_node
, NULL
);
3160 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3161 tree avar
= create_tmp_var_raw (atype
, NULL
);
3162 if (TREE_ADDRESSABLE (new_var
))
3163 TREE_ADDRESSABLE (avar
) = 1;
3164 DECL_ATTRIBUTES (avar
)
3165 = tree_cons (get_identifier ("omp simd array"), NULL
,
3166 DECL_ATTRIBUTES (avar
));
3167 gimple_add_tmp_var (avar
);
3168 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3169 NULL_TREE
, NULL_TREE
);
3170 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3171 NULL_TREE
, NULL_TREE
);
3172 if (DECL_P (new_var
))
3174 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3175 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3180 /* Helper function of lower_rec_input_clauses. For a reference
3181 in simd reduction, add an underlying variable it will reference. */
3184 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3186 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3187 if (TREE_CONSTANT (z
))
3189 const char *name
= NULL
;
3190 if (DECL_NAME (new_vard
))
3191 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3193 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3194 gimple_add_tmp_var (z
);
3195 TREE_ADDRESSABLE (z
) = 1;
3196 z
= build_fold_addr_expr_loc (loc
, z
);
3197 gimplify_assign (new_vard
, z
, ilist
);
3201 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3202 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3203 private variables. Initialization statements go in ILIST, while calls
3204 to destructors go in DLIST. */
3207 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3208 omp_context
*ctx
, struct omp_for_data
*fd
)
3210 tree c
, dtor
, copyin_seq
, x
, ptr
;
3211 bool copyin_by_ref
= false;
3212 bool lastprivate_firstprivate
= false;
3213 bool reduction_omp_orig_ref
= false;
3215 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3216 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3218 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3219 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3220 gimple_seq llist
[2] = { NULL
, NULL
};
3224 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3225 with data sharing clauses referencing variable sized vars. That
3226 is unnecessarily hard to support and very unlikely to result in
3227 vectorized code anyway. */
3229 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3230 switch (OMP_CLAUSE_CODE (c
))
3232 case OMP_CLAUSE_LINEAR
:
3233 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3236 case OMP_CLAUSE_REDUCTION
:
3237 case OMP_CLAUSE_PRIVATE
:
3238 case OMP_CLAUSE_FIRSTPRIVATE
:
3239 case OMP_CLAUSE_LASTPRIVATE
:
3240 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3247 /* Do all the fixed sized types in the first pass, and the variable sized
3248 types in the second pass. This makes sure that the scalar arguments to
3249 the variable sized types are processed before we use them in the
3250 variable sized operations. */
3251 for (pass
= 0; pass
< 2; ++pass
)
3253 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3255 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3258 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3262 case OMP_CLAUSE_PRIVATE
:
3263 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3266 case OMP_CLAUSE_SHARED
:
3267 /* Ignore shared directives in teams construct. */
3268 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3270 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3272 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3275 case OMP_CLAUSE_FIRSTPRIVATE
:
3276 case OMP_CLAUSE_COPYIN
:
3277 case OMP_CLAUSE_LINEAR
:
3279 case OMP_CLAUSE_REDUCTION
:
3280 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3281 reduction_omp_orig_ref
= true;
3283 case OMP_CLAUSE__LOOPTEMP_
:
3284 /* Handle _looptemp_ clauses only on parallel. */
3288 case OMP_CLAUSE_LASTPRIVATE
:
3289 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3291 lastprivate_firstprivate
= true;
3295 /* Even without corresponding firstprivate, if
3296 decl is Fortran allocatable, it needs outer var
3299 && lang_hooks
.decls
.omp_private_outer_ref
3300 (OMP_CLAUSE_DECL (c
)))
3301 lastprivate_firstprivate
= true;
3303 case OMP_CLAUSE_ALIGNED
:
3306 var
= OMP_CLAUSE_DECL (c
);
3307 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3308 && !is_global_var (var
))
3310 new_var
= maybe_lookup_decl (var
, ctx
);
3311 if (new_var
== NULL_TREE
)
3312 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3313 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3314 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3315 omp_clause_aligned_alignment (c
));
3316 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3317 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3318 gimplify_and_add (x
, ilist
);
3320 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3321 && is_global_var (var
))
3323 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3324 new_var
= lookup_decl (var
, ctx
);
3325 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3326 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3327 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3328 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3329 omp_clause_aligned_alignment (c
));
3330 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3331 x
= create_tmp_var (ptype
, NULL
);
3332 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3333 gimplify_and_add (t
, ilist
);
3334 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3335 SET_DECL_VALUE_EXPR (new_var
, t
);
3336 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3343 new_var
= var
= OMP_CLAUSE_DECL (c
);
3344 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3345 new_var
= lookup_decl (var
, ctx
);
3347 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3352 else if (is_variable_sized (var
))
3354 /* For variable sized types, we need to allocate the
3355 actual storage here. Call alloca and store the
3356 result in the pointer decl that we created elsewhere. */
3360 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3365 ptr
= DECL_VALUE_EXPR (new_var
);
3366 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3367 ptr
= TREE_OPERAND (ptr
, 0);
3368 gcc_assert (DECL_P (ptr
));
3369 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3371 /* void *tmp = __builtin_alloca */
3372 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3373 stmt
= gimple_build_call (atmp
, 1, x
);
3374 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3375 gimple_add_tmp_var (tmp
);
3376 gimple_call_set_lhs (stmt
, tmp
);
3378 gimple_seq_add_stmt (ilist
, stmt
);
3380 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3381 gimplify_assign (ptr
, x
, ilist
);
3384 else if (is_reference (var
))
3386 /* For references that are being privatized for Fortran,
3387 allocate new backing storage for the new pointer
3388 variable. This allows us to avoid changing all the
3389 code that expects a pointer to something that expects
3390 a direct variable. */
3394 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3395 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3397 x
= build_receiver_ref (var
, false, ctx
);
3398 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3400 else if (TREE_CONSTANT (x
))
3402 /* For reduction in SIMD loop, defer adding the
3403 initialization of the reference, because if we decide
3404 to use SIMD array for it, the initilization could cause
3406 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3410 const char *name
= NULL
;
3411 if (DECL_NAME (var
))
3412 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3414 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3416 gimple_add_tmp_var (x
);
3417 TREE_ADDRESSABLE (x
) = 1;
3418 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3423 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3424 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3429 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3430 gimplify_assign (new_var
, x
, ilist
);
3433 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3435 else if (c_kind
== OMP_CLAUSE_REDUCTION
3436 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3444 switch (OMP_CLAUSE_CODE (c
))
3446 case OMP_CLAUSE_SHARED
:
3447 /* Ignore shared directives in teams construct. */
3448 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3450 /* Shared global vars are just accessed directly. */
3451 if (is_global_var (new_var
))
3453 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3454 needs to be delayed until after fixup_child_record_type so
3455 that we get the correct type during the dereference. */
3456 by_ref
= use_pointer_for_field (var
, ctx
);
3457 x
= build_receiver_ref (var
, by_ref
, ctx
);
3458 SET_DECL_VALUE_EXPR (new_var
, x
);
3459 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3461 /* ??? If VAR is not passed by reference, and the variable
3462 hasn't been initialized yet, then we'll get a warning for
3463 the store into the omp_data_s structure. Ideally, we'd be
3464 able to notice this and not store anything at all, but
3465 we're generating code too early. Suppress the warning. */
3467 TREE_NO_WARNING (var
) = 1;
3470 case OMP_CLAUSE_LASTPRIVATE
:
3471 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3475 case OMP_CLAUSE_PRIVATE
:
3476 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3477 x
= build_outer_var_ref (var
, ctx
);
3478 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3480 if (is_task_ctx (ctx
))
3481 x
= build_receiver_ref (var
, false, ctx
);
3483 x
= build_outer_var_ref (var
, ctx
);
3489 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3492 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3493 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3494 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3495 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3496 idx
, lane
, ivar
, lvar
))
3499 x
= lang_hooks
.decls
.omp_clause_default_ctor
3500 (c
, unshare_expr (ivar
), x
);
3502 gimplify_and_add (x
, &llist
[0]);
3505 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3508 gimple_seq tseq
= NULL
;
3511 gimplify_stmt (&dtor
, &tseq
);
3512 gimple_seq_add_seq (&llist
[1], tseq
);
3519 gimplify_and_add (nx
, ilist
);
3523 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3526 gimple_seq tseq
= NULL
;
3529 gimplify_stmt (&dtor
, &tseq
);
3530 gimple_seq_add_seq (dlist
, tseq
);
3534 case OMP_CLAUSE_LINEAR
:
3535 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3536 goto do_firstprivate
;
3537 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3540 x
= build_outer_var_ref (var
, ctx
);
3543 case OMP_CLAUSE_FIRSTPRIVATE
:
3544 if (is_task_ctx (ctx
))
3546 if (is_reference (var
) || is_variable_sized (var
))
3548 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3550 || use_pointer_for_field (var
, NULL
))
3552 x
= build_receiver_ref (var
, false, ctx
);
3553 SET_DECL_VALUE_EXPR (new_var
, x
);
3554 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3559 x
= build_outer_var_ref (var
, ctx
);
3562 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3563 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3565 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3566 tree stept
= TREE_TYPE (t
);
3567 tree ct
= find_omp_clause (clauses
,
3568 OMP_CLAUSE__LOOPTEMP_
);
3570 tree l
= OMP_CLAUSE_DECL (ct
);
3571 tree n1
= fd
->loop
.n1
;
3572 tree step
= fd
->loop
.step
;
3573 tree itype
= TREE_TYPE (l
);
3574 if (POINTER_TYPE_P (itype
))
3575 itype
= signed_type_for (itype
);
3576 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3577 if (TYPE_UNSIGNED (itype
)
3578 && fd
->loop
.cond_code
== GT_EXPR
)
3579 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3580 fold_build1 (NEGATE_EXPR
, itype
, l
),
3581 fold_build1 (NEGATE_EXPR
,
3584 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3585 t
= fold_build2 (MULT_EXPR
, stept
,
3586 fold_convert (stept
, l
), t
);
3588 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3590 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3592 gimplify_and_add (x
, ilist
);
3596 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3597 x
= fold_build2 (POINTER_PLUS_EXPR
,
3598 TREE_TYPE (x
), x
, t
);
3600 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3603 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3604 || TREE_ADDRESSABLE (new_var
))
3605 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3606 idx
, lane
, ivar
, lvar
))
3608 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3610 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3611 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3612 gimplify_and_add (x
, ilist
);
3613 gimple_stmt_iterator gsi
3614 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3616 = gimple_build_assign (unshare_expr (lvar
), iv
);
3617 gsi_insert_before_without_update (&gsi
, g
,
3619 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3620 enum tree_code code
= PLUS_EXPR
;
3621 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3622 code
= POINTER_PLUS_EXPR
;
3623 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3624 gsi_insert_before_without_update (&gsi
, g
,
3628 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3629 (c
, unshare_expr (ivar
), x
);
3630 gimplify_and_add (x
, &llist
[0]);
3631 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3634 gimple_seq tseq
= NULL
;
3637 gimplify_stmt (&dtor
, &tseq
);
3638 gimple_seq_add_seq (&llist
[1], tseq
);
3643 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3644 gimplify_and_add (x
, ilist
);
3647 case OMP_CLAUSE__LOOPTEMP_
:
3648 gcc_assert (is_parallel_ctx (ctx
));
3649 x
= build_outer_var_ref (var
, ctx
);
3650 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3651 gimplify_and_add (x
, ilist
);
3654 case OMP_CLAUSE_COPYIN
:
3655 by_ref
= use_pointer_for_field (var
, NULL
);
3656 x
= build_receiver_ref (var
, by_ref
, ctx
);
3657 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3658 append_to_statement_list (x
, ©in_seq
);
3659 copyin_by_ref
|= by_ref
;
3662 case OMP_CLAUSE_REDUCTION
:
3663 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3665 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3667 x
= build_outer_var_ref (var
, ctx
);
3669 if (is_reference (var
)
3670 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3672 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3673 SET_DECL_VALUE_EXPR (placeholder
, x
);
3674 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3675 tree new_vard
= new_var
;
3676 if (is_reference (var
))
3678 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3679 new_vard
= TREE_OPERAND (new_var
, 0);
3680 gcc_assert (DECL_P (new_vard
));
3683 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3684 idx
, lane
, ivar
, lvar
))
3686 if (new_vard
== new_var
)
3688 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3689 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3693 SET_DECL_VALUE_EXPR (new_vard
,
3694 build_fold_addr_expr (ivar
));
3695 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3697 x
= lang_hooks
.decls
.omp_clause_default_ctor
3698 (c
, unshare_expr (ivar
),
3699 build_outer_var_ref (var
, ctx
));
3701 gimplify_and_add (x
, &llist
[0]);
3702 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3704 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3705 lower_omp (&tseq
, ctx
);
3706 gimple_seq_add_seq (&llist
[0], tseq
);
3708 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3709 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3710 lower_omp (&tseq
, ctx
);
3711 gimple_seq_add_seq (&llist
[1], tseq
);
3712 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3713 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3714 if (new_vard
== new_var
)
3715 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3717 SET_DECL_VALUE_EXPR (new_vard
,
3718 build_fold_addr_expr (lvar
));
3719 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3724 gimplify_stmt (&dtor
, &tseq
);
3725 gimple_seq_add_seq (&llist
[1], tseq
);
3729 /* If this is a reference to constant size reduction var
3730 with placeholder, we haven't emitted the initializer
3731 for it because it is undesirable if SIMD arrays are used.
3732 But if they aren't used, we need to emit the deferred
3733 initialization now. */
3734 else if (is_reference (var
) && is_simd
)
3735 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3736 x
= lang_hooks
.decls
.omp_clause_default_ctor
3737 (c
, unshare_expr (new_var
),
3738 build_outer_var_ref (var
, ctx
));
3740 gimplify_and_add (x
, ilist
);
3741 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3743 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3744 lower_omp (&tseq
, ctx
);
3745 gimple_seq_add_seq (ilist
, tseq
);
3747 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3750 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3751 lower_omp (&tseq
, ctx
);
3752 gimple_seq_add_seq (dlist
, tseq
);
3753 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3755 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3760 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3761 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3762 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3764 /* reduction(-:var) sums up the partial results, so it
3765 acts identically to reduction(+:var). */
3766 if (code
== MINUS_EXPR
)
3769 tree new_vard
= new_var
;
3770 if (is_simd
&& is_reference (var
))
3772 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3773 new_vard
= TREE_OPERAND (new_var
, 0);
3774 gcc_assert (DECL_P (new_vard
));
3777 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3778 idx
, lane
, ivar
, lvar
))
3780 tree ref
= build_outer_var_ref (var
, ctx
);
3782 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3784 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3785 ref
= build_outer_var_ref (var
, ctx
);
3786 gimplify_assign (ref
, x
, &llist
[1]);
3788 if (new_vard
!= new_var
)
3790 SET_DECL_VALUE_EXPR (new_vard
,
3791 build_fold_addr_expr (lvar
));
3792 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3797 if (is_reference (var
) && is_simd
)
3798 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3799 gimplify_assign (new_var
, x
, ilist
);
3802 tree ref
= build_outer_var_ref (var
, ctx
);
3804 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3805 ref
= build_outer_var_ref (var
, ctx
);
3806 gimplify_assign (ref
, x
, dlist
);
3820 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3821 /* Don't want uninit warnings on simduid, it is always uninitialized,
3822 but we use it not for the value, but for the DECL_UID only. */
3823 TREE_NO_WARNING (uid
) = 1;
3825 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3826 gimple_call_set_lhs (g
, lane
);
3827 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3828 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3829 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3830 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3831 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3832 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3833 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3834 build_int_cst (unsigned_type_node
, 0),
3836 gimple_seq_add_stmt (ilist
, g
);
3837 for (int i
= 0; i
< 2; i
++)
3840 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3841 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3842 gimple_call_set_lhs (g
, vf
);
3843 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3844 gimple_seq_add_stmt (seq
, g
);
3845 tree t
= build_int_cst (unsigned_type_node
, 0);
3846 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3847 gimple_seq_add_stmt (seq
, g
);
3848 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3849 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3850 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3851 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3852 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3853 gimple_seq_add_seq (seq
, llist
[i
]);
3854 t
= build_int_cst (unsigned_type_node
, 1);
3855 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3856 gimple_seq_add_stmt (seq
, g
);
3857 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3858 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3859 gimple_seq_add_stmt (seq
, g
);
3860 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3864 /* The copyin sequence is not to be executed by the main thread, since
3865 that would result in self-copies. Perhaps not visible to scalars,
3866 but it certainly is to C++ operator=. */
3869 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3871 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3872 build_int_cst (TREE_TYPE (x
), 0));
3873 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3874 gimplify_and_add (x
, ilist
);
3877 /* If any copyin variable is passed by reference, we must ensure the
3878 master thread doesn't modify it before it is copied over in all
3879 threads. Similarly for variables in both firstprivate and
3880 lastprivate clauses we need to ensure the lastprivate copying
3881 happens after firstprivate copying in all threads. And similarly
3882 for UDRs if initializer expression refers to omp_orig. */
3883 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3885 /* Don't add any barrier for #pragma omp simd or
3886 #pragma omp distribute. */
3887 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3888 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3889 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3892 /* If max_vf is non-zero, then we can use only a vectorization factor
3893 up to the max_vf we chose. So stick it into the safelen clause. */
3896 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3897 OMP_CLAUSE_SAFELEN
);
3899 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
3900 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3903 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3904 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3906 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3907 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3913 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3914 both parallel and workshare constructs. PREDICATE may be NULL if it's
3918 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3921 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3922 bool par_clauses
= false;
3923 tree simduid
= NULL
, lastlane
= NULL
;
3925 /* Early exit if there are no lastprivate or linear clauses. */
3926 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3927 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3928 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3929 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3931 if (clauses
== NULL
)
3933 /* If this was a workshare clause, see if it had been combined
3934 with its parallel. In that case, look for the clauses on the
3935 parallel statement itself. */
3936 if (is_parallel_ctx (ctx
))
3940 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3943 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3944 OMP_CLAUSE_LASTPRIVATE
);
3945 if (clauses
== NULL
)
3953 tree label_true
, arm1
, arm2
;
3955 label
= create_artificial_label (UNKNOWN_LOCATION
);
3956 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3957 arm1
= TREE_OPERAND (predicate
, 0);
3958 arm2
= TREE_OPERAND (predicate
, 1);
3959 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3960 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3961 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3963 gimple_seq_add_stmt (stmt_list
, stmt
);
3964 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3967 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3968 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3970 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3972 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3975 for (c
= clauses
; c
;)
3978 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3980 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3981 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3982 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3984 var
= OMP_CLAUSE_DECL (c
);
3985 new_var
= lookup_decl (var
, ctx
);
3987 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3989 tree val
= DECL_VALUE_EXPR (new_var
);
3990 if (TREE_CODE (val
) == ARRAY_REF
3991 && VAR_P (TREE_OPERAND (val
, 0))
3992 && lookup_attribute ("omp simd array",
3993 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3996 if (lastlane
== NULL
)
3998 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
4000 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4002 TREE_OPERAND (val
, 1));
4003 gimple_call_set_lhs (g
, lastlane
);
4004 gimple_seq_add_stmt (stmt_list
, g
);
4006 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4007 TREE_OPERAND (val
, 0), lastlane
,
4008 NULL_TREE
, NULL_TREE
);
4012 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4013 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4015 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4016 gimple_seq_add_seq (stmt_list
,
4017 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4018 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4020 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4021 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4023 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4024 gimple_seq_add_seq (stmt_list
,
4025 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4026 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4029 x
= build_outer_var_ref (var
, ctx
);
4030 if (is_reference (var
))
4031 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4032 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4033 gimplify_and_add (x
, stmt_list
);
4035 c
= OMP_CLAUSE_CHAIN (c
);
4036 if (c
== NULL
&& !par_clauses
)
4038 /* If this was a workshare clause, see if it had been combined
4039 with its parallel. In that case, continue looking for the
4040 clauses also on the parallel statement itself. */
4041 if (is_parallel_ctx (ctx
))
4045 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4048 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4049 OMP_CLAUSE_LASTPRIVATE
);
4055 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4059 /* Generate code to implement the REDUCTION clauses. */
4062 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4064 gimple_seq sub_seq
= NULL
;
4069 /* SIMD reductions are handled in lower_rec_input_clauses. */
4070 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4071 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4074 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4075 update in that case, otherwise use a lock. */
4076 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4077 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4079 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4081 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4091 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4093 tree var
, ref
, new_var
;
4094 enum tree_code code
;
4095 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4097 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4100 var
= OMP_CLAUSE_DECL (c
);
4101 new_var
= lookup_decl (var
, ctx
);
4102 if (is_reference (var
))
4103 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4104 ref
= build_outer_var_ref (var
, ctx
);
4105 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4107 /* reduction(-:var) sums up the partial results, so it acts
4108 identically to reduction(+:var). */
4109 if (code
== MINUS_EXPR
)
4114 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4116 addr
= save_expr (addr
);
4117 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4118 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4119 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4120 gimplify_and_add (x
, stmt_seqp
);
4124 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4126 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4128 if (is_reference (var
)
4129 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4131 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4132 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4133 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4134 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4135 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4136 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4137 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4141 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4142 ref
= build_outer_var_ref (var
, ctx
);
4143 gimplify_assign (ref
, x
, &sub_seq
);
4147 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4149 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4151 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4153 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4155 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4159 /* Generate code to implement the COPYPRIVATE clauses. */
4162 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4167 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4169 tree var
, new_var
, ref
, x
;
4171 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4173 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4176 var
= OMP_CLAUSE_DECL (c
);
4177 by_ref
= use_pointer_for_field (var
, NULL
);
4179 ref
= build_sender_ref (var
, ctx
);
4180 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4183 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4184 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4186 gimplify_assign (ref
, x
, slist
);
4188 ref
= build_receiver_ref (var
, false, ctx
);
4191 ref
= fold_convert_loc (clause_loc
,
4192 build_pointer_type (TREE_TYPE (new_var
)),
4194 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4196 if (is_reference (var
))
4198 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4199 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4200 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4202 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4203 gimplify_and_add (x
, rlist
);
4208 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4209 and REDUCTION from the sender (aka parent) side. */
4212 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4217 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4219 tree val
, ref
, x
, var
;
4220 bool by_ref
, do_in
= false, do_out
= false;
4221 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4223 switch (OMP_CLAUSE_CODE (c
))
4225 case OMP_CLAUSE_PRIVATE
:
4226 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4229 case OMP_CLAUSE_FIRSTPRIVATE
:
4230 case OMP_CLAUSE_COPYIN
:
4231 case OMP_CLAUSE_LASTPRIVATE
:
4232 case OMP_CLAUSE_REDUCTION
:
4233 case OMP_CLAUSE__LOOPTEMP_
:
4239 val
= OMP_CLAUSE_DECL (c
);
4240 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4242 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4243 && is_global_var (var
))
4245 if (is_variable_sized (val
))
4247 by_ref
= use_pointer_for_field (val
, NULL
);
4249 switch (OMP_CLAUSE_CODE (c
))
4251 case OMP_CLAUSE_PRIVATE
:
4252 case OMP_CLAUSE_FIRSTPRIVATE
:
4253 case OMP_CLAUSE_COPYIN
:
4254 case OMP_CLAUSE__LOOPTEMP_
:
4258 case OMP_CLAUSE_LASTPRIVATE
:
4259 if (by_ref
|| is_reference (val
))
4261 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4268 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4273 case OMP_CLAUSE_REDUCTION
:
4275 do_out
= !(by_ref
|| is_reference (val
));
4284 ref
= build_sender_ref (val
, ctx
);
4285 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4286 gimplify_assign (ref
, x
, ilist
);
4287 if (is_task_ctx (ctx
))
4288 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4293 ref
= build_sender_ref (val
, ctx
);
4294 gimplify_assign (var
, ref
, olist
);
4299 /* Generate code to implement SHARED from the sender (aka parent)
4300 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4301 list things that got automatically shared. */
4304 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4306 tree var
, ovar
, nvar
, f
, x
, record_type
;
4308 if (ctx
->record_type
== NULL
)
4311 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4312 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4314 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4315 nvar
= maybe_lookup_decl (ovar
, ctx
);
4316 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4319 /* If CTX is a nested parallel directive. Find the immediately
4320 enclosing parallel or workshare construct that contains a
4321 mapping for OVAR. */
4322 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4324 if (use_pointer_for_field (ovar
, ctx
))
4326 x
= build_sender_ref (ovar
, ctx
);
4327 var
= build_fold_addr_expr (var
);
4328 gimplify_assign (x
, var
, ilist
);
4332 x
= build_sender_ref (ovar
, ctx
);
4333 gimplify_assign (x
, var
, ilist
);
4335 if (!TREE_READONLY (var
)
4336 /* We don't need to receive a new reference to a result
4337 or parm decl. In fact we may not store to it as we will
4338 invalidate any pending RSO and generate wrong gimple
4340 && !((TREE_CODE (var
) == RESULT_DECL
4341 || TREE_CODE (var
) == PARM_DECL
)
4342 && DECL_BY_REFERENCE (var
)))
4344 x
= build_sender_ref (ovar
, ctx
);
4345 gimplify_assign (var
, x
, olist
);
4352 /* A convenience function to build an empty GIMPLE_COND with just the
4356 gimple_build_cond_empty (tree cond
)
4358 enum tree_code pred_code
;
4361 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4362 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4366 /* Build the function calls to GOMP_parallel_start etc to actually
4367 generate the parallel operation. REGION is the parallel region
4368 being expanded. BB is the block where to insert the code. WS_ARGS
4369 will be set if this is a call to a combined parallel+workshare
4370 construct, it contains the list of additional arguments needed by
4371 the workshare construct. */
4374 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4375 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4377 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4378 gimple_stmt_iterator gsi
;
4380 enum built_in_function start_ix
;
4382 location_t clause_loc
;
4383 vec
<tree
, va_gc
> *args
;
4385 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4387 /* Determine what flavor of GOMP_parallel we will be
4389 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4390 if (is_combined_parallel (region
))
4392 switch (region
->inner
->type
)
4394 case GIMPLE_OMP_FOR
:
4395 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4396 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4397 + (region
->inner
->sched_kind
4398 == OMP_CLAUSE_SCHEDULE_RUNTIME
4399 ? 3 : region
->inner
->sched_kind
));
4400 start_ix
= (enum built_in_function
)start_ix2
;
4402 case GIMPLE_OMP_SECTIONS
:
4403 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4410 /* By default, the value of NUM_THREADS is zero (selected at run time)
4411 and there is no conditional. */
4413 val
= build_int_cst (unsigned_type_node
, 0);
4414 flags
= build_int_cst (unsigned_type_node
, 0);
4416 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4418 cond
= OMP_CLAUSE_IF_EXPR (c
);
4420 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4423 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4424 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4427 clause_loc
= gimple_location (entry_stmt
);
4429 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4431 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4433 /* Ensure 'val' is of the correct type. */
4434 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4436 /* If we found the clause 'if (cond)', build either
4437 (cond != 0) or (cond ? val : 1u). */
4440 cond
= gimple_boolify (cond
);
4442 if (integer_zerop (val
))
4443 val
= fold_build2_loc (clause_loc
,
4444 EQ_EXPR
, unsigned_type_node
, cond
,
4445 build_int_cst (TREE_TYPE (cond
), 0));
4448 basic_block cond_bb
, then_bb
, else_bb
;
4449 edge e
, e_then
, e_else
;
4450 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4452 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4453 if (gimple_in_ssa_p (cfun
))
4455 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4456 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4457 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4466 e
= split_block (bb
, NULL
);
4471 then_bb
= create_empty_bb (cond_bb
);
4472 else_bb
= create_empty_bb (then_bb
);
4473 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4474 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4476 stmt
= gimple_build_cond_empty (cond
);
4477 gsi
= gsi_start_bb (cond_bb
);
4478 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4480 gsi
= gsi_start_bb (then_bb
);
4481 stmt
= gimple_build_assign (tmp_then
, val
);
4482 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4484 gsi
= gsi_start_bb (else_bb
);
4485 stmt
= gimple_build_assign
4486 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4487 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4489 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4490 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4491 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4492 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4493 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4494 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4496 if (gimple_in_ssa_p (cfun
))
4498 gimple phi
= create_phi_node (tmp_join
, bb
);
4499 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4500 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4506 gsi
= gsi_start_bb (bb
);
4507 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4508 false, GSI_CONTINUE_LINKING
);
4511 gsi
= gsi_last_bb (bb
);
4512 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4514 t1
= null_pointer_node
;
4516 t1
= build_fold_addr_expr (t
);
4517 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4519 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4520 args
->quick_push (t2
);
4521 args
->quick_push (t1
);
4522 args
->quick_push (val
);
4524 args
->splice (*ws_args
);
4525 args
->quick_push (flags
);
4527 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4528 builtin_decl_explicit (start_ix
), args
);
4530 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4531 false, GSI_CONTINUE_LINKING
);
4534 /* Insert a function call whose name is FUNC_NAME with the information from
4535 ENTRY_STMT into the basic_block BB. */
4538 expand_cilk_for_call (basic_block bb
, gimple entry_stmt
,
4539 vec
<tree
, va_gc
> *ws_args
)
4542 gimple_stmt_iterator gsi
;
4543 vec
<tree
, va_gc
> *args
;
4545 gcc_assert (vec_safe_length (ws_args
) == 2);
4546 tree func_name
= (*ws_args
)[0];
4547 tree grain
= (*ws_args
)[1];
4549 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4550 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
4551 gcc_assert (count
!= NULL_TREE
);
4552 count
= OMP_CLAUSE_OPERAND (count
, 0);
4554 gsi
= gsi_last_bb (bb
);
4555 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4557 t1
= null_pointer_node
;
4559 t1
= build_fold_addr_expr (t
);
4560 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4562 vec_alloc (args
, 4);
4563 args
->quick_push (t2
);
4564 args
->quick_push (t1
);
4565 args
->quick_push (count
);
4566 args
->quick_push (grain
);
4567 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
4569 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
4570 GSI_CONTINUE_LINKING
);
4573 /* Build the function call to GOMP_task to actually
4574 generate the task operation. BB is the block where to insert the code. */
4577 expand_task_call (basic_block bb
, gimple entry_stmt
)
4579 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4580 gimple_stmt_iterator gsi
;
4581 location_t loc
= gimple_location (entry_stmt
);
4583 clauses
= gimple_omp_task_clauses (entry_stmt
);
4585 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4587 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4589 cond
= boolean_true_node
;
4591 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4592 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4593 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4594 flags
= build_int_cst (unsigned_type_node
,
4595 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4597 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4600 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4601 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4602 build_int_cst (unsigned_type_node
, 2),
4603 build_int_cst (unsigned_type_node
, 0));
4604 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4607 depend
= OMP_CLAUSE_DECL (depend
);
4609 depend
= build_int_cst (ptr_type_node
, 0);
4611 gsi
= gsi_last_bb (bb
);
4612 t
= gimple_omp_task_data_arg (entry_stmt
);
4614 t2
= null_pointer_node
;
4616 t2
= build_fold_addr_expr_loc (loc
, t
);
4617 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4618 t
= gimple_omp_task_copy_fn (entry_stmt
);
4620 t3
= null_pointer_node
;
4622 t3
= build_fold_addr_expr_loc (loc
, t
);
4624 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4626 gimple_omp_task_arg_size (entry_stmt
),
4627 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4630 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4631 false, GSI_CONTINUE_LINKING
);
4635 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4636 catch handler and return it. This prevents programs from violating the
4637 structured block semantics with throws. */
4640 maybe_catch_exception (gimple_seq body
)
4645 if (!flag_exceptions
)
4648 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4649 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4651 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4653 g
= gimple_build_eh_must_not_throw (decl
);
4654 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4657 return gimple_seq_alloc_with_stmt (g
);
4660 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4663 vec2chain (vec
<tree
, va_gc
> *v
)
4665 tree chain
= NULL_TREE
, t
;
4668 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4670 DECL_CHAIN (t
) = chain
;
4678 /* Remove barriers in REGION->EXIT's block. Note that this is only
4679 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4680 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4681 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4685 remove_exit_barrier (struct omp_region
*region
)
4687 gimple_stmt_iterator gsi
;
4688 basic_block exit_bb
;
4692 int any_addressable_vars
= -1;
4694 exit_bb
= region
->exit
;
4696 /* If the parallel region doesn't return, we don't have REGION->EXIT
4701 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4702 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4703 statements that can appear in between are extremely limited -- no
4704 memory operations at all. Here, we allow nothing at all, so the
4705 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4706 gsi
= gsi_last_bb (exit_bb
);
4707 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4709 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4712 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4714 gsi
= gsi_last_bb (e
->src
);
4715 if (gsi_end_p (gsi
))
4717 stmt
= gsi_stmt (gsi
);
4718 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4719 && !gimple_omp_return_nowait_p (stmt
))
4721 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4722 in many cases. If there could be tasks queued, the barrier
4723 might be needed to let the tasks run before some local
4724 variable of the parallel that the task uses as shared
4725 runs out of scope. The task can be spawned either
4726 from within current function (this would be easy to check)
4727 or from some function it calls and gets passed an address
4728 of such a variable. */
4729 if (any_addressable_vars
< 0)
4731 gimple parallel_stmt
= last_stmt (region
->entry
);
4732 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4733 tree local_decls
, block
, decl
;
4736 any_addressable_vars
= 0;
4737 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4738 if (TREE_ADDRESSABLE (decl
))
4740 any_addressable_vars
= 1;
4743 for (block
= gimple_block (stmt
);
4744 !any_addressable_vars
4746 && TREE_CODE (block
) == BLOCK
;
4747 block
= BLOCK_SUPERCONTEXT (block
))
4749 for (local_decls
= BLOCK_VARS (block
);
4751 local_decls
= DECL_CHAIN (local_decls
))
4752 if (TREE_ADDRESSABLE (local_decls
))
4754 any_addressable_vars
= 1;
4757 if (block
== gimple_block (parallel_stmt
))
4761 if (!any_addressable_vars
)
4762 gimple_omp_return_set_nowait (stmt
);
4768 remove_exit_barriers (struct omp_region
*region
)
4770 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4771 remove_exit_barrier (region
);
4775 region
= region
->inner
;
4776 remove_exit_barriers (region
);
4777 while (region
->next
)
4779 region
= region
->next
;
4780 remove_exit_barriers (region
);
4785 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4786 calls. These can't be declared as const functions, but
4787 within one parallel body they are constant, so they can be
4788 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4789 which are declared const. Similarly for task body, except
4790 that in untied task omp_get_thread_num () can change at any task
4791 scheduling point. */
4794 optimize_omp_library_calls (gimple entry_stmt
)
4797 gimple_stmt_iterator gsi
;
4798 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4799 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4800 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4801 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4802 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4803 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4804 OMP_CLAUSE_UNTIED
) != NULL
);
4806 FOR_EACH_BB_FN (bb
, cfun
)
4807 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4809 gimple call
= gsi_stmt (gsi
);
4812 if (is_gimple_call (call
)
4813 && (decl
= gimple_call_fndecl (call
))
4814 && DECL_EXTERNAL (decl
)
4815 && TREE_PUBLIC (decl
)
4816 && DECL_INITIAL (decl
) == NULL
)
4820 if (DECL_NAME (decl
) == thr_num_id
)
4822 /* In #pragma omp task untied omp_get_thread_num () can change
4823 during the execution of the task region. */
4826 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4828 else if (DECL_NAME (decl
) == num_thr_id
)
4829 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4833 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4834 || gimple_call_num_args (call
) != 0)
4837 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4840 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4841 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4842 TREE_TYPE (TREE_TYPE (built_in
))))
4845 gimple_call_set_fndecl (call
, built_in
);
4850 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4854 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4858 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4859 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4862 if (TREE_CODE (t
) == ADDR_EXPR
)
4863 recompute_tree_invariant_for_addr_expr (t
);
4865 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4869 /* Prepend TO = FROM assignment before *GSI_P. */
4872 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4874 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4875 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4876 true, GSI_SAME_STMT
);
4877 gimple stmt
= gimple_build_assign (to
, from
);
4878 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4879 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4880 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4882 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4883 gimple_regimplify_operands (stmt
, &gsi
);
4887 /* Expand the OpenMP parallel or task directive starting at REGION. */
4890 expand_omp_taskreg (struct omp_region
*region
)
4892 basic_block entry_bb
, exit_bb
, new_bb
;
4893 struct function
*child_cfun
;
4894 tree child_fn
, block
, t
;
4895 gimple_stmt_iterator gsi
;
4896 gimple entry_stmt
, stmt
;
4898 vec
<tree
, va_gc
> *ws_args
;
4900 entry_stmt
= last_stmt (region
->entry
);
4901 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4902 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4904 entry_bb
= region
->entry
;
4905 exit_bb
= region
->exit
;
4909 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
4910 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
4911 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
4914 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
4915 and the inner statement contains the name of the built-in function
4917 ws_args
= region
->inner
->ws_args
;
4918 else if (is_combined_parallel (region
))
4919 ws_args
= region
->ws_args
;
4923 if (child_cfun
->cfg
)
4925 /* Due to inlining, it may happen that we have already outlined
4926 the region, in which case all we need to do is make the
4927 sub-graph unreachable and emit the parallel call. */
4928 edge entry_succ_e
, exit_succ_e
;
4930 entry_succ_e
= single_succ_edge (entry_bb
);
4932 gsi
= gsi_last_bb (entry_bb
);
4933 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4934 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4935 gsi_remove (&gsi
, true);
4940 exit_succ_e
= single_succ_edge (exit_bb
);
4941 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4943 remove_edge_and_dominated_blocks (entry_succ_e
);
4947 unsigned srcidx
, dstidx
, num
;
4949 /* If the parallel region needs data sent from the parent
4950 function, then the very first statement (except possible
4951 tree profile counter updates) of the parallel body
4952 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4953 &.OMP_DATA_O is passed as an argument to the child function,
4954 we need to replace it with the argument as seen by the child
4957 In most cases, this will end up being the identity assignment
4958 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4959 a function call that has been inlined, the original PARM_DECL
4960 .OMP_DATA_I may have been converted into a different local
4961 variable. In which case, we need to keep the assignment. */
4962 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4964 basic_block entry_succ_bb
= single_succ (entry_bb
);
4966 gimple parcopy_stmt
= NULL
;
4968 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4972 gcc_assert (!gsi_end_p (gsi
));
4973 stmt
= gsi_stmt (gsi
);
4974 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4977 if (gimple_num_ops (stmt
) == 2)
4979 tree arg
= gimple_assign_rhs1 (stmt
);
4981 /* We're ignore the subcode because we're
4982 effectively doing a STRIP_NOPS. */
4984 if (TREE_CODE (arg
) == ADDR_EXPR
4985 && TREE_OPERAND (arg
, 0)
4986 == gimple_omp_taskreg_data_arg (entry_stmt
))
4988 parcopy_stmt
= stmt
;
4994 gcc_assert (parcopy_stmt
!= NULL
);
4995 arg
= DECL_ARGUMENTS (child_fn
);
4997 if (!gimple_in_ssa_p (cfun
))
4999 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5000 gsi_remove (&gsi
, true);
5003 /* ?? Is setting the subcode really necessary ?? */
5004 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5005 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5010 /* If we are in ssa form, we must load the value from the default
5011 definition of the argument. That should not be defined now,
5012 since the argument is not used uninitialized. */
5013 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5014 narg
= make_ssa_name (arg
, gimple_build_nop ());
5015 set_ssa_default_def (cfun
, arg
, narg
);
5016 /* ?? Is setting the subcode really necessary ?? */
5017 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5018 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5019 update_stmt (parcopy_stmt
);
5023 /* Declare local variables needed in CHILD_CFUN. */
5024 block
= DECL_INITIAL (child_fn
);
5025 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5026 /* The gimplifier could record temporaries in parallel/task block
5027 rather than in containing function's local_decls chain,
5028 which would mean cgraph missed finalizing them. Do it now. */
5029 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5030 if (TREE_CODE (t
) == VAR_DECL
5032 && !DECL_EXTERNAL (t
))
5033 varpool_node::finalize_decl (t
);
5034 DECL_SAVED_TREE (child_fn
) = NULL
;
5035 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5036 gimple_set_body (child_fn
, NULL
);
5037 TREE_USED (block
) = 1;
5039 /* Reset DECL_CONTEXT on function arguments. */
5040 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5041 DECL_CONTEXT (t
) = child_fn
;
5043 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5044 so that it can be moved to the child function. */
5045 gsi
= gsi_last_bb (entry_bb
);
5046 stmt
= gsi_stmt (gsi
);
5047 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5048 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5049 gsi_remove (&gsi
, true);
5050 e
= split_block (entry_bb
, stmt
);
5052 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5054 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5057 gsi
= gsi_last_bb (exit_bb
);
5058 gcc_assert (!gsi_end_p (gsi
)
5059 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5060 stmt
= gimple_build_return (NULL
);
5061 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5062 gsi_remove (&gsi
, true);
5065 /* Move the parallel region into CHILD_CFUN. */
5067 if (gimple_in_ssa_p (cfun
))
5069 init_tree_ssa (child_cfun
);
5070 init_ssa_operands (child_cfun
);
5071 child_cfun
->gimple_df
->in_ssa_p
= true;
5075 block
= gimple_block (entry_stmt
);
5077 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5079 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5080 /* When the OMP expansion process cannot guarantee an up-to-date
5081 loop tree arrange for the child function to fixup loops. */
5082 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5083 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5085 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5086 num
= vec_safe_length (child_cfun
->local_decls
);
5087 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5089 t
= (*child_cfun
->local_decls
)[srcidx
];
5090 if (DECL_CONTEXT (t
) == cfun
->decl
)
5092 if (srcidx
!= dstidx
)
5093 (*child_cfun
->local_decls
)[dstidx
] = t
;
5097 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5099 /* Inform the callgraph about the new function. */
5100 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5101 cgraph_node::add_new_function (child_fn
, true);
5103 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5104 fixed in a following pass. */
5105 push_cfun (child_cfun
);
5107 optimize_omp_library_calls (entry_stmt
);
5108 cgraph_edge::rebuild_edges ();
5110 /* Some EH regions might become dead, see PR34608. If
5111 pass_cleanup_cfg isn't the first pass to happen with the
5112 new child, these dead EH edges might cause problems.
5113 Clean them up now. */
5114 if (flag_exceptions
)
5117 bool changed
= false;
5119 FOR_EACH_BB_FN (bb
, cfun
)
5120 changed
|= gimple_purge_dead_eh_edges (bb
);
5122 cleanup_tree_cfg ();
5124 if (gimple_in_ssa_p (cfun
))
5125 update_ssa (TODO_update_ssa
);
5129 /* Emit a library call to launch the children threads. */
5131 expand_cilk_for_call (new_bb
, entry_stmt
, ws_args
);
5132 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5133 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
5135 expand_task_call (new_bb
, entry_stmt
);
5136 if (gimple_in_ssa_p (cfun
))
5137 update_ssa (TODO_update_ssa_only_virtuals
);
5141 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5142 of the combined collapse > 1 loop constructs, generate code like:
5143 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5148 count3 = (adj + N32 - N31) / STEP3;
5149 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5154 count2 = (adj + N22 - N21) / STEP2;
5155 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5160 count1 = (adj + N12 - N11) / STEP1;
5161 count = count1 * count2 * count3;
5162 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5164 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5165 of the combined loop constructs, just initialize COUNTS array
5166 from the _looptemp_ clauses. */
5168 /* NOTE: It *could* be better to moosh all of the BBs together,
5169 creating one larger BB with all the computation and the unexpected
5170 jump at the end. I.e.
5172 bool zero3, zero2, zero1, zero;
5175 count3 = (N32 - N31) /[cl] STEP3;
5177 count2 = (N22 - N21) /[cl] STEP2;
5179 count1 = (N12 - N11) /[cl] STEP1;
5180 zero = zero3 || zero2 || zero1;
5181 count = count1 * count2 * count3;
5182 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5184 After all, we expect the zero=false, and thus we expect to have to
5185 evaluate all of the comparison expressions, so short-circuiting
5186 oughtn't be a win. Since the condition isn't protecting a
5187 denominator, we're not concerned about divide-by-zero, so we can
5188 fully evaluate count even if a numerator turned out to be wrong.
5190 It seems like putting this all together would create much better
5191 scheduling opportunities, and less pressure on the chip's branch
5195 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5196 basic_block
&entry_bb
, tree
*counts
,
5197 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5198 basic_block
&l2_dom_bb
)
5200 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5205 /* Collapsed loops need work for expansion into SSA form. */
5206 gcc_assert (!gimple_in_ssa_p (cfun
));
5208 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5209 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5211 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5212 isn't supposed to be handled, as the inner loop doesn't
5214 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5215 OMP_CLAUSE__LOOPTEMP_
);
5216 gcc_assert (innerc
);
5217 for (i
= 0; i
< fd
->collapse
; i
++)
5219 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5220 OMP_CLAUSE__LOOPTEMP_
);
5221 gcc_assert (innerc
);
5223 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5225 counts
[0] = NULL_TREE
;
5230 for (i
= 0; i
< fd
->collapse
; i
++)
5232 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5234 if (SSA_VAR_P (fd
->loop
.n2
)
5235 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5236 fold_convert (itype
, fd
->loops
[i
].n1
),
5237 fold_convert (itype
, fd
->loops
[i
].n2
)))
5238 == NULL_TREE
|| !integer_onep (t
)))
5241 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5242 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5243 true, GSI_SAME_STMT
);
5244 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5245 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5246 true, GSI_SAME_STMT
);
5247 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5248 NULL_TREE
, NULL_TREE
);
5249 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5250 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5251 expand_omp_regimplify_p
, NULL
, NULL
)
5252 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5253 expand_omp_regimplify_p
, NULL
, NULL
))
5255 *gsi
= gsi_for_stmt (stmt
);
5256 gimple_regimplify_operands (stmt
, gsi
);
5258 e
= split_block (entry_bb
, stmt
);
5259 if (zero_iter_bb
== NULL
)
5261 first_zero_iter
= i
;
5262 zero_iter_bb
= create_empty_bb (entry_bb
);
5263 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5264 *gsi
= gsi_after_labels (zero_iter_bb
);
5265 stmt
= gimple_build_assign (fd
->loop
.n2
,
5266 build_zero_cst (type
));
5267 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5268 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5271 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5272 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5273 e
->flags
= EDGE_TRUE_VALUE
;
5274 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5275 if (l2_dom_bb
== NULL
)
5276 l2_dom_bb
= entry_bb
;
5278 *gsi
= gsi_last_bb (entry_bb
);
5281 if (POINTER_TYPE_P (itype
))
5282 itype
= signed_type_for (itype
);
5283 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5285 t
= fold_build2 (PLUS_EXPR
, itype
,
5286 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5287 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5288 fold_convert (itype
, fd
->loops
[i
].n2
));
5289 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5290 fold_convert (itype
, fd
->loops
[i
].n1
));
5291 /* ?? We could probably use CEIL_DIV_EXPR instead of
5292 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5293 generate the same code in the end because generically we
5294 don't know that the values involved must be negative for
5296 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5297 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5298 fold_build1 (NEGATE_EXPR
, itype
, t
),
5299 fold_build1 (NEGATE_EXPR
, itype
,
5300 fold_convert (itype
,
5301 fd
->loops
[i
].step
)));
5303 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5304 fold_convert (itype
, fd
->loops
[i
].step
));
5305 t
= fold_convert (type
, t
);
5306 if (TREE_CODE (t
) == INTEGER_CST
)
5310 counts
[i
] = create_tmp_reg (type
, ".count");
5311 expand_omp_build_assign (gsi
, counts
[i
], t
);
5313 if (SSA_VAR_P (fd
->loop
.n2
))
5318 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5319 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5325 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5327 V3 = N31 + (T % count3) * STEP3;
5329 V2 = N21 + (T % count2) * STEP2;
5331 V1 = N11 + T * STEP1;
5332 if this loop doesn't have an inner loop construct combined with it.
5333 If it does have an inner loop construct combined with it and the
5334 iteration count isn't known constant, store values from counts array
5335 into its _looptemp_ temporaries instead. */
5338 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5339 tree
*counts
, gimple inner_stmt
, tree startvar
)
5342 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5344 /* If fd->loop.n2 is constant, then no propagation of the counts
5345 is needed, they are constant. */
5346 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5349 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5350 ? gimple_omp_parallel_clauses (inner_stmt
)
5351 : gimple_omp_for_clauses (inner_stmt
);
5352 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5353 isn't supposed to be handled, as the inner loop doesn't
5355 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5356 gcc_assert (innerc
);
5357 for (i
= 0; i
< fd
->collapse
; i
++)
5359 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5360 OMP_CLAUSE__LOOPTEMP_
);
5361 gcc_assert (innerc
);
5364 tree tem
= OMP_CLAUSE_DECL (innerc
);
5365 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5366 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5367 false, GSI_CONTINUE_LINKING
);
5368 gimple stmt
= gimple_build_assign (tem
, t
);
5369 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5375 tree type
= TREE_TYPE (fd
->loop
.v
);
5376 tree tem
= create_tmp_reg (type
, ".tem");
5377 gimple stmt
= gimple_build_assign (tem
, startvar
);
5378 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5380 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5382 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5384 if (POINTER_TYPE_P (vtype
))
5385 itype
= signed_type_for (vtype
);
5387 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5390 t
= fold_convert (itype
, t
);
5391 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5392 fold_convert (itype
, fd
->loops
[i
].step
));
5393 if (POINTER_TYPE_P (vtype
))
5394 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5396 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5397 t
= force_gimple_operand_gsi (gsi
, t
,
5398 DECL_P (fd
->loops
[i
].v
)
5399 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5401 GSI_CONTINUE_LINKING
);
5402 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5403 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5406 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5407 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5408 false, GSI_CONTINUE_LINKING
);
5409 stmt
= gimple_build_assign (tem
, t
);
5410 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5416 /* Helper function for expand_omp_for_*. Generate code like:
5419 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5423 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5430 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5431 basic_block body_bb
)
5433 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5435 gimple_stmt_iterator gsi
;
5441 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5443 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5445 bb
= create_empty_bb (last_bb
);
5446 add_bb_to_loop (bb
, last_bb
->loop_father
);
5447 gsi
= gsi_start_bb (bb
);
5449 if (i
< fd
->collapse
- 1)
5451 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5452 e
->probability
= REG_BR_PROB_BASE
/ 8;
5454 t
= fd
->loops
[i
+ 1].n1
;
5455 t
= force_gimple_operand_gsi (&gsi
, t
,
5456 DECL_P (fd
->loops
[i
+ 1].v
)
5457 && TREE_ADDRESSABLE (fd
->loops
[i
5460 GSI_CONTINUE_LINKING
);
5461 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5462 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5467 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5469 if (POINTER_TYPE_P (vtype
))
5470 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5472 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5473 t
= force_gimple_operand_gsi (&gsi
, t
,
5474 DECL_P (fd
->loops
[i
].v
)
5475 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5476 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5477 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5478 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5482 t
= fd
->loops
[i
].n2
;
5483 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5484 false, GSI_CONTINUE_LINKING
);
5485 tree v
= fd
->loops
[i
].v
;
5486 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5487 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5488 false, GSI_CONTINUE_LINKING
);
5489 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5490 stmt
= gimple_build_cond_empty (t
);
5491 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5492 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5493 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5496 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5504 /* A subroutine of expand_omp_for. Generate code for a parallel
5505 loop with any schedule. Given parameters:
5507 for (V = N1; V cond N2; V += STEP) BODY;
5509 where COND is "<" or ">", we generate pseudocode
5511 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5512 if (more) goto L0; else goto L3;
5519 if (V cond iend) goto L1; else goto L2;
5521 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5524 If this is a combined omp parallel loop, instead of the call to
5525 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5526 If this is gimple_omp_for_combined_p loop, then instead of assigning
5527 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5528 inner GIMPLE_OMP_FOR and V += STEP; and
5529 if (V cond iend) goto L1; else goto L2; are removed.
5531 For collapsed loops, given parameters:
5533 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5534 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5535 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5538 we generate pseudocode
5540 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5545 count3 = (adj + N32 - N31) / STEP3;
5546 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5551 count2 = (adj + N22 - N21) / STEP2;
5552 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5557 count1 = (adj + N12 - N11) / STEP1;
5558 count = count1 * count2 * count3;
5563 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5564 if (more) goto L0; else goto L3;
5568 V3 = N31 + (T % count3) * STEP3;
5570 V2 = N21 + (T % count2) * STEP2;
5572 V1 = N11 + T * STEP1;
5577 if (V < iend) goto L10; else goto L2;
5580 if (V3 cond3 N32) goto L1; else goto L11;
5584 if (V2 cond2 N22) goto L1; else goto L12;
5590 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5596 expand_omp_for_generic (struct omp_region
*region
,
5597 struct omp_for_data
*fd
,
5598 enum built_in_function start_fn
,
5599 enum built_in_function next_fn
,
5602 tree type
, istart0
, iend0
, iend
;
5603 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5604 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5605 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5606 gimple_stmt_iterator gsi
;
5608 bool in_combined_parallel
= is_combined_parallel (region
);
5609 bool broken_loop
= region
->cont
== NULL
;
5611 tree
*counts
= NULL
;
5614 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5615 gcc_assert (fd
->iter_type
== long_integer_type_node
5616 || !in_combined_parallel
);
5618 type
= TREE_TYPE (fd
->loop
.v
);
5619 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5620 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5621 TREE_ADDRESSABLE (istart0
) = 1;
5622 TREE_ADDRESSABLE (iend0
) = 1;
5624 /* See if we need to bias by LLONG_MIN. */
5625 if (fd
->iter_type
== long_long_unsigned_type_node
5626 && TREE_CODE (type
) == INTEGER_TYPE
5627 && !TYPE_UNSIGNED (type
))
5631 if (fd
->loop
.cond_code
== LT_EXPR
)
5634 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5638 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5641 if (TREE_CODE (n1
) != INTEGER_CST
5642 || TREE_CODE (n2
) != INTEGER_CST
5643 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5644 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5647 entry_bb
= region
->entry
;
5648 cont_bb
= region
->cont
;
5650 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5651 gcc_assert (broken_loop
5652 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5653 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5654 l1_bb
= single_succ (l0_bb
);
5657 l2_bb
= create_empty_bb (cont_bb
);
5658 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5659 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5663 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5664 exit_bb
= region
->exit
;
5666 gsi
= gsi_last_bb (entry_bb
);
5668 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5669 if (fd
->collapse
> 1)
5671 int first_zero_iter
= -1;
5672 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5674 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5675 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5676 zero_iter_bb
, first_zero_iter
,
5681 /* Some counts[i] vars might be uninitialized if
5682 some loop has zero iterations. But the body shouldn't
5683 be executed in that case, so just avoid uninit warnings. */
5684 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5685 if (SSA_VAR_P (counts
[i
]))
5686 TREE_NO_WARNING (counts
[i
]) = 1;
5688 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5690 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5691 gsi
= gsi_last_bb (entry_bb
);
5692 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5693 get_immediate_dominator (CDI_DOMINATORS
,
5697 if (in_combined_parallel
)
5699 /* In a combined parallel loop, emit a call to
5700 GOMP_loop_foo_next. */
5701 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5702 build_fold_addr_expr (istart0
),
5703 build_fold_addr_expr (iend0
));
5707 tree t0
, t1
, t2
, t3
, t4
;
5708 /* If this is not a combined parallel loop, emit a call to
5709 GOMP_loop_foo_start in ENTRY_BB. */
5710 t4
= build_fold_addr_expr (iend0
);
5711 t3
= build_fold_addr_expr (istart0
);
5712 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5715 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5717 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5718 OMP_CLAUSE__LOOPTEMP_
);
5719 gcc_assert (innerc
);
5720 t0
= OMP_CLAUSE_DECL (innerc
);
5721 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5722 OMP_CLAUSE__LOOPTEMP_
);
5723 gcc_assert (innerc
);
5724 t1
= OMP_CLAUSE_DECL (innerc
);
5726 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5727 && TYPE_PRECISION (TREE_TYPE (t0
))
5728 != TYPE_PRECISION (fd
->iter_type
))
5730 /* Avoid casting pointers to integer of a different size. */
5731 tree itype
= signed_type_for (type
);
5732 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5733 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5737 t1
= fold_convert (fd
->iter_type
, t1
);
5738 t0
= fold_convert (fd
->iter_type
, t0
);
5742 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5743 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5745 if (fd
->iter_type
== long_integer_type_node
)
5749 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5750 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5751 6, t0
, t1
, t2
, t
, t3
, t4
);
5754 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5755 5, t0
, t1
, t2
, t3
, t4
);
5763 /* The GOMP_loop_ull_*start functions have additional boolean
5764 argument, true for < loops and false for > loops.
5765 In Fortran, the C bool type can be different from
5766 boolean_type_node. */
5767 bfn_decl
= builtin_decl_explicit (start_fn
);
5768 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5769 t5
= build_int_cst (c_bool_type
,
5770 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5773 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5774 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5775 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5778 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5779 6, t5
, t0
, t1
, t2
, t3
, t4
);
5782 if (TREE_TYPE (t
) != boolean_type_node
)
5783 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5784 t
, build_int_cst (TREE_TYPE (t
), 0));
5785 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5786 true, GSI_SAME_STMT
);
5787 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5789 /* Remove the GIMPLE_OMP_FOR statement. */
5790 gsi_remove (&gsi
, true);
5792 /* Iteration setup for sequential loop goes in L0_BB. */
5793 tree startvar
= fd
->loop
.v
;
5794 tree endvar
= NULL_TREE
;
5796 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5798 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5799 && gimple_omp_for_kind (inner_stmt
)
5800 == GF_OMP_FOR_KIND_SIMD
);
5801 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5802 OMP_CLAUSE__LOOPTEMP_
);
5803 gcc_assert (innerc
);
5804 startvar
= OMP_CLAUSE_DECL (innerc
);
5805 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5806 OMP_CLAUSE__LOOPTEMP_
);
5807 gcc_assert (innerc
);
5808 endvar
= OMP_CLAUSE_DECL (innerc
);
5811 gsi
= gsi_start_bb (l0_bb
);
5814 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5815 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5816 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5817 t
= fold_convert (TREE_TYPE (startvar
), t
);
5818 t
= force_gimple_operand_gsi (&gsi
, t
,
5820 && TREE_ADDRESSABLE (startvar
),
5821 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5822 stmt
= gimple_build_assign (startvar
, t
);
5823 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5827 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5828 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5829 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5830 t
= fold_convert (TREE_TYPE (startvar
), t
);
5831 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5832 false, GSI_CONTINUE_LINKING
);
5835 stmt
= gimple_build_assign (endvar
, iend
);
5836 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5837 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
5838 stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
5840 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, iend
,
5842 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5844 if (fd
->collapse
> 1)
5845 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5849 /* Code to control the increment and predicate for the sequential
5850 loop goes in the CONT_BB. */
5851 gsi
= gsi_last_bb (cont_bb
);
5852 stmt
= gsi_stmt (gsi
);
5853 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5854 vmain
= gimple_omp_continue_control_use (stmt
);
5855 vback
= gimple_omp_continue_control_def (stmt
);
5857 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5859 if (POINTER_TYPE_P (type
))
5860 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5862 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5863 t
= force_gimple_operand_gsi (&gsi
, t
,
5865 && TREE_ADDRESSABLE (vback
),
5866 NULL_TREE
, true, GSI_SAME_STMT
);
5867 stmt
= gimple_build_assign (vback
, t
);
5868 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5870 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5871 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5873 stmt
= gimple_build_cond_empty (t
);
5874 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5877 /* Remove GIMPLE_OMP_CONTINUE. */
5878 gsi_remove (&gsi
, true);
5880 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5881 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5883 /* Emit code to get the next parallel iteration in L2_BB. */
5884 gsi
= gsi_start_bb (l2_bb
);
5886 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5887 build_fold_addr_expr (istart0
),
5888 build_fold_addr_expr (iend0
));
5889 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5890 false, GSI_CONTINUE_LINKING
);
5891 if (TREE_TYPE (t
) != boolean_type_node
)
5892 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5893 t
, build_int_cst (TREE_TYPE (t
), 0));
5894 stmt
= gimple_build_cond_empty (t
);
5895 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5898 /* Add the loop cleanup function. */
5899 gsi
= gsi_last_bb (exit_bb
);
5900 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5901 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5902 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5903 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5905 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5906 stmt
= gimple_build_call (t
, 0);
5907 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5908 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5909 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5910 gsi_remove (&gsi
, true);
5912 /* Connect the new blocks. */
5913 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5914 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5920 e
= find_edge (cont_bb
, l3_bb
);
5921 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5923 phis
= phi_nodes (l3_bb
);
5924 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5926 gimple phi
= gsi_stmt (gsi
);
5927 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5928 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5932 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5933 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5934 e
= find_edge (cont_bb
, l1_bb
);
5935 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5940 else if (fd
->collapse
> 1)
5943 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5946 e
->flags
= EDGE_TRUE_VALUE
;
5949 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5950 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5954 e
= find_edge (cont_bb
, l2_bb
);
5955 e
->flags
= EDGE_FALLTHRU
;
5957 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5959 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5960 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5961 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5962 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5963 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5964 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5965 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5966 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5968 struct loop
*outer_loop
= alloc_loop ();
5969 outer_loop
->header
= l0_bb
;
5970 outer_loop
->latch
= l2_bb
;
5971 add_loop (outer_loop
, l0_bb
->loop_father
);
5973 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5975 struct loop
*loop
= alloc_loop ();
5976 loop
->header
= l1_bb
;
5977 /* The loop may have multiple latches. */
5978 add_loop (loop
, outer_loop
);
5984 /* A subroutine of expand_omp_for. Generate code for a parallel
5985 loop with static schedule and no specified chunk size. Given
5988 for (V = N1; V cond N2; V += STEP) BODY;
5990 where COND is "<" or ">", we generate pseudocode
5992 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5997 if ((__typeof (V)) -1 > 0 && cond is >)
5998 n = -(adj + N2 - N1) / -STEP;
6000 n = (adj + N2 - N1) / STEP;
6003 if (threadid < tt) goto L3; else goto L4;
6008 s0 = q * threadid + tt;
6011 if (s0 >= e0) goto L2; else goto L0;
6017 if (V cond e) goto L1;
6022 expand_omp_for_static_nochunk (struct omp_region
*region
,
6023 struct omp_for_data
*fd
,
6026 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6027 tree type
, itype
, vmain
, vback
;
6028 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6029 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6031 gimple_stmt_iterator gsi
;
6034 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6035 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6036 bool broken_loop
= region
->cont
== NULL
;
6037 tree
*counts
= NULL
;
6040 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6041 if (POINTER_TYPE_P (type
))
6042 itype
= signed_type_for (type
);
6044 entry_bb
= region
->entry
;
6045 cont_bb
= region
->cont
;
6046 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6047 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6048 gcc_assert (broken_loop
6049 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6050 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6051 body_bb
= single_succ (seq_start_bb
);
6054 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6055 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6057 exit_bb
= region
->exit
;
6059 /* Iteration space partitioning goes in ENTRY_BB. */
6060 gsi
= gsi_last_bb (entry_bb
);
6061 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6063 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6065 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6066 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6069 if (fd
->collapse
> 1)
6071 int first_zero_iter
= -1;
6072 basic_block l2_dom_bb
= NULL
;
6074 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6075 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6076 fin_bb
, first_zero_iter
,
6080 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6081 t
= integer_one_node
;
6083 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6084 fold_convert (type
, fd
->loop
.n1
),
6085 fold_convert (type
, fd
->loop
.n2
));
6086 if (fd
->collapse
== 1
6087 && TYPE_UNSIGNED (type
)
6088 && (t
== NULL_TREE
|| !integer_onep (t
)))
6090 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6091 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6092 true, GSI_SAME_STMT
);
6093 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6094 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6095 true, GSI_SAME_STMT
);
6096 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6097 NULL_TREE
, NULL_TREE
);
6098 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6099 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6100 expand_omp_regimplify_p
, NULL
, NULL
)
6101 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6102 expand_omp_regimplify_p
, NULL
, NULL
))
6104 gsi
= gsi_for_stmt (stmt
);
6105 gimple_regimplify_operands (stmt
, &gsi
);
6107 ep
= split_block (entry_bb
, stmt
);
6108 ep
->flags
= EDGE_TRUE_VALUE
;
6109 entry_bb
= ep
->dest
;
6110 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6111 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6112 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6113 if (gimple_in_ssa_p (cfun
))
6115 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6116 for (gsi
= gsi_start_phis (fin_bb
);
6117 !gsi_end_p (gsi
); gsi_next (&gsi
))
6119 gimple phi
= gsi_stmt (gsi
);
6120 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6121 ep
, UNKNOWN_LOCATION
);
6124 gsi
= gsi_last_bb (entry_bb
);
6127 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6128 t
= fold_convert (itype
, t
);
6129 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6130 true, GSI_SAME_STMT
);
6132 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6133 t
= fold_convert (itype
, t
);
6134 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6135 true, GSI_SAME_STMT
);
6139 step
= fd
->loop
.step
;
6140 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6142 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6143 OMP_CLAUSE__LOOPTEMP_
);
6144 gcc_assert (innerc
);
6145 n1
= OMP_CLAUSE_DECL (innerc
);
6146 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6147 OMP_CLAUSE__LOOPTEMP_
);
6148 gcc_assert (innerc
);
6149 n2
= OMP_CLAUSE_DECL (innerc
);
6151 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6152 true, NULL_TREE
, true, GSI_SAME_STMT
);
6153 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6154 true, NULL_TREE
, true, GSI_SAME_STMT
);
6155 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6156 true, NULL_TREE
, true, GSI_SAME_STMT
);
6158 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6159 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6160 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6161 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6162 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6163 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6164 fold_build1 (NEGATE_EXPR
, itype
, t
),
6165 fold_build1 (NEGATE_EXPR
, itype
, step
));
6167 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6168 t
= fold_convert (itype
, t
);
6169 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6171 q
= create_tmp_reg (itype
, "q");
6172 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6173 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6174 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6176 tt
= create_tmp_reg (itype
, "tt");
6177 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6178 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6179 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6181 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6182 stmt
= gimple_build_cond_empty (t
);
6183 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6185 second_bb
= split_block (entry_bb
, stmt
)->dest
;
6186 gsi
= gsi_last_bb (second_bb
);
6187 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6189 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6191 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
6192 build_int_cst (itype
, 1));
6193 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6195 third_bb
= split_block (second_bb
, stmt
)->dest
;
6196 gsi
= gsi_last_bb (third_bb
);
6197 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6199 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6200 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6201 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6203 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6204 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6206 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6207 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6209 /* Remove the GIMPLE_OMP_FOR statement. */
6210 gsi_remove (&gsi
, true);
6212 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6213 gsi
= gsi_start_bb (seq_start_bb
);
6215 tree startvar
= fd
->loop
.v
;
6216 tree endvar
= NULL_TREE
;
6218 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6220 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6221 ? gimple_omp_parallel_clauses (inner_stmt
)
6222 : gimple_omp_for_clauses (inner_stmt
);
6223 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6224 gcc_assert (innerc
);
6225 startvar
= OMP_CLAUSE_DECL (innerc
);
6226 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6227 OMP_CLAUSE__LOOPTEMP_
);
6228 gcc_assert (innerc
);
6229 endvar
= OMP_CLAUSE_DECL (innerc
);
6231 t
= fold_convert (itype
, s0
);
6232 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6233 if (POINTER_TYPE_P (type
))
6234 t
= fold_build_pointer_plus (n1
, t
);
6236 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6237 t
= fold_convert (TREE_TYPE (startvar
), t
);
6238 t
= force_gimple_operand_gsi (&gsi
, t
,
6240 && TREE_ADDRESSABLE (startvar
),
6241 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6242 stmt
= gimple_build_assign (startvar
, t
);
6243 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6245 t
= fold_convert (itype
, e0
);
6246 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6247 if (POINTER_TYPE_P (type
))
6248 t
= fold_build_pointer_plus (n1
, t
);
6250 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6251 t
= fold_convert (TREE_TYPE (startvar
), t
);
6252 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6253 false, GSI_CONTINUE_LINKING
);
6256 stmt
= gimple_build_assign (endvar
, e
);
6257 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6258 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6259 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6261 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6263 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6265 if (fd
->collapse
> 1)
6266 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6270 /* The code controlling the sequential loop replaces the
6271 GIMPLE_OMP_CONTINUE. */
6272 gsi
= gsi_last_bb (cont_bb
);
6273 stmt
= gsi_stmt (gsi
);
6274 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6275 vmain
= gimple_omp_continue_control_use (stmt
);
6276 vback
= gimple_omp_continue_control_def (stmt
);
6278 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6280 if (POINTER_TYPE_P (type
))
6281 t
= fold_build_pointer_plus (vmain
, step
);
6283 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6284 t
= force_gimple_operand_gsi (&gsi
, t
,
6286 && TREE_ADDRESSABLE (vback
),
6287 NULL_TREE
, true, GSI_SAME_STMT
);
6288 stmt
= gimple_build_assign (vback
, t
);
6289 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6291 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6292 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6294 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6297 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6298 gsi_remove (&gsi
, true);
6300 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6301 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6304 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6305 gsi
= gsi_last_bb (exit_bb
);
6306 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6308 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6309 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6311 gsi_remove (&gsi
, true);
6313 /* Connect all the blocks. */
6314 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6315 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6316 ep
= find_edge (entry_bb
, second_bb
);
6317 ep
->flags
= EDGE_TRUE_VALUE
;
6318 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6319 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6320 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6324 ep
= find_edge (cont_bb
, body_bb
);
6325 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6330 else if (fd
->collapse
> 1)
6333 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6336 ep
->flags
= EDGE_TRUE_VALUE
;
6337 find_edge (cont_bb
, fin_bb
)->flags
6338 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6341 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6342 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6343 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6345 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6346 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6347 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6348 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6350 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6352 struct loop
*loop
= alloc_loop ();
6353 loop
->header
= body_bb
;
6354 if (collapse_bb
== NULL
)
6355 loop
->latch
= cont_bb
;
6356 add_loop (loop
, body_bb
->loop_father
);
6361 /* A subroutine of expand_omp_for. Generate code for a parallel
6362 loop with static schedule and a specified chunk size. Given
6365 for (V = N1; V cond N2; V += STEP) BODY;
6367 where COND is "<" or ">", we generate pseudocode
6369 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6374 if ((__typeof (V)) -1 > 0 && cond is >)
6375 n = -(adj + N2 - N1) / -STEP;
6377 n = (adj + N2 - N1) / STEP;
6379 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6380 here so that V is defined
6381 if the loop is not entered
6383 s0 = (trip * nthreads + threadid) * CHUNK;
6384 e0 = min(s0 + CHUNK, n);
6385 if (s0 < n) goto L1; else goto L4;
6392 if (V cond e) goto L2; else goto L3;
6400 expand_omp_for_static_chunk (struct omp_region
*region
,
6401 struct omp_for_data
*fd
, gimple inner_stmt
)
6403 tree n
, s0
, e0
, e
, t
;
6404 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6405 tree type
, itype
, vmain
, vback
, vextra
;
6406 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6407 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6408 gimple_stmt_iterator gsi
;
6411 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6412 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6413 bool broken_loop
= region
->cont
== NULL
;
6414 tree
*counts
= NULL
;
6417 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6418 if (POINTER_TYPE_P (type
))
6419 itype
= signed_type_for (type
);
6421 entry_bb
= region
->entry
;
6422 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6424 iter_part_bb
= se
->dest
;
6425 cont_bb
= region
->cont
;
6426 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6427 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6428 gcc_assert (broken_loop
6429 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6430 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6431 body_bb
= single_succ (seq_start_bb
);
6434 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6435 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6436 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6438 exit_bb
= region
->exit
;
6440 /* Trip and adjustment setup goes in ENTRY_BB. */
6441 gsi
= gsi_last_bb (entry_bb
);
6442 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6444 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6446 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6447 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6450 if (fd
->collapse
> 1)
6452 int first_zero_iter
= -1;
6453 basic_block l2_dom_bb
= NULL
;
6455 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6456 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6457 fin_bb
, first_zero_iter
,
6461 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6462 t
= integer_one_node
;
6464 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6465 fold_convert (type
, fd
->loop
.n1
),
6466 fold_convert (type
, fd
->loop
.n2
));
6467 if (fd
->collapse
== 1
6468 && TYPE_UNSIGNED (type
)
6469 && (t
== NULL_TREE
|| !integer_onep (t
)))
6471 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6472 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6473 true, GSI_SAME_STMT
);
6474 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6475 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6476 true, GSI_SAME_STMT
);
6477 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6478 NULL_TREE
, NULL_TREE
);
6479 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6480 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6481 expand_omp_regimplify_p
, NULL
, NULL
)
6482 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6483 expand_omp_regimplify_p
, NULL
, NULL
))
6485 gsi
= gsi_for_stmt (stmt
);
6486 gimple_regimplify_operands (stmt
, &gsi
);
6488 se
= split_block (entry_bb
, stmt
);
6489 se
->flags
= EDGE_TRUE_VALUE
;
6490 entry_bb
= se
->dest
;
6491 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6492 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6493 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6494 if (gimple_in_ssa_p (cfun
))
6496 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6497 for (gsi
= gsi_start_phis (fin_bb
);
6498 !gsi_end_p (gsi
); gsi_next (&gsi
))
6500 gimple phi
= gsi_stmt (gsi
);
6501 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6502 se
, UNKNOWN_LOCATION
);
6505 gsi
= gsi_last_bb (entry_bb
);
6508 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6509 t
= fold_convert (itype
, t
);
6510 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6511 true, GSI_SAME_STMT
);
6513 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6514 t
= fold_convert (itype
, t
);
6515 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6516 true, GSI_SAME_STMT
);
6520 step
= fd
->loop
.step
;
6521 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6523 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6524 OMP_CLAUSE__LOOPTEMP_
);
6525 gcc_assert (innerc
);
6526 n1
= OMP_CLAUSE_DECL (innerc
);
6527 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6528 OMP_CLAUSE__LOOPTEMP_
);
6529 gcc_assert (innerc
);
6530 n2
= OMP_CLAUSE_DECL (innerc
);
6532 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6533 true, NULL_TREE
, true, GSI_SAME_STMT
);
6534 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6535 true, NULL_TREE
, true, GSI_SAME_STMT
);
6536 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6537 true, NULL_TREE
, true, GSI_SAME_STMT
);
6539 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
6540 true, NULL_TREE
, true, GSI_SAME_STMT
);
6542 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6543 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6544 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6545 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6546 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6547 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6548 fold_build1 (NEGATE_EXPR
, itype
, t
),
6549 fold_build1 (NEGATE_EXPR
, itype
, step
));
6551 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6552 t
= fold_convert (itype
, t
);
6553 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6554 true, GSI_SAME_STMT
);
6556 trip_var
= create_tmp_reg (itype
, ".trip");
6557 if (gimple_in_ssa_p (cfun
))
6559 trip_init
= make_ssa_name (trip_var
, NULL
);
6560 trip_main
= make_ssa_name (trip_var
, NULL
);
6561 trip_back
= make_ssa_name (trip_var
, NULL
);
6565 trip_init
= trip_var
;
6566 trip_main
= trip_var
;
6567 trip_back
= trip_var
;
6570 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6571 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6573 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6574 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6575 if (POINTER_TYPE_P (type
))
6576 t
= fold_build_pointer_plus (n1
, t
);
6578 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6579 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6580 true, GSI_SAME_STMT
);
6582 /* Remove the GIMPLE_OMP_FOR. */
6583 gsi_remove (&gsi
, true);
6585 /* Iteration space partitioning goes in ITER_PART_BB. */
6586 gsi
= gsi_last_bb (iter_part_bb
);
6588 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6589 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6590 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6591 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6592 false, GSI_CONTINUE_LINKING
);
6594 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6595 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6596 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6597 false, GSI_CONTINUE_LINKING
);
6599 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6600 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6602 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6603 gsi
= gsi_start_bb (seq_start_bb
);
6605 tree startvar
= fd
->loop
.v
;
6606 tree endvar
= NULL_TREE
;
6608 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6610 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6611 ? gimple_omp_parallel_clauses (inner_stmt
)
6612 : gimple_omp_for_clauses (inner_stmt
);
6613 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6614 gcc_assert (innerc
);
6615 startvar
= OMP_CLAUSE_DECL (innerc
);
6616 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6617 OMP_CLAUSE__LOOPTEMP_
);
6618 gcc_assert (innerc
);
6619 endvar
= OMP_CLAUSE_DECL (innerc
);
6622 t
= fold_convert (itype
, s0
);
6623 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6624 if (POINTER_TYPE_P (type
))
6625 t
= fold_build_pointer_plus (n1
, t
);
6627 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6628 t
= fold_convert (TREE_TYPE (startvar
), t
);
6629 t
= force_gimple_operand_gsi (&gsi
, t
,
6631 && TREE_ADDRESSABLE (startvar
),
6632 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6633 stmt
= gimple_build_assign (startvar
, t
);
6634 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6636 t
= fold_convert (itype
, e0
);
6637 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6638 if (POINTER_TYPE_P (type
))
6639 t
= fold_build_pointer_plus (n1
, t
);
6641 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6642 t
= fold_convert (TREE_TYPE (startvar
), t
);
6643 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6644 false, GSI_CONTINUE_LINKING
);
6647 stmt
= gimple_build_assign (endvar
, e
);
6648 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6649 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6650 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6652 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6654 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6656 if (fd
->collapse
> 1)
6657 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6661 /* The code controlling the sequential loop goes in CONT_BB,
6662 replacing the GIMPLE_OMP_CONTINUE. */
6663 gsi
= gsi_last_bb (cont_bb
);
6664 stmt
= gsi_stmt (gsi
);
6665 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6666 vmain
= gimple_omp_continue_control_use (stmt
);
6667 vback
= gimple_omp_continue_control_def (stmt
);
6669 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6671 if (POINTER_TYPE_P (type
))
6672 t
= fold_build_pointer_plus (vmain
, step
);
6674 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6675 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
6676 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6677 true, GSI_SAME_STMT
);
6678 stmt
= gimple_build_assign (vback
, t
);
6679 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6681 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6682 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6684 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6687 /* Remove GIMPLE_OMP_CONTINUE. */
6688 gsi_remove (&gsi
, true);
6690 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6691 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6693 /* Trip update code goes into TRIP_UPDATE_BB. */
6694 gsi
= gsi_start_bb (trip_update_bb
);
6696 t
= build_int_cst (itype
, 1);
6697 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6698 stmt
= gimple_build_assign (trip_back
, t
);
6699 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6702 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6703 gsi
= gsi_last_bb (exit_bb
);
6704 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6706 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6707 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6709 gsi_remove (&gsi
, true);
6711 /* Connect the new blocks. */
6712 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6713 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6717 se
= find_edge (cont_bb
, body_bb
);
6718 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6723 else if (fd
->collapse
> 1)
6726 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6729 se
->flags
= EDGE_TRUE_VALUE
;
6730 find_edge (cont_bb
, trip_update_bb
)->flags
6731 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6733 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6736 if (gimple_in_ssa_p (cfun
))
6738 gimple_stmt_iterator psi
;
6744 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6746 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6747 remove arguments of the phi nodes in fin_bb. We need to create
6748 appropriate phi nodes in iter_part_bb instead. */
6749 se
= single_pred_edge (fin_bb
);
6750 re
= single_succ_edge (trip_update_bb
);
6751 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
6752 ene
= single_succ_edge (entry_bb
);
6754 psi
= gsi_start_phis (fin_bb
);
6755 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6756 gsi_next (&psi
), ++i
)
6759 source_location locus
;
6761 phi
= gsi_stmt (psi
);
6762 t
= gimple_phi_result (phi
);
6763 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6764 nphi
= create_phi_node (t
, iter_part_bb
);
6766 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6767 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6769 /* A special case -- fd->loop.v is not yet computed in
6770 iter_part_bb, we need to use vextra instead. */
6771 if (t
== fd
->loop
.v
)
6773 add_phi_arg (nphi
, t
, ene
, locus
);
6774 locus
= redirect_edge_var_map_location (vm
);
6775 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6777 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6778 redirect_edge_var_map_clear (re
);
6781 psi
= gsi_start_phis (fin_bb
);
6782 if (gsi_end_p (psi
))
6784 remove_phi_node (&psi
, false);
6787 /* Make phi node for trip. */
6788 phi
= create_phi_node (trip_main
, iter_part_bb
);
6789 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6791 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6796 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6797 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6798 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6799 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6800 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6801 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6802 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6803 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6804 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6808 struct loop
*trip_loop
= alloc_loop ();
6809 trip_loop
->header
= iter_part_bb
;
6810 trip_loop
->latch
= trip_update_bb
;
6811 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6813 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6815 struct loop
*loop
= alloc_loop ();
6816 loop
->header
= body_bb
;
6817 if (collapse_bb
== NULL
)
6818 loop
->latch
= cont_bb
;
6819 add_loop (loop
, trip_loop
);
6824 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
6826 for (V = N1; V cond N2; V += STEP) BODY;
6828 where COND is "<" or ">" or "!=", we generate pseudocode
6830 for (ind_var = low; ind_var < high; ind_var++)
6832 V = n1 + (ind_var * STEP)
6837 In the above pseudocode, low and high are function parameters of the
6838 child function. In the function below, we are inserting a temp.
6839 variable that will be making a call to two OMP functions that will not be
6840 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
6841 with _Cilk_for). These functions are replaced with low and high
6842 by the function that handles taskreg. */
6846 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
6848 bool broken_loop
= region
->cont
== NULL
;
6849 basic_block entry_bb
= region
->entry
;
6850 basic_block cont_bb
= region
->cont
;
6852 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6853 gcc_assert (broken_loop
6854 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6855 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6856 basic_block l1_bb
, l2_bb
;
6860 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6861 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6862 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6863 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6867 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6868 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6869 l2_bb
= single_succ (l1_bb
);
6871 basic_block exit_bb
= region
->exit
;
6872 basic_block l2_dom_bb
= NULL
;
6874 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
6876 /* Below statements until the "tree high_val = ..." are pseudo statements
6877 used to pass information to be used by expand_omp_taskreg.
6878 low_val and high_val will be replaced by the __low and __high
6879 parameter from the child function.
6881 The call_exprs part is a place-holder, it is mainly used
6882 to distinctly identify to the top-level part that this is
6883 where we should put low and high (reasoning given in header
6887 = gimple_omp_parallel_child_fn (last_stmt (region
->outer
->entry
));
6888 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
6889 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
6891 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
6893 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
6896 gcc_assert (low_val
&& high_val
);
6898 tree type
= TREE_TYPE (low_val
);
6899 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
6900 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6902 /* Not needed in SSA form right now. */
6903 gcc_assert (!gimple_in_ssa_p (cfun
));
6904 if (l2_dom_bb
== NULL
)
6910 gimple stmt
= gimple_build_assign (ind_var
, n1
);
6912 /* Replace the GIMPLE_OMP_FOR statement. */
6913 gsi_replace (&gsi
, stmt
, true);
6917 /* Code to control the increment goes in the CONT_BB. */
6918 gsi
= gsi_last_bb (cont_bb
);
6919 stmt
= gsi_stmt (gsi
);
6920 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6921 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, ind_var
, ind_var
,
6922 build_one_cst (type
));
6924 /* Replace GIMPLE_OMP_CONTINUE. */
6925 gsi_replace (&gsi
, stmt
, true);
6928 /* Emit the condition in L1_BB. */
6929 gsi
= gsi_after_labels (l1_bb
);
6930 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
6931 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
6933 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
6934 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
6935 fd
->loop
.n1
, fold_convert (sizetype
, t
));
6937 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
6938 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
6939 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
6940 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6942 /* The condition is always '<' since the runtime will fill in the low
6944 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
6945 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6947 /* Remove GIMPLE_OMP_RETURN. */
6948 gsi
= gsi_last_bb (exit_bb
);
6949 gsi_remove (&gsi
, true);
6951 /* Connect the new blocks. */
6952 remove_edge (FALLTHRU_EDGE (entry_bb
));
6957 remove_edge (BRANCH_EDGE (entry_bb
));
6958 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6960 e
= BRANCH_EDGE (l1_bb
);
6961 ne
= FALLTHRU_EDGE (l1_bb
);
6962 e
->flags
= EDGE_TRUE_VALUE
;
6966 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6968 ne
= single_succ_edge (l1_bb
);
6969 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6972 ne
->flags
= EDGE_FALSE_VALUE
;
6973 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6974 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6976 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6977 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6978 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6982 struct loop
*loop
= alloc_loop ();
6983 loop
->header
= l1_bb
;
6984 loop
->latch
= cont_bb
;
6985 add_loop (loop
, l1_bb
->loop_father
);
6986 loop
->safelen
= INT_MAX
;
6989 /* Pick the correct library function based on the precision of the
6990 induction variable type. */
6991 tree lib_fun
= NULL_TREE
;
6992 if (TYPE_PRECISION (type
) == 32)
6993 lib_fun
= cilk_for_32_fndecl
;
6994 else if (TYPE_PRECISION (type
) == 64)
6995 lib_fun
= cilk_for_64_fndecl
;
6999 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7001 /* WS_ARGS contains the library function flavor to call:
7002 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7003 user-defined grain value. If the user does not define one, then zero
7004 is passed in by the parser. */
7005 vec_alloc (region
->ws_args
, 2);
7006 region
->ws_args
->quick_push (lib_fun
);
7007 region
->ws_args
->quick_push (fd
->chunk_size
);
7010 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7011 loop. Given parameters:
7013 for (V = N1; V cond N2; V += STEP) BODY;
7015 where COND is "<" or ">", we generate pseudocode
7023 if (V cond N2) goto L0; else goto L2;
7026 For collapsed loops, given parameters:
7028 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7029 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7030 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7033 we generate pseudocode
7039 count3 = (adj + N32 - N31) / STEP3;
7044 count2 = (adj + N22 - N21) / STEP2;
7049 count1 = (adj + N12 - N11) / STEP1;
7050 count = count1 * count2 * count3;
7060 V2 += (V3 cond3 N32) ? 0 : STEP2;
7061 V3 = (V3 cond3 N32) ? V3 : N31;
7062 V1 += (V2 cond2 N22) ? 0 : STEP1;
7063 V2 = (V2 cond2 N22) ? V2 : N21;
7065 if (V < count) goto L0; else goto L2;
7071 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7074 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7075 gimple_stmt_iterator gsi
;
7077 bool broken_loop
= region
->cont
== NULL
;
7079 tree
*counts
= NULL
;
7081 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7082 OMP_CLAUSE_SAFELEN
);
7083 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7084 OMP_CLAUSE__SIMDUID_
);
7087 type
= TREE_TYPE (fd
->loop
.v
);
7088 entry_bb
= region
->entry
;
7089 cont_bb
= region
->cont
;
7090 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7091 gcc_assert (broken_loop
7092 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7093 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7096 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7097 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7098 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7099 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7103 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7104 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7105 l2_bb
= single_succ (l1_bb
);
7107 exit_bb
= region
->exit
;
7110 gsi
= gsi_last_bb (entry_bb
);
7112 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7113 /* Not needed in SSA form right now. */
7114 gcc_assert (!gimple_in_ssa_p (cfun
));
7115 if (fd
->collapse
> 1)
7117 int first_zero_iter
= -1;
7118 basic_block zero_iter_bb
= l2_bb
;
7120 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7121 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7122 zero_iter_bb
, first_zero_iter
,
7125 if (l2_dom_bb
== NULL
)
7130 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7132 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7133 OMP_CLAUSE__LOOPTEMP_
);
7134 gcc_assert (innerc
);
7135 n1
= OMP_CLAUSE_DECL (innerc
);
7136 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7137 OMP_CLAUSE__LOOPTEMP_
);
7138 gcc_assert (innerc
);
7139 n2
= OMP_CLAUSE_DECL (innerc
);
7140 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7141 fold_convert (type
, n1
));
7142 if (fd
->collapse
> 1)
7145 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7151 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7152 fold_convert (type
, fd
->loop
.n1
));
7153 if (fd
->collapse
> 1)
7154 for (i
= 0; i
< fd
->collapse
; i
++)
7156 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7157 if (POINTER_TYPE_P (itype
))
7158 itype
= signed_type_for (itype
);
7159 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7160 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7164 /* Remove the GIMPLE_OMP_FOR statement. */
7165 gsi_remove (&gsi
, true);
7169 /* Code to control the increment goes in the CONT_BB. */
7170 gsi
= gsi_last_bb (cont_bb
);
7171 stmt
= gsi_stmt (gsi
);
7172 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7174 if (POINTER_TYPE_P (type
))
7175 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7177 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7178 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7180 if (fd
->collapse
> 1)
7182 i
= fd
->collapse
- 1;
7183 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7185 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7186 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7190 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7192 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7195 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7197 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7199 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7200 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7201 if (POINTER_TYPE_P (itype2
))
7202 itype2
= signed_type_for (itype2
);
7203 t
= build3 (COND_EXPR
, itype2
,
7204 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7206 fold_convert (itype
, fd
->loops
[i
].n2
)),
7207 build_int_cst (itype2
, 0),
7208 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7209 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7210 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7212 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7213 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7215 t
= build3 (COND_EXPR
, itype
,
7216 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7218 fold_convert (itype
, fd
->loops
[i
].n2
)),
7220 fold_convert (itype
, fd
->loops
[i
].n1
));
7221 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7225 /* Remove GIMPLE_OMP_CONTINUE. */
7226 gsi_remove (&gsi
, true);
7229 /* Emit the condition in L1_BB. */
7230 gsi
= gsi_start_bb (l1_bb
);
7232 t
= fold_convert (type
, n2
);
7233 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7234 false, GSI_CONTINUE_LINKING
);
7235 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7236 stmt
= gimple_build_cond_empty (t
);
7237 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7238 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
7240 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
7243 gsi
= gsi_for_stmt (stmt
);
7244 gimple_regimplify_operands (stmt
, &gsi
);
7247 /* Remove GIMPLE_OMP_RETURN. */
7248 gsi
= gsi_last_bb (exit_bb
);
7249 gsi_remove (&gsi
, true);
7251 /* Connect the new blocks. */
7252 remove_edge (FALLTHRU_EDGE (entry_bb
));
7256 remove_edge (BRANCH_EDGE (entry_bb
));
7257 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7259 e
= BRANCH_EDGE (l1_bb
);
7260 ne
= FALLTHRU_EDGE (l1_bb
);
7261 e
->flags
= EDGE_TRUE_VALUE
;
7265 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7267 ne
= single_succ_edge (l1_bb
);
7268 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7271 ne
->flags
= EDGE_FALSE_VALUE
;
7272 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7273 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7275 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7276 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7277 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7281 struct loop
*loop
= alloc_loop ();
7282 loop
->header
= l1_bb
;
7283 loop
->latch
= cont_bb
;
7284 add_loop (loop
, l1_bb
->loop_father
);
7285 if (safelen
== NULL_TREE
)
7286 loop
->safelen
= INT_MAX
;
7289 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7290 if (TREE_CODE (safelen
) != INTEGER_CST
)
7292 else if (!tree_fits_uhwi_p (safelen
)
7293 || tree_to_uhwi (safelen
) > INT_MAX
)
7294 loop
->safelen
= INT_MAX
;
7296 loop
->safelen
= tree_to_uhwi (safelen
);
7297 if (loop
->safelen
== 1)
7302 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7303 cfun
->has_simduid_loops
= true;
7305 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7307 if ((flag_tree_loop_vectorize
7308 || (!global_options_set
.x_flag_tree_loop_vectorize
7309 && !global_options_set
.x_flag_tree_vectorize
))
7310 && flag_tree_loop_optimize
7311 && loop
->safelen
> 1)
7313 loop
->force_vectorize
= true;
7314 cfun
->has_force_vectorize_loops
= true;
7320 /* Expand the OpenMP loop defined by REGION. */
7323 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7325 struct omp_for_data fd
;
7326 struct omp_for_data_loop
*loops
;
7329 = (struct omp_for_data_loop
*)
7330 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7331 * sizeof (struct omp_for_data_loop
));
7332 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
7333 region
->sched_kind
= fd
.sched_kind
;
7335 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7336 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7337 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7340 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7341 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7342 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7345 /* If there isn't a continue then this is a degerate case where
7346 the introduction of abnormal edges during lowering will prevent
7347 original loops from being detected. Fix that up. */
7348 loops_state_set (LOOPS_NEED_FIXUP
);
7350 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7351 expand_omp_simd (region
, &fd
);
7352 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7353 expand_cilk_for (region
, &fd
);
7354 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7355 && !fd
.have_ordered
)
7357 if (fd
.chunk_size
== NULL
)
7358 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7360 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7364 int fn_index
, start_ix
, next_ix
;
7366 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7367 == GF_OMP_FOR_KIND_FOR
);
7368 if (fd
.chunk_size
== NULL
7369 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7370 fd
.chunk_size
= integer_zero_node
;
7371 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7372 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7373 ? 3 : fd
.sched_kind
;
7374 fn_index
+= fd
.have_ordered
* 4;
7375 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7376 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7377 if (fd
.iter_type
== long_long_unsigned_type_node
)
7379 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7380 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7381 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7382 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7384 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7385 (enum built_in_function
) next_ix
, inner_stmt
);
7388 if (gimple_in_ssa_p (cfun
))
7389 update_ssa (TODO_update_ssa_only_virtuals
);
7393 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7395 v = GOMP_sections_start (n);
7412 v = GOMP_sections_next ();
7417 If this is a combined parallel sections, replace the call to
7418 GOMP_sections_start with call to GOMP_sections_next. */
7421 expand_omp_sections (struct omp_region
*region
)
7423 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7425 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7426 gimple_stmt_iterator si
, switch_si
;
7427 gimple sections_stmt
, stmt
, cont
;
7430 struct omp_region
*inner
;
7432 bool exit_reachable
= region
->cont
!= NULL
;
7434 gcc_assert (region
->exit
!= NULL
);
7435 entry_bb
= region
->entry
;
7436 l0_bb
= single_succ (entry_bb
);
7437 l1_bb
= region
->cont
;
7438 l2_bb
= region
->exit
;
7439 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7440 l2
= gimple_block_label (l2_bb
);
7443 /* This can happen if there are reductions. */
7444 len
= EDGE_COUNT (l0_bb
->succs
);
7445 gcc_assert (len
> 0);
7446 e
= EDGE_SUCC (l0_bb
, len
- 1);
7447 si
= gsi_last_bb (e
->dest
);
7450 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7451 l2
= gimple_block_label (e
->dest
);
7453 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7455 si
= gsi_last_bb (e
->dest
);
7457 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7459 l2
= gimple_block_label (e
->dest
);
7465 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7467 default_bb
= create_empty_bb (l0_bb
);
7469 /* We will build a switch() with enough cases for all the
7470 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7471 and a default case to abort if something goes wrong. */
7472 len
= EDGE_COUNT (l0_bb
->succs
);
7474 /* Use vec::quick_push on label_vec throughout, since we know the size
7476 auto_vec
<tree
> label_vec (len
);
7478 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7479 GIMPLE_OMP_SECTIONS statement. */
7480 si
= gsi_last_bb (entry_bb
);
7481 sections_stmt
= gsi_stmt (si
);
7482 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7483 vin
= gimple_omp_sections_control (sections_stmt
);
7484 if (!is_combined_parallel (region
))
7486 /* If we are not inside a combined parallel+sections region,
7487 call GOMP_sections_start. */
7488 t
= build_int_cst (unsigned_type_node
, len
- 1);
7489 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7490 stmt
= gimple_build_call (u
, 1, t
);
7494 /* Otherwise, call GOMP_sections_next. */
7495 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7496 stmt
= gimple_build_call (u
, 0);
7498 gimple_call_set_lhs (stmt
, vin
);
7499 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7500 gsi_remove (&si
, true);
7502 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7504 switch_si
= gsi_last_bb (l0_bb
);
7505 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
7508 cont
= last_stmt (l1_bb
);
7509 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
7510 vmain
= gimple_omp_continue_control_use (cont
);
7511 vnext
= gimple_omp_continue_control_def (cont
);
7519 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7520 label_vec
.quick_push (t
);
7523 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7524 for (inner
= region
->inner
, casei
= 1;
7526 inner
= inner
->next
, i
++, casei
++)
7528 basic_block s_entry_bb
, s_exit_bb
;
7530 /* Skip optional reduction region. */
7531 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7538 s_entry_bb
= inner
->entry
;
7539 s_exit_bb
= inner
->exit
;
7541 t
= gimple_block_label (s_entry_bb
);
7542 u
= build_int_cst (unsigned_type_node
, casei
);
7543 u
= build_case_label (u
, NULL
, t
);
7544 label_vec
.quick_push (u
);
7546 si
= gsi_last_bb (s_entry_bb
);
7547 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7548 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7549 gsi_remove (&si
, true);
7550 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7552 if (s_exit_bb
== NULL
)
7555 si
= gsi_last_bb (s_exit_bb
);
7556 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7557 gsi_remove (&si
, true);
7559 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7562 /* Error handling code goes in DEFAULT_BB. */
7563 t
= gimple_block_label (default_bb
);
7564 u
= build_case_label (NULL
, NULL
, t
);
7565 make_edge (l0_bb
, default_bb
, 0);
7566 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7568 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7569 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7570 gsi_remove (&switch_si
, true);
7572 si
= gsi_start_bb (default_bb
);
7573 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7574 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7580 /* Code to get the next section goes in L1_BB. */
7581 si
= gsi_last_bb (l1_bb
);
7582 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7584 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7585 stmt
= gimple_build_call (bfn_decl
, 0);
7586 gimple_call_set_lhs (stmt
, vnext
);
7587 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7588 gsi_remove (&si
, true);
7590 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7593 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7594 si
= gsi_last_bb (l2_bb
);
7595 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7596 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7597 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7598 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7600 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7601 stmt
= gimple_build_call (t
, 0);
7602 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7603 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7604 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7605 gsi_remove (&si
, true);
7607 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7611 /* Expand code for an OpenMP single directive. We've already expanded
7612 much of the code, here we simply place the GOMP_barrier call. */
7615 expand_omp_single (struct omp_region
*region
)
7617 basic_block entry_bb
, exit_bb
;
7618 gimple_stmt_iterator si
;
7620 entry_bb
= region
->entry
;
7621 exit_bb
= region
->exit
;
7623 si
= gsi_last_bb (entry_bb
);
7624 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7625 gsi_remove (&si
, true);
7626 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7628 si
= gsi_last_bb (exit_bb
);
7629 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7631 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7632 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7634 gsi_remove (&si
, true);
7635 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7639 /* Generic expansion for OpenMP synchronization directives: master,
7640 ordered and critical. All we need to do here is remove the entry
7641 and exit markers for REGION. */
7644 expand_omp_synch (struct omp_region
*region
)
7646 basic_block entry_bb
, exit_bb
;
7647 gimple_stmt_iterator si
;
7649 entry_bb
= region
->entry
;
7650 exit_bb
= region
->exit
;
7652 si
= gsi_last_bb (entry_bb
);
7653 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7654 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7655 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7656 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7657 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7658 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7659 gsi_remove (&si
, true);
7660 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7664 si
= gsi_last_bb (exit_bb
);
7665 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7666 gsi_remove (&si
, true);
7667 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7671 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7672 operation as a normal volatile load. */
7675 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7676 tree loaded_val
, int index
)
7678 enum built_in_function tmpbase
;
7679 gimple_stmt_iterator gsi
;
7680 basic_block store_bb
;
7683 tree decl
, call
, type
, itype
;
7685 gsi
= gsi_last_bb (load_bb
);
7686 stmt
= gsi_stmt (gsi
);
7687 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7688 loc
= gimple_location (stmt
);
7690 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7691 is smaller than word size, then expand_atomic_load assumes that the load
7692 is atomic. We could avoid the builtin entirely in this case. */
7694 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7695 decl
= builtin_decl_explicit (tmpbase
);
7696 if (decl
== NULL_TREE
)
7699 type
= TREE_TYPE (loaded_val
);
7700 itype
= TREE_TYPE (TREE_TYPE (decl
));
7702 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7703 build_int_cst (NULL
,
7704 gimple_omp_atomic_seq_cst_p (stmt
)
7706 : MEMMODEL_RELAXED
));
7707 if (!useless_type_conversion_p (type
, itype
))
7708 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7709 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7711 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7712 gsi_remove (&gsi
, true);
7714 store_bb
= single_succ (load_bb
);
7715 gsi
= gsi_last_bb (store_bb
);
7716 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7717 gsi_remove (&gsi
, true);
7719 if (gimple_in_ssa_p (cfun
))
7720 update_ssa (TODO_update_ssa_no_phi
);
7725 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7726 operation as a normal volatile store. */
7729 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7730 tree loaded_val
, tree stored_val
, int index
)
7732 enum built_in_function tmpbase
;
7733 gimple_stmt_iterator gsi
;
7734 basic_block store_bb
= single_succ (load_bb
);
7737 tree decl
, call
, type
, itype
;
7741 gsi
= gsi_last_bb (load_bb
);
7742 stmt
= gsi_stmt (gsi
);
7743 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7745 /* If the load value is needed, then this isn't a store but an exchange. */
7746 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7748 gsi
= gsi_last_bb (store_bb
);
7749 stmt
= gsi_stmt (gsi
);
7750 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7751 loc
= gimple_location (stmt
);
7753 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7754 is smaller than word size, then expand_atomic_store assumes that the store
7755 is atomic. We could avoid the builtin entirely in this case. */
7757 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7758 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7759 decl
= builtin_decl_explicit (tmpbase
);
7760 if (decl
== NULL_TREE
)
7763 type
= TREE_TYPE (stored_val
);
7765 /* Dig out the type of the function's second argument. */
7766 itype
= TREE_TYPE (decl
);
7767 itype
= TYPE_ARG_TYPES (itype
);
7768 itype
= TREE_CHAIN (itype
);
7769 itype
= TREE_VALUE (itype
);
7770 imode
= TYPE_MODE (itype
);
7772 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7775 if (!useless_type_conversion_p (itype
, type
))
7776 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7777 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7778 build_int_cst (NULL
,
7779 gimple_omp_atomic_seq_cst_p (stmt
)
7781 : MEMMODEL_RELAXED
));
7784 if (!useless_type_conversion_p (type
, itype
))
7785 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7786 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7789 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7790 gsi_remove (&gsi
, true);
7792 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7793 gsi
= gsi_last_bb (load_bb
);
7794 gsi_remove (&gsi
, true);
7796 if (gimple_in_ssa_p (cfun
))
7797 update_ssa (TODO_update_ssa_no_phi
);
7802 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7803 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7804 size of the data type, and thus usable to find the index of the builtin
7805 decl. Returns false if the expression is not of the proper form. */
7808 expand_omp_atomic_fetch_op (basic_block load_bb
,
7809 tree addr
, tree loaded_val
,
7810 tree stored_val
, int index
)
7812 enum built_in_function oldbase
, newbase
, tmpbase
;
7813 tree decl
, itype
, call
;
7815 basic_block store_bb
= single_succ (load_bb
);
7816 gimple_stmt_iterator gsi
;
7819 enum tree_code code
;
7820 bool need_old
, need_new
;
7824 /* We expect to find the following sequences:
7827 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7830 val = tmp OP something; (or: something OP tmp)
7831 GIMPLE_OMP_STORE (val)
7833 ???FIXME: Allow a more flexible sequence.
7834 Perhaps use data flow to pick the statements.
7838 gsi
= gsi_after_labels (store_bb
);
7839 stmt
= gsi_stmt (gsi
);
7840 loc
= gimple_location (stmt
);
7841 if (!is_gimple_assign (stmt
))
7844 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7846 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7847 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7848 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7849 gcc_checking_assert (!need_old
|| !need_new
);
7851 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7854 /* Check for one of the supported fetch-op operations. */
7855 code
= gimple_assign_rhs_code (stmt
);
7859 case POINTER_PLUS_EXPR
:
7860 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7861 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7864 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7865 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7868 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7869 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7872 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7873 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7876 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7877 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7883 /* Make sure the expression is of the proper form. */
7884 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7885 rhs
= gimple_assign_rhs2 (stmt
);
7886 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7887 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7888 rhs
= gimple_assign_rhs1 (stmt
);
7892 tmpbase
= ((enum built_in_function
)
7893 ((need_new
? newbase
: oldbase
) + index
+ 1));
7894 decl
= builtin_decl_explicit (tmpbase
);
7895 if (decl
== NULL_TREE
)
7897 itype
= TREE_TYPE (TREE_TYPE (decl
));
7898 imode
= TYPE_MODE (itype
);
7900 /* We could test all of the various optabs involved, but the fact of the
7901 matter is that (with the exception of i486 vs i586 and xadd) all targets
7902 that support any atomic operaton optab also implements compare-and-swap.
7903 Let optabs.c take care of expanding any compare-and-swap loop. */
7904 if (!can_compare_and_swap_p (imode
, true))
7907 gsi
= gsi_last_bb (load_bb
);
7908 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7910 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7911 It only requires that the operation happen atomically. Thus we can
7912 use the RELAXED memory model. */
7913 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7914 fold_convert_loc (loc
, itype
, rhs
),
7915 build_int_cst (NULL
,
7916 seq_cst
? MEMMODEL_SEQ_CST
7917 : MEMMODEL_RELAXED
));
7919 if (need_old
|| need_new
)
7921 lhs
= need_old
? loaded_val
: stored_val
;
7922 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7923 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7926 call
= fold_convert_loc (loc
, void_type_node
, call
);
7927 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7928 gsi_remove (&gsi
, true);
7930 gsi
= gsi_last_bb (store_bb
);
7931 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7932 gsi_remove (&gsi
, true);
7933 gsi
= gsi_last_bb (store_bb
);
7934 gsi_remove (&gsi
, true);
7936 if (gimple_in_ssa_p (cfun
))
7937 update_ssa (TODO_update_ssa_no_phi
);
7942 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7946 newval = rhs; // with oldval replacing *addr in rhs
7947 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7948 if (oldval != newval)
7951 INDEX is log2 of the size of the data type, and thus usable to find the
7952 index of the builtin decl. */
7955 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7956 tree addr
, tree loaded_val
, tree stored_val
,
7959 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7960 tree type
, itype
, cmpxchg
, iaddr
;
7961 gimple_stmt_iterator si
;
7962 basic_block loop_header
= single_succ (load_bb
);
7965 enum built_in_function fncode
;
7967 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7968 order to use the RELAXED memory model effectively. */
7969 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7971 cmpxchg
= builtin_decl_explicit (fncode
);
7972 if (cmpxchg
== NULL_TREE
)
7974 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7975 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7977 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7980 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7981 si
= gsi_last_bb (load_bb
);
7982 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7984 /* For floating-point values, we'll need to view-convert them to integers
7985 so that we can perform the atomic compare and swap. Simplify the
7986 following code by always setting up the "i"ntegral variables. */
7987 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7991 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7994 = force_gimple_operand_gsi (&si
,
7995 fold_convert (TREE_TYPE (iaddr
), addr
),
7996 false, NULL_TREE
, true, GSI_SAME_STMT
);
7997 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7998 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7999 loadedi
= create_tmp_var (itype
, NULL
);
8000 if (gimple_in_ssa_p (cfun
))
8001 loadedi
= make_ssa_name (loadedi
, NULL
);
8006 loadedi
= loaded_val
;
8009 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8010 tree loaddecl
= builtin_decl_explicit (fncode
);
8013 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8014 build_call_expr (loaddecl
, 2, iaddr
,
8015 build_int_cst (NULL_TREE
,
8016 MEMMODEL_RELAXED
)));
8018 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8019 build_int_cst (TREE_TYPE (iaddr
), 0));
8022 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8025 /* Move the value to the LOADEDI temporary. */
8026 if (gimple_in_ssa_p (cfun
))
8028 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8029 phi
= create_phi_node (loadedi
, loop_header
);
8030 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8034 gsi_insert_before (&si
,
8035 gimple_build_assign (loadedi
, initial
),
8037 if (loadedi
!= loaded_val
)
8039 gimple_stmt_iterator gsi2
;
8042 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8043 gsi2
= gsi_start_bb (loop_header
);
8044 if (gimple_in_ssa_p (cfun
))
8047 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8048 true, GSI_SAME_STMT
);
8049 stmt
= gimple_build_assign (loaded_val
, x
);
8050 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8054 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8055 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8056 true, GSI_SAME_STMT
);
8059 gsi_remove (&si
, true);
8061 si
= gsi_last_bb (store_bb
);
8062 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8065 storedi
= stored_val
;
8068 force_gimple_operand_gsi (&si
,
8069 build1 (VIEW_CONVERT_EXPR
, itype
,
8070 stored_val
), true, NULL_TREE
, true,
8073 /* Build the compare&swap statement. */
8074 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8075 new_storedi
= force_gimple_operand_gsi (&si
,
8076 fold_convert (TREE_TYPE (loadedi
),
8079 true, GSI_SAME_STMT
);
8081 if (gimple_in_ssa_p (cfun
))
8085 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
8086 stmt
= gimple_build_assign (old_vali
, loadedi
);
8087 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8089 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8090 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8093 /* Note that we always perform the comparison as an integer, even for
8094 floating point. This allows the atomic operation to properly
8095 succeed even with NaNs and -0.0. */
8096 stmt
= gimple_build_cond_empty
8097 (build2 (NE_EXPR
, boolean_type_node
,
8098 new_storedi
, old_vali
));
8099 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8102 e
= single_succ_edge (store_bb
);
8103 e
->flags
&= ~EDGE_FALLTHRU
;
8104 e
->flags
|= EDGE_FALSE_VALUE
;
8106 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8108 /* Copy the new value to loadedi (we already did that before the condition
8109 if we are not in SSA). */
8110 if (gimple_in_ssa_p (cfun
))
8112 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8113 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8116 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8117 gsi_remove (&si
, true);
8119 struct loop
*loop
= alloc_loop ();
8120 loop
->header
= loop_header
;
8121 loop
->latch
= store_bb
;
8122 add_loop (loop
, loop_header
->loop_father
);
8124 if (gimple_in_ssa_p (cfun
))
8125 update_ssa (TODO_update_ssa_no_phi
);
8130 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8132 GOMP_atomic_start ();
8136 The result is not globally atomic, but works so long as all parallel
8137 references are within #pragma omp atomic directives. According to
8138 responses received from omp@openmp.org, appears to be within spec.
8139 Which makes sense, since that's how several other compilers handle
8140 this situation as well.
8141 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8142 expanding. STORED_VAL is the operand of the matching
8143 GIMPLE_OMP_ATOMIC_STORE.
8146 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8150 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8155 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8156 tree addr
, tree loaded_val
, tree stored_val
)
8158 gimple_stmt_iterator si
;
8162 si
= gsi_last_bb (load_bb
);
8163 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8165 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8166 t
= build_call_expr (t
, 0);
8167 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8169 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8170 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8171 gsi_remove (&si
, true);
8173 si
= gsi_last_bb (store_bb
);
8174 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8176 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8178 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8180 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8181 t
= build_call_expr (t
, 0);
8182 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8183 gsi_remove (&si
, true);
8185 if (gimple_in_ssa_p (cfun
))
8186 update_ssa (TODO_update_ssa_no_phi
);
8190 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8191 using expand_omp_atomic_fetch_op. If it failed, we try to
8192 call expand_omp_atomic_pipeline, and if it fails too, the
8193 ultimate fallback is wrapping the operation in a mutex
8194 (expand_omp_atomic_mutex). REGION is the atomic region built
8195 by build_omp_regions_1(). */
8198 expand_omp_atomic (struct omp_region
*region
)
8200 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8201 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
8202 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8203 tree addr
= gimple_omp_atomic_load_rhs (load
);
8204 tree stored_val
= gimple_omp_atomic_store_val (store
);
8205 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8206 HOST_WIDE_INT index
;
8208 /* Make sure the type is one of the supported sizes. */
8209 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8210 index
= exact_log2 (index
);
8211 if (index
>= 0 && index
<= 4)
8213 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8215 /* __sync builtins require strict data alignment. */
8216 if (exact_log2 (align
) >= index
)
8219 if (loaded_val
== stored_val
8220 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8221 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8222 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8223 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8227 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8228 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8229 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8230 && store_bb
== single_succ (load_bb
)
8231 && first_stmt (store_bb
) == store
8232 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8236 /* When possible, use specialized atomic update functions. */
8237 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8238 && store_bb
== single_succ (load_bb
)
8239 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8240 loaded_val
, stored_val
, index
))
8243 /* If we don't have specialized __sync builtins, try and implement
8244 as a compare and swap loop. */
8245 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8246 loaded_val
, stored_val
, index
))
8251 /* The ultimate fallback is wrapping the operation in a mutex. */
8252 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8256 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
8259 expand_omp_target (struct omp_region
*region
)
8261 basic_block entry_bb
, exit_bb
, new_bb
;
8262 struct function
*child_cfun
= NULL
;
8263 tree child_fn
= NULL_TREE
, block
, t
;
8264 gimple_stmt_iterator gsi
;
8265 gimple entry_stmt
, stmt
;
8268 entry_stmt
= last_stmt (region
->entry
);
8269 new_bb
= region
->entry
;
8270 int kind
= gimple_omp_target_kind (entry_stmt
);
8271 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8273 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8274 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8277 entry_bb
= region
->entry
;
8278 exit_bb
= region
->exit
;
8280 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8282 unsigned srcidx
, dstidx
, num
;
8284 /* If the target region needs data sent from the parent
8285 function, then the very first statement (except possible
8286 tree profile counter updates) of the parallel body
8287 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8288 &.OMP_DATA_O is passed as an argument to the child function,
8289 we need to replace it with the argument as seen by the child
8292 In most cases, this will end up being the identity assignment
8293 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
8294 a function call that has been inlined, the original PARM_DECL
8295 .OMP_DATA_I may have been converted into a different local
8296 variable. In which case, we need to keep the assignment. */
8297 if (gimple_omp_target_data_arg (entry_stmt
))
8299 basic_block entry_succ_bb
= single_succ (entry_bb
);
8300 gimple_stmt_iterator gsi
;
8302 gimple tgtcopy_stmt
= NULL
;
8304 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
8306 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8308 gcc_assert (!gsi_end_p (gsi
));
8309 stmt
= gsi_stmt (gsi
);
8310 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8313 if (gimple_num_ops (stmt
) == 2)
8315 tree arg
= gimple_assign_rhs1 (stmt
);
8317 /* We're ignoring the subcode because we're
8318 effectively doing a STRIP_NOPS. */
8320 if (TREE_CODE (arg
) == ADDR_EXPR
8321 && TREE_OPERAND (arg
, 0) == sender
)
8323 tgtcopy_stmt
= stmt
;
8329 gcc_assert (tgtcopy_stmt
!= NULL
);
8330 arg
= DECL_ARGUMENTS (child_fn
);
8332 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8333 gsi_remove (&gsi
, true);
8336 /* Declare local variables needed in CHILD_CFUN. */
8337 block
= DECL_INITIAL (child_fn
);
8338 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8339 /* The gimplifier could record temporaries in target block
8340 rather than in containing function's local_decls chain,
8341 which would mean cgraph missed finalizing them. Do it now. */
8342 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8343 if (TREE_CODE (t
) == VAR_DECL
8345 && !DECL_EXTERNAL (t
))
8346 varpool_node::finalize_decl (t
);
8347 DECL_SAVED_TREE (child_fn
) = NULL
;
8348 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8349 gimple_set_body (child_fn
, NULL
);
8350 TREE_USED (block
) = 1;
8352 /* Reset DECL_CONTEXT on function arguments. */
8353 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8354 DECL_CONTEXT (t
) = child_fn
;
8356 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
8357 so that it can be moved to the child function. */
8358 gsi
= gsi_last_bb (entry_bb
);
8359 stmt
= gsi_stmt (gsi
);
8360 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
8361 && gimple_omp_target_kind (stmt
)
8362 == GF_OMP_TARGET_KIND_REGION
);
8363 gsi_remove (&gsi
, true);
8364 e
= split_block (entry_bb
, stmt
);
8366 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8368 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8371 gsi
= gsi_last_bb (exit_bb
);
8372 gcc_assert (!gsi_end_p (gsi
)
8373 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8374 stmt
= gimple_build_return (NULL
);
8375 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8376 gsi_remove (&gsi
, true);
8379 /* Move the target region into CHILD_CFUN. */
8381 block
= gimple_block (entry_stmt
);
8383 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8385 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8386 /* When the OMP expansion process cannot guarantee an up-to-date
8387 loop tree arrange for the child function to fixup loops. */
8388 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8389 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8391 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8392 num
= vec_safe_length (child_cfun
->local_decls
);
8393 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8395 t
= (*child_cfun
->local_decls
)[srcidx
];
8396 if (DECL_CONTEXT (t
) == cfun
->decl
)
8398 if (srcidx
!= dstidx
)
8399 (*child_cfun
->local_decls
)[dstidx
] = t
;
8403 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8405 /* Inform the callgraph about the new function. */
8406 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8407 cgraph_node::add_new_function (child_fn
, true);
8409 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8410 fixed in a following pass. */
8411 push_cfun (child_cfun
);
8412 cgraph_edge::rebuild_edges ();
8414 /* Some EH regions might become dead, see PR34608. If
8415 pass_cleanup_cfg isn't the first pass to happen with the
8416 new child, these dead EH edges might cause problems.
8417 Clean them up now. */
8418 if (flag_exceptions
)
8421 bool changed
= false;
8423 FOR_EACH_BB_FN (bb
, cfun
)
8424 changed
|= gimple_purge_dead_eh_edges (bb
);
8426 cleanup_tree_cfg ();
8431 /* Emit a library call to launch the target region, or do data
8433 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8434 enum built_in_function start_ix
;
8435 location_t clause_loc
;
8437 clauses
= gimple_omp_target_clauses (entry_stmt
);
8439 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8440 start_ix
= BUILT_IN_GOMP_TARGET
;
8441 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
8442 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8444 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8446 /* By default, the value of DEVICE is -1 (let runtime library choose)
8447 and there is no conditional. */
8449 device
= build_int_cst (integer_type_node
, -1);
8451 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
8453 cond
= OMP_CLAUSE_IF_EXPR (c
);
8455 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
8458 device
= OMP_CLAUSE_DEVICE_ID (c
);
8459 clause_loc
= OMP_CLAUSE_LOCATION (c
);
8462 clause_loc
= gimple_location (entry_stmt
);
8464 /* Ensure 'device' is of the correct type. */
8465 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
8467 /* If we found the clause 'if (cond)', build
8468 (cond ? device : -2). */
8471 cond
= gimple_boolify (cond
);
8473 basic_block cond_bb
, then_bb
, else_bb
;
8477 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
8478 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8480 gsi
= gsi_last_bb (new_bb
);
8482 e
= split_block (new_bb
, gsi_stmt (gsi
));
8485 e
= split_block (new_bb
, NULL
);
8490 then_bb
= create_empty_bb (cond_bb
);
8491 else_bb
= create_empty_bb (then_bb
);
8492 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8493 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8495 stmt
= gimple_build_cond_empty (cond
);
8496 gsi
= gsi_last_bb (cond_bb
);
8497 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8499 gsi
= gsi_start_bb (then_bb
);
8500 stmt
= gimple_build_assign (tmp_var
, device
);
8501 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8503 gsi
= gsi_start_bb (else_bb
);
8504 stmt
= gimple_build_assign (tmp_var
,
8505 build_int_cst (integer_type_node
, -2));
8506 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8508 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8509 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8510 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8511 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8512 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8513 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8518 gsi
= gsi_last_bb (new_bb
);
8519 t
= gimple_omp_target_data_arg (entry_stmt
);
8522 t1
= size_zero_node
;
8523 t2
= build_zero_cst (ptr_type_node
);
8529 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8530 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8531 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8532 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8533 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8537 /* FIXME: This will be address of
8538 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8539 symbol, as soon as the linker plugin is able to create it for us. */
8540 tree openmp_target
= build_zero_cst (ptr_type_node
);
8541 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8543 tree fnaddr
= build_fold_addr_expr (child_fn
);
8544 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8545 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8548 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8549 device
, openmp_target
, t1
, t2
, t3
, t4
);
8550 gimple_set_location (g
, gimple_location (entry_stmt
));
8551 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8552 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8555 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8556 gsi_remove (&gsi
, true);
8558 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8560 gsi
= gsi_last_bb (region
->exit
);
8562 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8563 gsi_remove (&gsi
, true);
8568 /* Expand the parallel region tree rooted at REGION. Expansion
8569 proceeds in depth-first order. Innermost regions are expanded
8570 first. This way, parallel regions that require a new function to
8571 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8572 internal dependencies in their body. */
8575 expand_omp (struct omp_region
*region
)
8579 location_t saved_location
;
8580 gimple inner_stmt
= NULL
;
8582 /* First, determine whether this is a combined parallel+workshare
8584 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8585 determine_parallel_type (region
);
8587 if (region
->type
== GIMPLE_OMP_FOR
8588 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8589 inner_stmt
= last_stmt (region
->inner
->entry
);
8592 expand_omp (region
->inner
);
8594 saved_location
= input_location
;
8595 if (gimple_has_location (last_stmt (region
->entry
)))
8596 input_location
= gimple_location (last_stmt (region
->entry
));
8598 switch (region
->type
)
8600 case GIMPLE_OMP_PARALLEL
:
8601 case GIMPLE_OMP_TASK
:
8602 expand_omp_taskreg (region
);
8605 case GIMPLE_OMP_FOR
:
8606 expand_omp_for (region
, inner_stmt
);
8609 case GIMPLE_OMP_SECTIONS
:
8610 expand_omp_sections (region
);
8613 case GIMPLE_OMP_SECTION
:
8614 /* Individual omp sections are handled together with their
8615 parent GIMPLE_OMP_SECTIONS region. */
8618 case GIMPLE_OMP_SINGLE
:
8619 expand_omp_single (region
);
8622 case GIMPLE_OMP_MASTER
:
8623 case GIMPLE_OMP_TASKGROUP
:
8624 case GIMPLE_OMP_ORDERED
:
8625 case GIMPLE_OMP_CRITICAL
:
8626 case GIMPLE_OMP_TEAMS
:
8627 expand_omp_synch (region
);
8630 case GIMPLE_OMP_ATOMIC_LOAD
:
8631 expand_omp_atomic (region
);
8634 case GIMPLE_OMP_TARGET
:
8635 expand_omp_target (region
);
8642 input_location
= saved_location
;
8643 region
= region
->next
;
8648 /* Helper for build_omp_regions. Scan the dominator tree starting at
8649 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8650 true, the function ends once a single tree is built (otherwise, whole
8651 forest of OMP constructs may be built). */
8654 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8657 gimple_stmt_iterator gsi
;
8661 gsi
= gsi_last_bb (bb
);
8662 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8664 struct omp_region
*region
;
8665 enum gimple_code code
;
8667 stmt
= gsi_stmt (gsi
);
8668 code
= gimple_code (stmt
);
8669 if (code
== GIMPLE_OMP_RETURN
)
8671 /* STMT is the return point out of region PARENT. Mark it
8672 as the exit point and make PARENT the immediately
8673 enclosing region. */
8674 gcc_assert (parent
);
8677 parent
= parent
->outer
;
8679 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8681 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8682 GIMPLE_OMP_RETURN, but matches with
8683 GIMPLE_OMP_ATOMIC_LOAD. */
8684 gcc_assert (parent
);
8685 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8688 parent
= parent
->outer
;
8691 else if (code
== GIMPLE_OMP_CONTINUE
)
8693 gcc_assert (parent
);
8696 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8698 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8699 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8702 else if (code
== GIMPLE_OMP_TARGET
8703 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8704 new_omp_region (bb
, code
, parent
);
8707 /* Otherwise, this directive becomes the parent for a new
8709 region
= new_omp_region (bb
, code
, parent
);
8714 if (single_tree
&& !parent
)
8717 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8719 son
= next_dom_son (CDI_DOMINATORS
, son
))
8720 build_omp_regions_1 (son
, parent
, single_tree
);
8723 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8727 build_omp_regions_root (basic_block root
)
8729 gcc_assert (root_omp_region
== NULL
);
8730 build_omp_regions_1 (root
, NULL
, true);
8731 gcc_assert (root_omp_region
!= NULL
);
8734 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8737 omp_expand_local (basic_block head
)
8739 build_omp_regions_root (head
);
8740 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8742 fprintf (dump_file
, "\nOMP region tree\n\n");
8743 dump_omp_region (dump_file
, root_omp_region
, 0);
8744 fprintf (dump_file
, "\n");
8747 remove_exit_barriers (root_omp_region
);
8748 expand_omp (root_omp_region
);
8750 free_omp_regions ();
8753 /* Scan the CFG and build a tree of OMP regions. Return the root of
8754 the OMP region tree. */
8757 build_omp_regions (void)
8759 gcc_assert (root_omp_region
== NULL
);
8760 calculate_dominance_info (CDI_DOMINATORS
);
8761 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8764 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8767 execute_expand_omp (void)
8769 build_omp_regions ();
8771 if (!root_omp_region
)
8776 fprintf (dump_file
, "\nOMP region tree\n\n");
8777 dump_omp_region (dump_file
, root_omp_region
, 0);
8778 fprintf (dump_file
, "\n");
8781 remove_exit_barriers (root_omp_region
);
8783 expand_omp (root_omp_region
);
8785 cleanup_tree_cfg ();
8787 free_omp_regions ();
8792 /* OMP expansion -- the default pass, run before creation of SSA form. */
8796 const pass_data pass_data_expand_omp
=
8798 GIMPLE_PASS
, /* type */
8799 "ompexp", /* name */
8800 OPTGROUP_NONE
, /* optinfo_flags */
8801 TV_NONE
, /* tv_id */
8802 PROP_gimple_any
, /* properties_required */
8803 0, /* properties_provided */
8804 0, /* properties_destroyed */
8805 0, /* todo_flags_start */
8806 0, /* todo_flags_finish */
8809 class pass_expand_omp
: public gimple_opt_pass
8812 pass_expand_omp (gcc::context
*ctxt
)
8813 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8816 /* opt_pass methods: */
8817 virtual bool gate (function
*)
8819 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8820 || flag_cilkplus
!= 0) && !seen_error ());
8823 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
8825 }; // class pass_expand_omp
8830 make_pass_expand_omp (gcc::context
*ctxt
)
8832 return new pass_expand_omp (ctxt
);
8835 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8837 /* If ctx is a worksharing context inside of a cancellable parallel
8838 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8839 and conditional branch to parallel's cancel_label to handle
8840 cancellation in the implicit barrier. */
8843 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8845 gimple omp_return
= gimple_seq_last_stmt (*body
);
8846 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8847 if (gimple_omp_return_nowait_p (omp_return
))
8850 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8851 && ctx
->outer
->cancellable
)
8853 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
8854 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
8855 tree lhs
= create_tmp_var (c_bool_type
, NULL
);
8856 gimple_omp_return_set_lhs (omp_return
, lhs
);
8857 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8858 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
8859 fold_convert (c_bool_type
,
8860 boolean_false_node
),
8861 ctx
->outer
->cancel_label
, fallthru_label
);
8862 gimple_seq_add_stmt (body
, g
);
8863 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8867 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8868 CTX is the enclosing OMP context for the current statement. */
8871 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8873 tree block
, control
;
8874 gimple_stmt_iterator tgsi
;
8875 gimple stmt
, new_stmt
, bind
, t
;
8876 gimple_seq ilist
, dlist
, olist
, new_body
;
8878 stmt
= gsi_stmt (*gsi_p
);
8880 push_gimplify_context ();
8884 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8885 &ilist
, &dlist
, ctx
, NULL
);
8887 new_body
= gimple_omp_body (stmt
);
8888 gimple_omp_set_body (stmt
, NULL
);
8889 tgsi
= gsi_start (new_body
);
8890 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8895 sec_start
= gsi_stmt (tgsi
);
8896 sctx
= maybe_lookup_ctx (sec_start
);
8899 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8900 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8901 GSI_CONTINUE_LINKING
);
8902 gimple_omp_set_body (sec_start
, NULL
);
8904 if (gsi_one_before_end_p (tgsi
))
8906 gimple_seq l
= NULL
;
8907 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8909 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8910 gimple_omp_section_set_last (sec_start
);
8913 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8914 GSI_CONTINUE_LINKING
);
8917 block
= make_node (BLOCK
);
8918 bind
= gimple_build_bind (NULL
, new_body
, block
);
8921 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8923 block
= make_node (BLOCK
);
8924 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8925 gsi_replace (gsi_p
, new_stmt
, true);
8927 pop_gimplify_context (new_stmt
);
8928 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8929 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8930 if (BLOCK_VARS (block
))
8931 TREE_USED (block
) = 1;
8934 gimple_seq_add_seq (&new_body
, ilist
);
8935 gimple_seq_add_stmt (&new_body
, stmt
);
8936 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8937 gimple_seq_add_stmt (&new_body
, bind
);
8939 control
= create_tmp_var (unsigned_type_node
, ".section");
8940 t
= gimple_build_omp_continue (control
, control
);
8941 gimple_omp_sections_set_control (stmt
, control
);
8942 gimple_seq_add_stmt (&new_body
, t
);
8944 gimple_seq_add_seq (&new_body
, olist
);
8945 if (ctx
->cancellable
)
8946 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8947 gimple_seq_add_seq (&new_body
, dlist
);
8949 new_body
= maybe_catch_exception (new_body
);
8951 t
= gimple_build_omp_return
8952 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8953 OMP_CLAUSE_NOWAIT
));
8954 gimple_seq_add_stmt (&new_body
, t
);
8955 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8957 gimple_bind_set_body (new_stmt
, new_body
);
8961 /* A subroutine of lower_omp_single. Expand the simple form of
8962 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8964 if (GOMP_single_start ())
8966 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8968 FIXME. It may be better to delay expanding the logic of this until
8969 pass_expand_omp. The expanded logic may make the job more difficult
8970 to a synchronization analysis pass. */
8973 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8975 location_t loc
= gimple_location (single_stmt
);
8976 tree tlabel
= create_artificial_label (loc
);
8977 tree flabel
= create_artificial_label (loc
);
8981 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8982 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8983 call
= gimple_build_call (decl
, 0);
8984 gimple_call_set_lhs (call
, lhs
);
8985 gimple_seq_add_stmt (pre_p
, call
);
8987 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8988 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8991 gimple_seq_add_stmt (pre_p
, cond
);
8992 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8993 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8994 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8998 /* A subroutine of lower_omp_single. Expand the simple form of
8999 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
9001 #pragma omp single copyprivate (a, b, c)
9003 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
9006 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
9012 GOMP_single_copy_end (©out);
9023 FIXME. It may be better to delay expanding the logic of this until
9024 pass_expand_omp. The expanded logic may make the job more difficult
9025 to a synchronization analysis pass. */
9028 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
9030 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
9031 gimple_seq copyin_seq
;
9032 location_t loc
= gimple_location (single_stmt
);
9034 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
9036 ptr_type
= build_pointer_type (ctx
->record_type
);
9037 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
9039 l0
= create_artificial_label (loc
);
9040 l1
= create_artificial_label (loc
);
9041 l2
= create_artificial_label (loc
);
9043 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
9044 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
9045 t
= fold_convert_loc (loc
, ptr_type
, t
);
9046 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
9048 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
9049 build_int_cst (ptr_type
, 0));
9050 t
= build3 (COND_EXPR
, void_type_node
, t
,
9051 build_and_jump (&l0
), build_and_jump (&l1
));
9052 gimplify_and_add (t
, pre_p
);
9054 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
9056 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
9059 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
9062 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9063 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
9064 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
9065 gimplify_and_add (t
, pre_p
);
9067 t
= build_and_jump (&l2
);
9068 gimplify_and_add (t
, pre_p
);
9070 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
9072 gimple_seq_add_seq (pre_p
, copyin_seq
);
9074 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
9078 /* Expand code for an OpenMP single directive. */
9081 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9084 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
9085 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
9087 push_gimplify_context ();
9089 block
= make_node (BLOCK
);
9090 bind
= gimple_build_bind (NULL
, NULL
, block
);
9091 gsi_replace (gsi_p
, bind
, true);
9094 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
9095 &bind_body
, &dlist
, ctx
, NULL
);
9096 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
9098 gimple_seq_add_stmt (&bind_body
, single_stmt
);
9100 if (ctx
->record_type
)
9101 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
9103 lower_omp_single_simple (single_stmt
, &bind_body
);
9105 gimple_omp_set_body (single_stmt
, NULL
);
9107 gimple_seq_add_seq (&bind_body
, dlist
);
9109 bind_body
= maybe_catch_exception (bind_body
);
9111 t
= gimple_build_omp_return
9112 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
9113 OMP_CLAUSE_NOWAIT
));
9114 gimple_seq_add_stmt (&bind_body_tail
, t
);
9115 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
9116 if (ctx
->record_type
)
9118 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
9119 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9120 TREE_THIS_VOLATILE (clobber
) = 1;
9121 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
9122 clobber
), GSI_SAME_STMT
);
9124 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
9125 gimple_bind_set_body (bind
, bind_body
);
9127 pop_gimplify_context (bind
);
9129 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9130 BLOCK_VARS (block
) = ctx
->block_vars
;
9131 if (BLOCK_VARS (block
))
9132 TREE_USED (block
) = 1;
9136 /* Expand code for an OpenMP master directive. */
9139 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9141 tree block
, lab
= NULL
, x
, bfn_decl
;
9142 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
9143 location_t loc
= gimple_location (stmt
);
9146 push_gimplify_context ();
9148 block
= make_node (BLOCK
);
9149 bind
= gimple_build_bind (NULL
, NULL
, block
);
9150 gsi_replace (gsi_p
, bind
, true);
9151 gimple_bind_add_stmt (bind
, stmt
);
9153 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9154 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
9155 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
9156 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
9158 gimplify_and_add (x
, &tseq
);
9159 gimple_bind_add_seq (bind
, tseq
);
9161 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9162 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9163 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9164 gimple_omp_set_body (stmt
, NULL
);
9166 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
9168 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9170 pop_gimplify_context (bind
);
9172 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9173 BLOCK_VARS (block
) = ctx
->block_vars
;
9177 /* Expand code for an OpenMP taskgroup directive. */
9180 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9182 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
9183 tree block
= make_node (BLOCK
);
9185 bind
= gimple_build_bind (NULL
, NULL
, block
);
9186 gsi_replace (gsi_p
, bind
, true);
9187 gimple_bind_add_stmt (bind
, stmt
);
9189 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
9191 gimple_bind_add_stmt (bind
, x
);
9193 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9194 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9195 gimple_omp_set_body (stmt
, NULL
);
9197 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9199 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9200 BLOCK_VARS (block
) = ctx
->block_vars
;
9204 /* Expand code for an OpenMP ordered directive. */
9207 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9210 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
9212 push_gimplify_context ();
9214 block
= make_node (BLOCK
);
9215 bind
= gimple_build_bind (NULL
, NULL
, block
);
9216 gsi_replace (gsi_p
, bind
, true);
9217 gimple_bind_add_stmt (bind
, stmt
);
9219 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
9221 gimple_bind_add_stmt (bind
, x
);
9223 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9224 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9225 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9226 gimple_omp_set_body (stmt
, NULL
);
9228 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
9229 gimple_bind_add_stmt (bind
, x
);
9231 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9233 pop_gimplify_context (bind
);
9235 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9236 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9240 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
9241 substitution of a couple of function calls. But in the NAMED case,
9242 requires that languages coordinate a symbol name. It is therefore
9243 best put here in common code. */
9245 static GTY((param1_is (tree
), param2_is (tree
)))
9246 splay_tree critical_name_mutexes
;
9249 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9252 tree name
, lock
, unlock
;
9253 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
9254 location_t loc
= gimple_location (stmt
);
9257 name
= gimple_omp_critical_name (stmt
);
9263 if (!critical_name_mutexes
)
9264 critical_name_mutexes
9265 = splay_tree_new_ggc (splay_tree_compare_pointers
,
9266 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
9267 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
9269 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
9274 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
9276 new_str
= ACONCAT ((".gomp_critical_user_",
9277 IDENTIFIER_POINTER (name
), NULL
));
9278 DECL_NAME (decl
) = get_identifier (new_str
);
9279 TREE_PUBLIC (decl
) = 1;
9280 TREE_STATIC (decl
) = 1;
9281 DECL_COMMON (decl
) = 1;
9282 DECL_ARTIFICIAL (decl
) = 1;
9283 DECL_IGNORED_P (decl
) = 1;
9284 varpool_node::finalize_decl (decl
);
9286 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
9287 (splay_tree_value
) decl
);
9290 decl
= (tree
) n
->value
;
9292 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
9293 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
9295 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
9296 unlock
= build_call_expr_loc (loc
, unlock
, 1,
9297 build_fold_addr_expr_loc (loc
, decl
));
9301 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
9302 lock
= build_call_expr_loc (loc
, lock
, 0);
9304 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
9305 unlock
= build_call_expr_loc (loc
, unlock
, 0);
9308 push_gimplify_context ();
9310 block
= make_node (BLOCK
);
9311 bind
= gimple_build_bind (NULL
, NULL
, block
);
9312 gsi_replace (gsi_p
, bind
, true);
9313 gimple_bind_add_stmt (bind
, stmt
);
9315 tbody
= gimple_bind_body (bind
);
9316 gimplify_and_add (lock
, &tbody
);
9317 gimple_bind_set_body (bind
, tbody
);
9319 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9320 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9321 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9322 gimple_omp_set_body (stmt
, NULL
);
9324 tbody
= gimple_bind_body (bind
);
9325 gimplify_and_add (unlock
, &tbody
);
9326 gimple_bind_set_body (bind
, tbody
);
9328 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9330 pop_gimplify_context (bind
);
9331 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9332 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9336 /* A subroutine of lower_omp_for. Generate code to emit the predicate
9337 for a lastprivate clause. Given a loop control predicate of (V
9338 cond N2), we gate the clause on (!(V cond N2)). The lowered form
9339 is appended to *DLIST, iterator initialization is appended to
9343 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
9344 gimple_seq
*dlist
, struct omp_context
*ctx
)
9346 tree clauses
, cond
, vinit
;
9347 enum tree_code cond_code
;
9350 cond_code
= fd
->loop
.cond_code
;
9351 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
9353 /* When possible, use a strict equality expression. This can let VRP
9354 type optimizations deduce the value and remove a copy. */
9355 if (tree_fits_shwi_p (fd
->loop
.step
))
9357 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
9358 if (step
== 1 || step
== -1)
9359 cond_code
= EQ_EXPR
;
9362 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
9364 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
9366 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
9367 if (!gimple_seq_empty_p (stmts
))
9369 gimple_seq_add_seq (&stmts
, *dlist
);
9372 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
9373 vinit
= fd
->loop
.n1
;
9374 if (cond_code
== EQ_EXPR
9375 && tree_fits_shwi_p (fd
->loop
.n2
)
9376 && ! integer_zerop (fd
->loop
.n2
))
9377 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
9379 vinit
= unshare_expr (vinit
);
9381 /* Initialize the iterator variable, so that threads that don't execute
9382 any iterations don't execute the lastprivate clauses by accident. */
9383 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
9388 /* Lower code for an OpenMP loop directive. */
9391 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9394 struct omp_for_data fd
, *fdp
= NULL
;
9395 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
9396 gimple_seq omp_for_body
, body
, dlist
;
9399 push_gimplify_context ();
9401 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
9403 block
= make_node (BLOCK
);
9404 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
9405 /* Replace at gsi right away, so that 'stmt' is no member
9406 of a sequence anymore as we're going to add to to a different
9408 gsi_replace (gsi_p
, new_stmt
, true);
9410 /* Move declaration of temporaries in the loop body before we make
9412 omp_for_body
= gimple_omp_body (stmt
);
9413 if (!gimple_seq_empty_p (omp_for_body
)
9414 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
9416 gimple inner_bind
= gimple_seq_first_stmt (omp_for_body
);
9417 tree vars
= gimple_bind_vars (inner_bind
);
9418 gimple_bind_append_vars (new_stmt
, vars
);
9419 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
9420 keep them on the inner_bind and it's block. */
9421 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
9422 if (gimple_bind_block (inner_bind
))
9423 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
9426 if (gimple_omp_for_combined_into_p (stmt
))
9428 extract_omp_for_data (stmt
, &fd
, NULL
);
9431 /* We need two temporaries with fd.loop.v type (istart/iend)
9432 and then (fd.collapse - 1) temporaries with the same
9433 type for count2 ... countN-1 vars if not constant. */
9435 tree type
= fd
.iter_type
;
9437 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
9438 count
+= fd
.collapse
- 1;
9439 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
9440 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
9444 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
9445 OMP_CLAUSE__LOOPTEMP_
);
9446 for (i
= 0; i
< count
; i
++)
9451 gcc_assert (outerc
);
9452 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
9453 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
9454 OMP_CLAUSE__LOOPTEMP_
);
9458 temp
= create_tmp_var (type
, NULL
);
9459 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
9461 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
9462 OMP_CLAUSE_DECL (*pc
) = temp
;
9463 pc
= &OMP_CLAUSE_CHAIN (*pc
);
9468 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9471 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
9473 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
9475 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9477 /* Lower the header expressions. At this point, we can assume that
9478 the header is of the form:
9480 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9482 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9483 using the .omp_data_s mapping, if needed. */
9484 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
9486 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
9487 if (!is_gimple_min_invariant (*rhs_p
))
9488 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9490 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
9491 if (!is_gimple_min_invariant (*rhs_p
))
9492 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9494 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
9495 if (!is_gimple_min_invariant (*rhs_p
))
9496 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9499 /* Once lowered, extract the bounds and clauses. */
9500 extract_omp_for_data (stmt
, &fd
, NULL
);
9502 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
9504 gimple_seq_add_stmt (&body
, stmt
);
9505 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
9507 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
9510 /* After the loop, add exit clauses. */
9511 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
9513 if (ctx
->cancellable
)
9514 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
9516 gimple_seq_add_seq (&body
, dlist
);
9518 body
= maybe_catch_exception (body
);
9520 /* Region exit marker goes at the end of the loop body. */
9521 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
9522 maybe_add_implicit_barrier_cancel (ctx
, &body
);
9523 pop_gimplify_context (new_stmt
);
9525 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9526 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9527 if (BLOCK_VARS (block
))
9528 TREE_USED (block
) = 1;
9530 gimple_bind_set_body (new_stmt
, body
);
9531 gimple_omp_set_body (stmt
, NULL
);
9532 gimple_omp_for_set_pre_body (stmt
, NULL
);
9535 /* Callback for walk_stmts. Check if the current statement only contains
9536 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9539 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9540 bool *handled_ops_p
,
9541 struct walk_stmt_info
*wi
)
9543 int *info
= (int *) wi
->info
;
9544 gimple stmt
= gsi_stmt (*gsi_p
);
9546 *handled_ops_p
= true;
9547 switch (gimple_code (stmt
))
9551 case GIMPLE_OMP_FOR
:
9552 case GIMPLE_OMP_SECTIONS
:
9553 *info
= *info
== 0 ? 1 : -1;
9562 struct omp_taskcopy_context
9564 /* This field must be at the beginning, as we do "inheritance": Some
9565 callback functions for tree-inline.c (e.g., omp_copy_decl)
9566 receive a copy_body_data pointer that is up-casted to an
9567 omp_context pointer. */
9573 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9575 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9577 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9578 return create_tmp_var (TREE_TYPE (var
), NULL
);
9584 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9586 tree name
, new_fields
= NULL
, type
, f
;
9588 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9589 name
= DECL_NAME (TYPE_NAME (orig_type
));
9590 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9591 TYPE_DECL
, name
, type
);
9592 TYPE_NAME (type
) = name
;
9594 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9596 tree new_f
= copy_node (f
);
9597 DECL_CONTEXT (new_f
) = type
;
9598 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9599 TREE_CHAIN (new_f
) = new_fields
;
9600 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9601 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9602 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9605 tcctx
->cb
.decl_map
->put (f
, new_f
);
9607 TYPE_FIELDS (type
) = nreverse (new_fields
);
9612 /* Create task copyfn. */
9615 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9617 struct function
*child_cfun
;
9618 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9619 tree record_type
, srecord_type
, bind
, list
;
9620 bool record_needs_remap
= false, srecord_needs_remap
= false;
9622 struct omp_taskcopy_context tcctx
;
9623 location_t loc
= gimple_location (task_stmt
);
9625 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9626 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9627 gcc_assert (child_cfun
->cfg
== NULL
);
9628 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9630 /* Reset DECL_CONTEXT on function arguments. */
9631 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9632 DECL_CONTEXT (t
) = child_fn
;
9634 /* Populate the function. */
9635 push_gimplify_context ();
9636 push_cfun (child_cfun
);
9638 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9639 TREE_SIDE_EFFECTS (bind
) = 1;
9641 DECL_SAVED_TREE (child_fn
) = bind
;
9642 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9644 /* Remap src and dst argument types if needed. */
9645 record_type
= ctx
->record_type
;
9646 srecord_type
= ctx
->srecord_type
;
9647 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9648 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9650 record_needs_remap
= true;
9653 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9654 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9656 srecord_needs_remap
= true;
9660 if (record_needs_remap
|| srecord_needs_remap
)
9662 memset (&tcctx
, '\0', sizeof (tcctx
));
9663 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9664 tcctx
.cb
.dst_fn
= child_fn
;
9665 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
9666 gcc_checking_assert (tcctx
.cb
.src_node
);
9667 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9668 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9669 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9670 tcctx
.cb
.eh_lp_nr
= 0;
9671 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9672 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
9675 if (record_needs_remap
)
9676 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9677 if (srecord_needs_remap
)
9678 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9681 tcctx
.cb
.decl_map
= NULL
;
9683 arg
= DECL_ARGUMENTS (child_fn
);
9684 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9685 sarg
= DECL_CHAIN (arg
);
9686 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9688 /* First pass: initialize temporaries used in record_type and srecord_type
9689 sizes and field offsets. */
9690 if (tcctx
.cb
.decl_map
)
9691 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9692 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9696 decl
= OMP_CLAUSE_DECL (c
);
9697 p
= tcctx
.cb
.decl_map
->get (decl
);
9700 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9701 sf
= (tree
) n
->value
;
9702 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9703 src
= build_simple_mem_ref_loc (loc
, sarg
);
9704 src
= omp_build_component_ref (src
, sf
);
9705 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9706 append_to_statement_list (t
, &list
);
9709 /* Second pass: copy shared var pointers and copy construct non-VLA
9710 firstprivate vars. */
9711 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9712 switch (OMP_CLAUSE_CODE (c
))
9714 case OMP_CLAUSE_SHARED
:
9715 decl
= OMP_CLAUSE_DECL (c
);
9716 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9719 f
= (tree
) n
->value
;
9720 if (tcctx
.cb
.decl_map
)
9721 f
= *tcctx
.cb
.decl_map
->get (f
);
9722 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9723 sf
= (tree
) n
->value
;
9724 if (tcctx
.cb
.decl_map
)
9725 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9726 src
= build_simple_mem_ref_loc (loc
, sarg
);
9727 src
= omp_build_component_ref (src
, sf
);
9728 dst
= build_simple_mem_ref_loc (loc
, arg
);
9729 dst
= omp_build_component_ref (dst
, f
);
9730 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9731 append_to_statement_list (t
, &list
);
9733 case OMP_CLAUSE_FIRSTPRIVATE
:
9734 decl
= OMP_CLAUSE_DECL (c
);
9735 if (is_variable_sized (decl
))
9737 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9740 f
= (tree
) n
->value
;
9741 if (tcctx
.cb
.decl_map
)
9742 f
= *tcctx
.cb
.decl_map
->get (f
);
9743 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9746 sf
= (tree
) n
->value
;
9747 if (tcctx
.cb
.decl_map
)
9748 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9749 src
= build_simple_mem_ref_loc (loc
, sarg
);
9750 src
= omp_build_component_ref (src
, sf
);
9751 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9752 src
= build_simple_mem_ref_loc (loc
, src
);
9756 dst
= build_simple_mem_ref_loc (loc
, arg
);
9757 dst
= omp_build_component_ref (dst
, f
);
9758 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9759 append_to_statement_list (t
, &list
);
9761 case OMP_CLAUSE_PRIVATE
:
9762 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9764 decl
= OMP_CLAUSE_DECL (c
);
9765 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9766 f
= (tree
) n
->value
;
9767 if (tcctx
.cb
.decl_map
)
9768 f
= *tcctx
.cb
.decl_map
->get (f
);
9769 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9772 sf
= (tree
) n
->value
;
9773 if (tcctx
.cb
.decl_map
)
9774 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9775 src
= build_simple_mem_ref_loc (loc
, sarg
);
9776 src
= omp_build_component_ref (src
, sf
);
9777 if (use_pointer_for_field (decl
, NULL
))
9778 src
= build_simple_mem_ref_loc (loc
, src
);
9782 dst
= build_simple_mem_ref_loc (loc
, arg
);
9783 dst
= omp_build_component_ref (dst
, f
);
9784 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9785 append_to_statement_list (t
, &list
);
9791 /* Last pass: handle VLA firstprivates. */
9792 if (tcctx
.cb
.decl_map
)
9793 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9794 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9798 decl
= OMP_CLAUSE_DECL (c
);
9799 if (!is_variable_sized (decl
))
9801 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9804 f
= (tree
) n
->value
;
9805 f
= *tcctx
.cb
.decl_map
->get (f
);
9806 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9807 ind
= DECL_VALUE_EXPR (decl
);
9808 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9809 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9810 n
= splay_tree_lookup (ctx
->sfield_map
,
9811 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9812 sf
= (tree
) n
->value
;
9813 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9814 src
= build_simple_mem_ref_loc (loc
, sarg
);
9815 src
= omp_build_component_ref (src
, sf
);
9816 src
= build_simple_mem_ref_loc (loc
, src
);
9817 dst
= build_simple_mem_ref_loc (loc
, arg
);
9818 dst
= omp_build_component_ref (dst
, f
);
9819 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9820 append_to_statement_list (t
, &list
);
9821 n
= splay_tree_lookup (ctx
->field_map
,
9822 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9823 df
= (tree
) n
->value
;
9824 df
= *tcctx
.cb
.decl_map
->get (df
);
9825 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9826 ptr
= omp_build_component_ref (ptr
, df
);
9827 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9828 build_fold_addr_expr_loc (loc
, dst
));
9829 append_to_statement_list (t
, &list
);
9832 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9833 append_to_statement_list (t
, &list
);
9835 if (tcctx
.cb
.decl_map
)
9836 delete tcctx
.cb
.decl_map
;
9837 pop_gimplify_context (NULL
);
9838 BIND_EXPR_BODY (bind
) = list
;
9843 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9847 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9849 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9851 gcc_assert (clauses
);
9852 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9853 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9854 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9856 case OMP_CLAUSE_DEPEND_IN
:
9859 case OMP_CLAUSE_DEPEND_OUT
:
9860 case OMP_CLAUSE_DEPEND_INOUT
:
9866 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9867 tree array
= create_tmp_var (type
, NULL
);
9868 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9870 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9871 gimple_seq_add_stmt (iseq
, g
);
9872 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9874 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9875 gimple_seq_add_stmt (iseq
, g
);
9876 for (i
= 0; i
< 2; i
++)
9878 if ((i
? n_in
: n_out
) == 0)
9880 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9881 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9882 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9884 tree t
= OMP_CLAUSE_DECL (c
);
9885 t
= fold_convert (ptr_type_node
, t
);
9886 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9887 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9888 NULL_TREE
, NULL_TREE
);
9889 g
= gimple_build_assign (r
, t
);
9890 gimple_seq_add_stmt (iseq
, g
);
9893 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9894 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9895 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9896 OMP_CLAUSE_CHAIN (c
) = *p
;
9898 tree clobber
= build_constructor (type
, NULL
);
9899 TREE_THIS_VOLATILE (clobber
) = 1;
9900 g
= gimple_build_assign (array
, clobber
);
9901 gimple_seq_add_stmt (oseq
, g
);
9904 /* Lower the OpenMP parallel or task directive in the current statement
9905 in GSI_P. CTX holds context information for the directive. */
9908 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9912 gimple stmt
= gsi_stmt (*gsi_p
);
9913 gimple par_bind
, bind
, dep_bind
= NULL
;
9914 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9915 location_t loc
= gimple_location (stmt
);
9917 clauses
= gimple_omp_taskreg_clauses (stmt
);
9918 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9919 par_body
= gimple_bind_body (par_bind
);
9920 child_fn
= ctx
->cb
.dst_fn
;
9921 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9922 && !gimple_omp_parallel_combined_p (stmt
))
9924 struct walk_stmt_info wi
;
9927 memset (&wi
, 0, sizeof (wi
));
9930 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9932 gimple_omp_parallel_set_combined_p (stmt
, true);
9934 gimple_seq dep_ilist
= NULL
;
9935 gimple_seq dep_olist
= NULL
;
9936 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9937 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9939 push_gimplify_context ();
9940 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9941 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9944 if (ctx
->srecord_type
)
9945 create_task_copyfn (stmt
, ctx
);
9947 push_gimplify_context ();
9952 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9953 lower_omp (&par_body
, ctx
);
9954 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9955 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9957 /* Declare all the variables created by mapping and the variables
9958 declared in the scope of the parallel body. */
9959 record_vars_into (ctx
->block_vars
, child_fn
);
9960 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9962 if (ctx
->record_type
)
9965 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9966 : ctx
->record_type
, ".omp_data_o");
9967 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9968 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9969 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9974 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9975 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9977 if (ctx
->record_type
)
9979 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9980 TREE_THIS_VOLATILE (clobber
) = 1;
9981 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9985 /* Once all the expansions are done, sequence all the different
9986 fragments inside gimple_omp_body. */
9990 if (ctx
->record_type
)
9992 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9993 /* fixup_child_record_type might have changed receiver_decl's type. */
9994 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9995 gimple_seq_add_stmt (&new_body
,
9996 gimple_build_assign (ctx
->receiver_decl
, t
));
9999 gimple_seq_add_seq (&new_body
, par_ilist
);
10000 gimple_seq_add_seq (&new_body
, par_body
);
10001 gimple_seq_add_seq (&new_body
, par_rlist
);
10002 if (ctx
->cancellable
)
10003 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10004 gimple_seq_add_seq (&new_body
, par_olist
);
10005 new_body
= maybe_catch_exception (new_body
);
10006 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
10007 gimple_omp_set_body (stmt
, new_body
);
10009 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
10010 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
10011 gimple_bind_add_seq (bind
, ilist
);
10012 gimple_bind_add_stmt (bind
, stmt
);
10013 gimple_bind_add_seq (bind
, olist
);
10015 pop_gimplify_context (NULL
);
10019 gimple_bind_add_seq (dep_bind
, dep_ilist
);
10020 gimple_bind_add_stmt (dep_bind
, bind
);
10021 gimple_bind_add_seq (dep_bind
, dep_olist
);
10022 pop_gimplify_context (dep_bind
);
10026 /* Lower the OpenMP target directive in the current statement
10027 in GSI_P. CTX holds context information for the directive. */
10030 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10033 tree child_fn
, t
, c
;
10034 gimple stmt
= gsi_stmt (*gsi_p
);
10035 gimple tgt_bind
= NULL
, bind
;
10036 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
10037 location_t loc
= gimple_location (stmt
);
10038 int kind
= gimple_omp_target_kind (stmt
);
10039 unsigned int map_cnt
= 0;
10041 clauses
= gimple_omp_target_clauses (stmt
);
10042 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10044 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
10045 tgt_body
= gimple_bind_body (tgt_bind
);
10047 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10048 tgt_body
= gimple_omp_body (stmt
);
10049 child_fn
= ctx
->cb
.dst_fn
;
10051 push_gimplify_context ();
10053 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10054 switch (OMP_CLAUSE_CODE (c
))
10060 case OMP_CLAUSE_MAP
:
10061 case OMP_CLAUSE_TO
:
10062 case OMP_CLAUSE_FROM
:
10063 var
= OMP_CLAUSE_DECL (c
);
10066 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
10067 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
10072 if (DECL_SIZE (var
)
10073 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
10075 tree var2
= DECL_VALUE_EXPR (var
);
10076 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
10077 var2
= TREE_OPERAND (var2
, 0);
10078 gcc_assert (DECL_P (var2
));
10082 if (!maybe_lookup_field (var
, ctx
))
10085 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10087 x
= build_receiver_ref (var
, true, ctx
);
10088 tree new_var
= lookup_decl (var
, ctx
);
10089 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10090 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
10091 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
10092 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
10093 x
= build_simple_mem_ref (x
);
10094 SET_DECL_VALUE_EXPR (new_var
, x
);
10095 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
10100 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10102 target_nesting_level
++;
10103 lower_omp (&tgt_body
, ctx
);
10104 target_nesting_level
--;
10106 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10107 lower_omp (&tgt_body
, ctx
);
10109 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10111 /* Declare all the variables created by mapping and the variables
10112 declared in the scope of the target body. */
10113 record_vars_into (ctx
->block_vars
, child_fn
);
10114 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
10119 if (ctx
->record_type
)
10122 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
10123 DECL_NAMELESS (ctx
->sender_decl
) = 1;
10124 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
10125 t
= make_tree_vec (3);
10126 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
10127 TREE_VEC_ELT (t
, 1)
10128 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
10129 ".omp_data_sizes");
10130 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
10131 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
10132 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
10133 TREE_VEC_ELT (t
, 2)
10134 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
10136 ".omp_data_kinds");
10137 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
10138 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
10139 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
10140 gimple_omp_target_set_data_arg (stmt
, t
);
10142 vec
<constructor_elt
, va_gc
> *vsize
;
10143 vec
<constructor_elt
, va_gc
> *vkind
;
10144 vec_alloc (vsize
, map_cnt
);
10145 vec_alloc (vkind
, map_cnt
);
10146 unsigned int map_idx
= 0;
10148 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10149 switch (OMP_CLAUSE_CODE (c
))
10155 case OMP_CLAUSE_MAP
:
10156 case OMP_CLAUSE_TO
:
10157 case OMP_CLAUSE_FROM
:
10159 ovar
= OMP_CLAUSE_DECL (c
);
10160 if (!DECL_P (ovar
))
10162 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10163 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
10165 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
10166 == get_base_address (ovar
));
10167 nc
= OMP_CLAUSE_CHAIN (c
);
10168 ovar
= OMP_CLAUSE_DECL (nc
);
10172 tree x
= build_sender_ref (ovar
, ctx
);
10174 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
10175 gimplify_assign (x
, v
, &ilist
);
10181 if (DECL_SIZE (ovar
)
10182 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
10184 tree ovar2
= DECL_VALUE_EXPR (ovar
);
10185 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
10186 ovar2
= TREE_OPERAND (ovar2
, 0);
10187 gcc_assert (DECL_P (ovar2
));
10190 if (!maybe_lookup_field (ovar
, ctx
))
10194 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
10195 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
10196 talign
= DECL_ALIGN_UNIT (ovar
);
10199 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
10200 tree x
= build_sender_ref (ovar
, ctx
);
10201 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10202 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
10203 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
10204 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
10206 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
10208 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
10209 mark_addressable (avar
);
10210 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
10211 talign
= DECL_ALIGN_UNIT (avar
);
10212 avar
= build_fold_addr_expr (avar
);
10213 gimplify_assign (x
, avar
, &ilist
);
10215 else if (is_gimple_reg (var
))
10217 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
10218 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
10219 mark_addressable (avar
);
10220 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
10221 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
10222 gimplify_assign (avar
, var
, &ilist
);
10223 avar
= build_fold_addr_expr (avar
);
10224 gimplify_assign (x
, avar
, &ilist
);
10225 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
10226 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
10227 && !TYPE_READONLY (TREE_TYPE (var
)))
10229 x
= build_sender_ref (ovar
, ctx
);
10230 x
= build_simple_mem_ref (x
);
10231 gimplify_assign (var
, x
, &olist
);
10236 var
= build_fold_addr_expr (var
);
10237 gimplify_assign (x
, var
, &ilist
);
10240 tree s
= OMP_CLAUSE_SIZE (c
);
10241 if (s
== NULL_TREE
)
10242 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
10243 s
= fold_convert (size_type_node
, s
);
10244 tree purpose
= size_int (map_idx
++);
10245 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
10246 if (TREE_CODE (s
) != INTEGER_CST
)
10247 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
10249 unsigned char tkind
= 0;
10250 switch (OMP_CLAUSE_CODE (c
))
10252 case OMP_CLAUSE_MAP
:
10253 tkind
= OMP_CLAUSE_MAP_KIND (c
);
10255 case OMP_CLAUSE_TO
:
10256 tkind
= OMP_CLAUSE_MAP_TO
;
10258 case OMP_CLAUSE_FROM
:
10259 tkind
= OMP_CLAUSE_MAP_FROM
;
10262 gcc_unreachable ();
10264 talign
= ceil_log2 (talign
);
10265 tkind
|= talign
<< 3;
10266 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
10267 build_int_cst (unsigned_char_type_node
,
10273 gcc_assert (map_idx
== map_cnt
);
10275 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
10276 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
10277 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
10278 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
10279 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
10281 gimple_seq initlist
= NULL
;
10282 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
10283 TREE_VEC_ELT (t
, 1)),
10284 &initlist
, true, NULL_TREE
);
10285 gimple_seq_add_seq (&ilist
, initlist
);
10287 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
10289 TREE_THIS_VOLATILE (clobber
) = 1;
10290 gimple_seq_add_stmt (&olist
,
10291 gimple_build_assign (TREE_VEC_ELT (t
, 1),
10295 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10296 TREE_THIS_VOLATILE (clobber
) = 1;
10297 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
10301 /* Once all the expansions are done, sequence all the different
10302 fragments inside gimple_omp_body. */
10306 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
10308 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10309 /* fixup_child_record_type might have changed receiver_decl's type. */
10310 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
10311 gimple_seq_add_stmt (&new_body
,
10312 gimple_build_assign (ctx
->receiver_decl
, t
));
10315 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10317 gimple_seq_add_seq (&new_body
, tgt_body
);
10318 new_body
= maybe_catch_exception (new_body
);
10320 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10321 new_body
= tgt_body
;
10322 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
10324 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
10325 gimple_omp_set_body (stmt
, new_body
);
10328 bind
= gimple_build_bind (NULL
, NULL
,
10329 tgt_bind
? gimple_bind_block (tgt_bind
)
10331 gsi_replace (gsi_p
, bind
, true);
10332 gimple_bind_add_seq (bind
, ilist
);
10333 gimple_bind_add_stmt (bind
, stmt
);
10334 gimple_bind_add_seq (bind
, olist
);
10336 pop_gimplify_context (NULL
);
10339 /* Expand code for an OpenMP teams directive. */
10342 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10344 gimple teams_stmt
= gsi_stmt (*gsi_p
);
10345 push_gimplify_context ();
10347 tree block
= make_node (BLOCK
);
10348 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
10349 gsi_replace (gsi_p
, bind
, true);
10350 gimple_seq bind_body
= NULL
;
10351 gimple_seq dlist
= NULL
;
10352 gimple_seq olist
= NULL
;
10354 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10355 OMP_CLAUSE_NUM_TEAMS
);
10356 if (num_teams
== NULL_TREE
)
10357 num_teams
= build_int_cst (unsigned_type_node
, 0);
10360 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
10361 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
10362 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
10364 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10365 OMP_CLAUSE_THREAD_LIMIT
);
10366 if (thread_limit
== NULL_TREE
)
10367 thread_limit
= build_int_cst (unsigned_type_node
, 0);
10370 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
10371 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
10372 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
10376 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
10377 &bind_body
, &dlist
, ctx
, NULL
);
10378 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
10379 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
10380 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
10382 location_t loc
= gimple_location (teams_stmt
);
10383 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
10384 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
10385 gimple_set_location (call
, loc
);
10386 gimple_seq_add_stmt (&bind_body
, call
);
10388 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
10389 gimple_omp_set_body (teams_stmt
, NULL
);
10390 gimple_seq_add_seq (&bind_body
, olist
);
10391 gimple_seq_add_seq (&bind_body
, dlist
);
10392 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
10393 gimple_bind_set_body (bind
, bind_body
);
10395 pop_gimplify_context (bind
);
10397 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10398 BLOCK_VARS (block
) = ctx
->block_vars
;
10399 if (BLOCK_VARS (block
))
10400 TREE_USED (block
) = 1;
10404 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
10405 regimplified. If DATA is non-NULL, lower_omp_1 is outside
10406 of OpenMP context, but with task_shared_vars set. */
10409 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
10414 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
10415 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
10418 if (task_shared_vars
10420 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
10423 /* If a global variable has been privatized, TREE_CONSTANT on
10424 ADDR_EXPR might be wrong. */
10425 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
10426 recompute_tree_invariant_for_addr_expr (t
);
10428 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
10433 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10435 gimple stmt
= gsi_stmt (*gsi_p
);
10436 struct walk_stmt_info wi
;
10438 if (gimple_has_location (stmt
))
10439 input_location
= gimple_location (stmt
);
10441 if (task_shared_vars
)
10442 memset (&wi
, '\0', sizeof (wi
));
10444 /* If we have issued syntax errors, avoid doing any heavy lifting.
10445 Just replace the OpenMP directives with a NOP to avoid
10446 confusing RTL expansion. */
10447 if (seen_error () && is_gimple_omp (stmt
))
10449 gsi_replace (gsi_p
, gimple_build_nop (), true);
10453 switch (gimple_code (stmt
))
10456 if ((ctx
|| task_shared_vars
)
10457 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
10458 ctx
? NULL
: &wi
, NULL
)
10459 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
10460 ctx
? NULL
: &wi
, NULL
)))
10461 gimple_regimplify_operands (stmt
, gsi_p
);
10464 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
10466 case GIMPLE_EH_FILTER
:
10467 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
10470 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
10471 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
10473 case GIMPLE_TRANSACTION
:
10474 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
10477 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
10479 case GIMPLE_OMP_PARALLEL
:
10480 case GIMPLE_OMP_TASK
:
10481 ctx
= maybe_lookup_ctx (stmt
);
10483 if (ctx
->cancellable
)
10484 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10485 lower_omp_taskreg (gsi_p
, ctx
);
10487 case GIMPLE_OMP_FOR
:
10488 ctx
= maybe_lookup_ctx (stmt
);
10490 if (ctx
->cancellable
)
10491 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10492 lower_omp_for (gsi_p
, ctx
);
10494 case GIMPLE_OMP_SECTIONS
:
10495 ctx
= maybe_lookup_ctx (stmt
);
10497 if (ctx
->cancellable
)
10498 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10499 lower_omp_sections (gsi_p
, ctx
);
10501 case GIMPLE_OMP_SINGLE
:
10502 ctx
= maybe_lookup_ctx (stmt
);
10504 lower_omp_single (gsi_p
, ctx
);
10506 case GIMPLE_OMP_MASTER
:
10507 ctx
= maybe_lookup_ctx (stmt
);
10509 lower_omp_master (gsi_p
, ctx
);
10511 case GIMPLE_OMP_TASKGROUP
:
10512 ctx
= maybe_lookup_ctx (stmt
);
10514 lower_omp_taskgroup (gsi_p
, ctx
);
10516 case GIMPLE_OMP_ORDERED
:
10517 ctx
= maybe_lookup_ctx (stmt
);
10519 lower_omp_ordered (gsi_p
, ctx
);
10521 case GIMPLE_OMP_CRITICAL
:
10522 ctx
= maybe_lookup_ctx (stmt
);
10524 lower_omp_critical (gsi_p
, ctx
);
10526 case GIMPLE_OMP_ATOMIC_LOAD
:
10527 if ((ctx
|| task_shared_vars
)
10528 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
10529 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10530 gimple_regimplify_operands (stmt
, gsi_p
);
10532 case GIMPLE_OMP_TARGET
:
10533 ctx
= maybe_lookup_ctx (stmt
);
10535 lower_omp_target (gsi_p
, ctx
);
10537 case GIMPLE_OMP_TEAMS
:
10538 ctx
= maybe_lookup_ctx (stmt
);
10540 lower_omp_teams (gsi_p
, ctx
);
10544 fndecl
= gimple_call_fndecl (stmt
);
10546 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10547 switch (DECL_FUNCTION_CODE (fndecl
))
10549 case BUILT_IN_GOMP_BARRIER
:
10553 case BUILT_IN_GOMP_CANCEL
:
10554 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10557 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10558 cctx
= cctx
->outer
;
10559 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10560 if (!cctx
->cancellable
)
10562 if (DECL_FUNCTION_CODE (fndecl
)
10563 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10565 stmt
= gimple_build_nop ();
10566 gsi_replace (gsi_p
, stmt
, false);
10570 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10572 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10573 gimple_call_set_fndecl (stmt
, fndecl
);
10574 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10577 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)), NULL
);
10578 gimple_call_set_lhs (stmt
, lhs
);
10579 tree fallthru_label
;
10580 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10582 g
= gimple_build_label (fallthru_label
);
10583 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10584 g
= gimple_build_cond (NE_EXPR
, lhs
,
10585 fold_convert (TREE_TYPE (lhs
),
10586 boolean_false_node
),
10587 cctx
->cancel_label
, fallthru_label
);
10588 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10595 if ((ctx
|| task_shared_vars
)
10596 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10599 /* Just remove clobbers, this should happen only if we have
10600 "privatized" local addressable variables in SIMD regions,
10601 the clobber isn't needed in that case and gimplifying address
10602 of the ARRAY_REF into a pointer and creating MEM_REF based
10603 clobber would create worse code than we get with the clobber
10605 if (gimple_clobber_p (stmt
))
10607 gsi_replace (gsi_p
, gimple_build_nop (), true);
10610 gimple_regimplify_operands (stmt
, gsi_p
);
10617 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10619 location_t saved_location
= input_location
;
10620 gimple_stmt_iterator gsi
;
10621 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10622 lower_omp_1 (&gsi
, ctx
);
10623 /* During gimplification, we have not always invoked fold_stmt
10624 (gimplify.c:maybe_fold_stmt); call it now. */
10625 if (target_nesting_level
)
10626 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10628 input_location
= saved_location
;
10631 /* Main entry point. */
10633 static unsigned int
10634 execute_lower_omp (void)
10640 /* This pass always runs, to provide PROP_gimple_lomp.
10641 But there is nothing to do unless -fopenmp is given. */
10642 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_cilkplus
== 0)
10645 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10646 delete_omp_context
);
10648 body
= gimple_body (current_function_decl
);
10649 scan_omp (&body
, NULL
);
10650 gcc_assert (taskreg_nesting_level
== 0);
10651 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
10652 finish_taskreg_scan (ctx
);
10653 taskreg_contexts
.release ();
10655 if (all_contexts
->root
)
10657 if (task_shared_vars
)
10658 push_gimplify_context ();
10659 lower_omp (&body
, NULL
);
10660 if (task_shared_vars
)
10661 pop_gimplify_context (NULL
);
10666 splay_tree_delete (all_contexts
);
10667 all_contexts
= NULL
;
10669 BITMAP_FREE (task_shared_vars
);
10675 const pass_data pass_data_lower_omp
=
10677 GIMPLE_PASS
, /* type */
10678 "omplower", /* name */
10679 OPTGROUP_NONE
, /* optinfo_flags */
10680 TV_NONE
, /* tv_id */
10681 PROP_gimple_any
, /* properties_required */
10682 PROP_gimple_lomp
, /* properties_provided */
10683 0, /* properties_destroyed */
10684 0, /* todo_flags_start */
10685 0, /* todo_flags_finish */
10688 class pass_lower_omp
: public gimple_opt_pass
10691 pass_lower_omp (gcc::context
*ctxt
)
10692 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10695 /* opt_pass methods: */
10696 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
10698 }; // class pass_lower_omp
10700 } // anon namespace
10703 make_pass_lower_omp (gcc::context
*ctxt
)
10705 return new pass_lower_omp (ctxt
);
10708 /* The following is a utility to diagnose OpenMP structured block violations.
10709 It is not part of the "omplower" pass, as that's invoked too late. It
10710 should be invoked by the respective front ends after gimplification. */
10712 static splay_tree all_labels
;
10714 /* Check for mismatched contexts and generate an error if needed. Return
10715 true if an error is detected. */
10718 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10719 gimple branch_ctx
, gimple label_ctx
)
10721 if (label_ctx
== branch_ctx
)
10726 Previously we kept track of the label's entire context in diagnose_sb_[12]
10727 so we could traverse it and issue a correct "exit" or "enter" error
10728 message upon a structured block violation.
10730 We built the context by building a list with tree_cons'ing, but there is
10731 no easy counterpart in gimple tuples. It seems like far too much work
10732 for issuing exit/enter error messages. If someone really misses the
10733 distinct error message... patches welcome.
10737 /* Try to avoid confusing the user by producing and error message
10738 with correct "exit" or "enter" verbiage. We prefer "exit"
10739 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10740 if (branch_ctx
== NULL
)
10746 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10751 label_ctx
= TREE_CHAIN (label_ctx
);
10756 error ("invalid exit from OpenMP structured block");
10758 error ("invalid entry to OpenMP structured block");
10761 bool cilkplus_block
= false;
10765 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10766 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10768 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10769 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10770 cilkplus_block
= true;
10773 /* If it's obvious we have an invalid entry, be specific about the error. */
10774 if (branch_ctx
== NULL
)
10776 if (cilkplus_block
)
10777 error ("invalid entry to Cilk Plus structured block");
10779 error ("invalid entry to OpenMP structured block");
10783 /* Otherwise, be vague and lazy, but efficient. */
10784 if (cilkplus_block
)
10785 error ("invalid branch to/from a Cilk Plus structured block");
10787 error ("invalid branch to/from an OpenMP structured block");
10790 gsi_replace (gsi_p
, gimple_build_nop (), false);
10794 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10795 where each label is found. */
10798 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10799 struct walk_stmt_info
*wi
)
10801 gimple context
= (gimple
) wi
->info
;
10802 gimple inner_context
;
10803 gimple stmt
= gsi_stmt (*gsi_p
);
10805 *handled_ops_p
= true;
10807 switch (gimple_code (stmt
))
10811 case GIMPLE_OMP_PARALLEL
:
10812 case GIMPLE_OMP_TASK
:
10813 case GIMPLE_OMP_SECTIONS
:
10814 case GIMPLE_OMP_SINGLE
:
10815 case GIMPLE_OMP_SECTION
:
10816 case GIMPLE_OMP_MASTER
:
10817 case GIMPLE_OMP_ORDERED
:
10818 case GIMPLE_OMP_CRITICAL
:
10819 case GIMPLE_OMP_TARGET
:
10820 case GIMPLE_OMP_TEAMS
:
10821 case GIMPLE_OMP_TASKGROUP
:
10822 /* The minimal context here is just the current OMP construct. */
10823 inner_context
= stmt
;
10824 wi
->info
= inner_context
;
10825 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10826 wi
->info
= context
;
10829 case GIMPLE_OMP_FOR
:
10830 inner_context
= stmt
;
10831 wi
->info
= inner_context
;
10832 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10834 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10835 diagnose_sb_1
, NULL
, wi
);
10836 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10837 wi
->info
= context
;
10841 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10842 (splay_tree_value
) context
);
10852 /* Pass 2: Check each branch and see if its context differs from that of
10853 the destination label's context. */
10856 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10857 struct walk_stmt_info
*wi
)
10859 gimple context
= (gimple
) wi
->info
;
10861 gimple stmt
= gsi_stmt (*gsi_p
);
10863 *handled_ops_p
= true;
10865 switch (gimple_code (stmt
))
10869 case GIMPLE_OMP_PARALLEL
:
10870 case GIMPLE_OMP_TASK
:
10871 case GIMPLE_OMP_SECTIONS
:
10872 case GIMPLE_OMP_SINGLE
:
10873 case GIMPLE_OMP_SECTION
:
10874 case GIMPLE_OMP_MASTER
:
10875 case GIMPLE_OMP_ORDERED
:
10876 case GIMPLE_OMP_CRITICAL
:
10877 case GIMPLE_OMP_TARGET
:
10878 case GIMPLE_OMP_TEAMS
:
10879 case GIMPLE_OMP_TASKGROUP
:
10881 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10882 wi
->info
= context
;
10885 case GIMPLE_OMP_FOR
:
10887 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10889 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10890 diagnose_sb_2
, NULL
, wi
);
10891 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10892 wi
->info
= context
;
10897 tree lab
= gimple_cond_true_label (stmt
);
10900 n
= splay_tree_lookup (all_labels
,
10901 (splay_tree_key
) lab
);
10902 diagnose_sb_0 (gsi_p
, context
,
10903 n
? (gimple
) n
->value
: NULL
);
10905 lab
= gimple_cond_false_label (stmt
);
10908 n
= splay_tree_lookup (all_labels
,
10909 (splay_tree_key
) lab
);
10910 diagnose_sb_0 (gsi_p
, context
,
10911 n
? (gimple
) n
->value
: NULL
);
10918 tree lab
= gimple_goto_dest (stmt
);
10919 if (TREE_CODE (lab
) != LABEL_DECL
)
10922 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10923 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10927 case GIMPLE_SWITCH
:
10930 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10932 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10933 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10934 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10940 case GIMPLE_RETURN
:
10941 diagnose_sb_0 (gsi_p
, context
, NULL
);
10951 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10954 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
10957 gimple last
= last_stmt (bb
);
10958 enum gimple_code code
= gimple_code (last
);
10959 struct omp_region
*cur_region
= *region
;
10960 bool fallthru
= false;
10964 case GIMPLE_OMP_PARALLEL
:
10965 case GIMPLE_OMP_TASK
:
10966 case GIMPLE_OMP_FOR
:
10967 case GIMPLE_OMP_SINGLE
:
10968 case GIMPLE_OMP_TEAMS
:
10969 case GIMPLE_OMP_MASTER
:
10970 case GIMPLE_OMP_TASKGROUP
:
10971 case GIMPLE_OMP_ORDERED
:
10972 case GIMPLE_OMP_CRITICAL
:
10973 case GIMPLE_OMP_SECTION
:
10974 cur_region
= new_omp_region (bb
, code
, cur_region
);
10978 case GIMPLE_OMP_TARGET
:
10979 cur_region
= new_omp_region (bb
, code
, cur_region
);
10981 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10982 cur_region
= cur_region
->outer
;
10985 case GIMPLE_OMP_SECTIONS
:
10986 cur_region
= new_omp_region (bb
, code
, cur_region
);
10990 case GIMPLE_OMP_SECTIONS_SWITCH
:
10994 case GIMPLE_OMP_ATOMIC_LOAD
:
10995 case GIMPLE_OMP_ATOMIC_STORE
:
10999 case GIMPLE_OMP_RETURN
:
11000 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
11001 somewhere other than the next block. This will be
11003 cur_region
->exit
= bb
;
11004 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
11005 cur_region
= cur_region
->outer
;
11008 case GIMPLE_OMP_CONTINUE
:
11009 cur_region
->cont
= bb
;
11010 switch (cur_region
->type
)
11012 case GIMPLE_OMP_FOR
:
11013 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
11014 succs edges as abnormal to prevent splitting
11016 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
11017 /* Make the loopback edge. */
11018 make_edge (bb
, single_succ (cur_region
->entry
),
11021 /* Create an edge from GIMPLE_OMP_FOR to exit, which
11022 corresponds to the case that the body of the loop
11023 is not executed at all. */
11024 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
11025 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
11029 case GIMPLE_OMP_SECTIONS
:
11030 /* Wire up the edges into and out of the nested sections. */
11032 basic_block switch_bb
= single_succ (cur_region
->entry
);
11034 struct omp_region
*i
;
11035 for (i
= cur_region
->inner
; i
; i
= i
->next
)
11037 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
11038 make_edge (switch_bb
, i
->entry
, 0);
11039 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
11042 /* Make the loopback edge to the block with
11043 GIMPLE_OMP_SECTIONS_SWITCH. */
11044 make_edge (bb
, switch_bb
, 0);
11046 /* Make the edge from the switch to exit. */
11047 make_edge (switch_bb
, bb
->next_bb
, 0);
11053 gcc_unreachable ();
11058 gcc_unreachable ();
11061 if (*region
!= cur_region
)
11063 *region
= cur_region
;
11065 *region_idx
= cur_region
->entry
->index
;
11073 static unsigned int
11074 diagnose_omp_structured_block_errors (void)
11076 struct walk_stmt_info wi
;
11077 gimple_seq body
= gimple_body (current_function_decl
);
11079 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
11081 memset (&wi
, 0, sizeof (wi
));
11082 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
11084 memset (&wi
, 0, sizeof (wi
));
11085 wi
.want_locations
= true;
11086 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
11088 gimple_set_body (current_function_decl
, body
);
11090 splay_tree_delete (all_labels
);
11098 const pass_data pass_data_diagnose_omp_blocks
=
11100 GIMPLE_PASS
, /* type */
11101 "*diagnose_omp_blocks", /* name */
11102 OPTGROUP_NONE
, /* optinfo_flags */
11103 TV_NONE
, /* tv_id */
11104 PROP_gimple_any
, /* properties_required */
11105 0, /* properties_provided */
11106 0, /* properties_destroyed */
11107 0, /* todo_flags_start */
11108 0, /* todo_flags_finish */
11111 class pass_diagnose_omp_blocks
: public gimple_opt_pass
11114 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
11115 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
11118 /* opt_pass methods: */
11119 virtual bool gate (function
*) { return flag_openmp
|| flag_cilkplus
; }
11120 virtual unsigned int execute (function
*)
11122 return diagnose_omp_structured_block_errors ();
11125 }; // class pass_diagnose_omp_blocks
11127 } // anon namespace
11130 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
11132 return new pass_diagnose_omp_blocks (ctxt
);
11135 /* SIMD clone supporting code. */
11137 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
11138 of arguments to reserve space for. */
11140 static struct cgraph_simd_clone
*
11141 simd_clone_struct_alloc (int nargs
)
11143 struct cgraph_simd_clone
*clone_info
;
11144 size_t len
= (sizeof (struct cgraph_simd_clone
)
11145 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
11146 clone_info
= (struct cgraph_simd_clone
*)
11147 ggc_internal_cleared_alloc (len
);
11151 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
11154 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
11155 struct cgraph_simd_clone
*from
)
11157 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
11158 + ((from
->nargs
- from
->inbranch
)
11159 * sizeof (struct cgraph_simd_clone_arg
))));
11162 /* Return vector of parameter types of function FNDECL. This uses
11163 TYPE_ARG_TYPES if available, otherwise falls back to types of
11164 DECL_ARGUMENTS types. */
11167 simd_clone_vector_of_formal_parm_types (tree fndecl
)
11169 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
11170 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
11171 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
11174 FOR_EACH_VEC_ELT (args
, i
, arg
)
11175 args
[i
] = TREE_TYPE (args
[i
]);
11179 /* Given a simd function in NODE, extract the simd specific
11180 information from the OMP clauses passed in CLAUSES, and return
11181 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
11182 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
11183 otherwise set to FALSE. */
11185 static struct cgraph_simd_clone
*
11186 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
11187 bool *inbranch_specified
)
11189 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11192 *inbranch_specified
= false;
11194 n
= args
.length ();
11195 if (n
> 0 && args
.last () == void_type_node
)
11198 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
11199 be cloned have a distinctive artificial label in addition to "omp
11203 && lookup_attribute ("cilk simd function",
11204 DECL_ATTRIBUTES (node
->decl
)));
11206 /* Allocate one more than needed just in case this is an in-branch
11207 clone which will require a mask argument. */
11208 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
11209 clone_info
->nargs
= n
;
11210 clone_info
->cilk_elemental
= cilk_clone
;
11217 clauses
= TREE_VALUE (clauses
);
11218 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
11221 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
11223 switch (OMP_CLAUSE_CODE (t
))
11225 case OMP_CLAUSE_INBRANCH
:
11226 clone_info
->inbranch
= 1;
11227 *inbranch_specified
= true;
11229 case OMP_CLAUSE_NOTINBRANCH
:
11230 clone_info
->inbranch
= 0;
11231 *inbranch_specified
= true;
11233 case OMP_CLAUSE_SIMDLEN
:
11234 clone_info
->simdlen
11235 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
11237 case OMP_CLAUSE_LINEAR
:
11239 tree decl
= OMP_CLAUSE_DECL (t
);
11240 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
11241 int argno
= TREE_INT_CST_LOW (decl
);
11242 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
11244 clone_info
->args
[argno
].arg_type
11245 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
11246 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
11247 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
11248 && clone_info
->args
[argno
].linear_step
< n
);
11252 if (POINTER_TYPE_P (args
[argno
]))
11253 step
= fold_convert (ssizetype
, step
);
11254 if (!tree_fits_shwi_p (step
))
11256 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
11257 "ignoring large linear step");
11261 else if (integer_zerop (step
))
11263 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
11264 "ignoring zero linear step");
11270 clone_info
->args
[argno
].arg_type
11271 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
11272 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
11277 case OMP_CLAUSE_UNIFORM
:
11279 tree decl
= OMP_CLAUSE_DECL (t
);
11280 int argno
= tree_to_uhwi (decl
);
11281 clone_info
->args
[argno
].arg_type
11282 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
11285 case OMP_CLAUSE_ALIGNED
:
11287 tree decl
= OMP_CLAUSE_DECL (t
);
11288 int argno
= tree_to_uhwi (decl
);
11289 clone_info
->args
[argno
].alignment
11290 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
11301 /* Given a SIMD clone in NODE, calculate the characteristic data
11302 type and return the coresponding type. The characteristic data
11303 type is computed as described in the Intel Vector ABI. */
11306 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
11307 struct cgraph_simd_clone
*clone_info
)
11309 tree type
= integer_type_node
;
11310 tree fndecl
= node
->decl
;
11312 /* a) For non-void function, the characteristic data type is the
11314 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
11315 type
= TREE_TYPE (TREE_TYPE (fndecl
));
11317 /* b) If the function has any non-uniform, non-linear parameters,
11318 then the characteristic data type is the type of the first
11322 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
11323 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
11324 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
11332 /* c) If the characteristic data type determined by a) or b) above
11333 is struct, union, or class type which is pass-by-value (except
11334 for the type that maps to the built-in complex data type), the
11335 characteristic data type is int. */
11336 if (RECORD_OR_UNION_TYPE_P (type
)
11337 && !aggregate_value_p (type
, NULL
)
11338 && TREE_CODE (type
) != COMPLEX_TYPE
)
11339 return integer_type_node
;
11341 /* d) If none of the above three classes is applicable, the
11342 characteristic data type is int. */
11346 /* e) For Intel Xeon Phi native and offload compilation, if the
11347 resulting characteristic data type is 8-bit or 16-bit integer
11348 data type, the characteristic data type is int. */
11349 /* Well, we don't handle Xeon Phi yet. */
11353 simd_clone_mangle (struct cgraph_node
*node
,
11354 struct cgraph_simd_clone
*clone_info
)
11356 char vecsize_mangle
= clone_info
->vecsize_mangle
;
11357 char mask
= clone_info
->inbranch
? 'M' : 'N';
11358 unsigned int simdlen
= clone_info
->simdlen
;
11362 gcc_assert (vecsize_mangle
&& simdlen
);
11364 pp_string (&pp
, "_ZGV");
11365 pp_character (&pp
, vecsize_mangle
);
11366 pp_character (&pp
, mask
);
11367 pp_decimal_int (&pp
, simdlen
);
11369 for (n
= 0; n
< clone_info
->nargs
; ++n
)
11371 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
11373 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
11374 pp_character (&pp
, 'u');
11375 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11377 gcc_assert (arg
.linear_step
!= 0);
11378 pp_character (&pp
, 'l');
11379 if (arg
.linear_step
> 1)
11380 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11381 else if (arg
.linear_step
< 0)
11383 pp_character (&pp
, 'n');
11384 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
11388 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
11390 pp_character (&pp
, 's');
11391 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11394 pp_character (&pp
, 'v');
11397 pp_character (&pp
, 'a');
11398 pp_decimal_int (&pp
, arg
.alignment
);
11402 pp_underscore (&pp
);
11404 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
11405 const char *str
= pp_formatted_text (&pp
);
11407 /* If there already is a SIMD clone with the same mangled name, don't
11408 add another one. This can happen e.g. for
11409 #pragma omp declare simd
11410 #pragma omp declare simd simdlen(8)
11411 int foo (int, int);
11412 if the simdlen is assumed to be 8 for the first one, etc. */
11413 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
11414 clone
= clone
->simdclone
->next_clone
)
11415 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
11419 return get_identifier (str
);
11422 /* Create a simd clone of OLD_NODE and return it. */
11424 static struct cgraph_node
*
11425 simd_clone_create (struct cgraph_node
*old_node
)
11427 struct cgraph_node
*new_node
;
11428 if (old_node
->definition
)
11430 if (!old_node
->has_gimple_body_p ())
11432 old_node
->get_body ();
11433 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
11439 tree old_decl
= old_node
->decl
;
11440 tree new_decl
= copy_node (old_node
->decl
);
11441 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
11442 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
11443 SET_DECL_RTL (new_decl
, NULL
);
11444 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
11445 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
11446 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
11447 symtab
->call_cgraph_insertion_hooks (new_node
);
11449 if (new_node
== NULL
)
11452 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
11454 /* The function cgraph_function_versioning () will force the new
11455 symbol local. Undo this, and inherit external visability from
11457 new_node
->local
.local
= old_node
->local
.local
;
11458 new_node
->externally_visible
= old_node
->externally_visible
;
11463 /* Adjust the return type of the given function to its appropriate
11464 vector counterpart. Returns a simd array to be used throughout the
11465 function as a return value. */
11468 simd_clone_adjust_return_type (struct cgraph_node
*node
)
11470 tree fndecl
= node
->decl
;
11471 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
11472 unsigned int veclen
;
11475 /* Adjust the function return type. */
11476 if (orig_rettype
== void_type_node
)
11478 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
11479 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
)))
11480 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
))))
11481 veclen
= node
->simdclone
->vecsize_int
;
11483 veclen
= node
->simdclone
->vecsize_float
;
11484 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl
))));
11485 if (veclen
> node
->simdclone
->simdlen
)
11486 veclen
= node
->simdclone
->simdlen
;
11487 if (veclen
== node
->simdclone
->simdlen
)
11488 TREE_TYPE (TREE_TYPE (fndecl
))
11489 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)),
11490 node
->simdclone
->simdlen
);
11493 t
= build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)), veclen
);
11494 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
11495 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
11497 if (!node
->definition
)
11500 t
= DECL_RESULT (fndecl
);
11501 /* Adjust the DECL_RESULT. */
11502 gcc_assert (TREE_TYPE (t
) != void_type_node
);
11503 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
11506 tree atype
= build_array_type_nelts (orig_rettype
,
11507 node
->simdclone
->simdlen
);
11508 if (veclen
!= node
->simdclone
->simdlen
)
11509 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
11511 /* Set up a SIMD array to use as the return value. */
11512 tree retval
= create_tmp_var_raw (atype
, "retval");
11513 gimple_add_tmp_var (retval
);
11517 /* Each vector argument has a corresponding array to be used locally
11518 as part of the eventual loop. Create such temporary array and
11521 PREFIX is the prefix to be used for the temporary.
11523 TYPE is the inner element type.
11525 SIMDLEN is the number of elements. */
11528 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
11530 tree atype
= build_array_type_nelts (type
, simdlen
);
11531 tree avar
= create_tmp_var_raw (atype
, prefix
);
11532 gimple_add_tmp_var (avar
);
11536 /* Modify the function argument types to their corresponding vector
11537 counterparts if appropriate. Also, create one array for each simd
11538 argument to be used locally when using the function arguments as
11541 NODE is the function whose arguments are to be adjusted.
11543 Returns an adjustment vector that will be filled describing how the
11544 argument types will be adjusted. */
11546 static ipa_parm_adjustment_vec
11547 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
11550 ipa_parm_adjustment_vec adjustments
;
11552 if (node
->definition
)
11553 args
= ipa_get_vector_of_formal_parms (node
->decl
);
11555 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11556 adjustments
.create (args
.length ());
11557 unsigned i
, j
, veclen
;
11558 struct ipa_parm_adjustment adj
;
11559 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
11561 memset (&adj
, 0, sizeof (adj
));
11562 tree parm
= args
[i
];
11563 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
11564 adj
.base_index
= i
;
11567 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
11568 node
->simdclone
->args
[i
].orig_type
= parm_type
;
11570 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
11572 /* No adjustment necessary for scalar arguments. */
11573 adj
.op
= IPA_PARM_OP_COPY
;
11577 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
11578 veclen
= node
->simdclone
->vecsize_int
;
11580 veclen
= node
->simdclone
->vecsize_float
;
11581 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
11582 if (veclen
> node
->simdclone
->simdlen
)
11583 veclen
= node
->simdclone
->simdlen
;
11584 adj
.arg_prefix
= "simd";
11585 adj
.type
= build_vector_type (parm_type
, veclen
);
11586 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
11587 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11589 adjustments
.safe_push (adj
);
11592 memset (&adj
, 0, sizeof (adj
));
11593 adj
.op
= IPA_PARM_OP_NEW
;
11594 adj
.arg_prefix
= "simd";
11595 adj
.base_index
= i
;
11596 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
11600 if (node
->definition
)
11601 node
->simdclone
->args
[i
].simd_array
11602 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
11603 parm_type
, node
->simdclone
->simdlen
);
11605 adjustments
.safe_push (adj
);
11608 if (node
->simdclone
->inbranch
)
11611 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
11614 memset (&adj
, 0, sizeof (adj
));
11615 adj
.op
= IPA_PARM_OP_NEW
;
11616 adj
.arg_prefix
= "mask";
11618 adj
.base_index
= i
;
11619 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
11620 veclen
= node
->simdclone
->vecsize_int
;
11622 veclen
= node
->simdclone
->vecsize_float
;
11623 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
11624 if (veclen
> node
->simdclone
->simdlen
)
11625 veclen
= node
->simdclone
->simdlen
;
11626 adj
.type
= build_vector_type (base_type
, veclen
);
11627 adjustments
.safe_push (adj
);
11629 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11630 adjustments
.safe_push (adj
);
11632 /* We have previously allocated one extra entry for the mask. Use
11634 struct cgraph_simd_clone
*sc
= node
->simdclone
;
11636 if (node
->definition
)
11638 sc
->args
[i
].orig_arg
11639 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
11640 sc
->args
[i
].simd_array
11641 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
11643 sc
->args
[i
].orig_type
= base_type
;
11644 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
11647 if (node
->definition
)
11648 ipa_modify_formal_parameters (node
->decl
, adjustments
);
11651 tree new_arg_types
= NULL_TREE
, new_reversed
;
11652 bool last_parm_void
= false;
11653 if (args
.length () > 0 && args
.last () == void_type_node
)
11654 last_parm_void
= true;
11656 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
11657 j
= adjustments
.length ();
11658 for (i
= 0; i
< j
; i
++)
11660 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
11662 if (adj
->op
== IPA_PARM_OP_COPY
)
11663 ptype
= args
[adj
->base_index
];
11666 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
11668 new_reversed
= nreverse (new_arg_types
);
11669 if (last_parm_void
)
11672 TREE_CHAIN (new_arg_types
) = void_list_node
;
11674 new_reversed
= void_list_node
;
11677 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
11678 TYPE_ARG_TYPES (new_type
) = new_reversed
;
11679 TREE_TYPE (node
->decl
) = new_type
;
11681 adjustments
.release ();
11684 return adjustments
;
11687 /* Initialize and copy the function arguments in NODE to their
11688 corresponding local simd arrays. Returns a fresh gimple_seq with
11689 the instruction sequence generated. */
11692 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
11693 ipa_parm_adjustment_vec adjustments
)
11695 gimple_seq seq
= NULL
;
11696 unsigned i
= 0, j
= 0, k
;
11698 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
11700 arg
= DECL_CHAIN (arg
), i
++, j
++)
11702 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
11705 node
->simdclone
->args
[i
].vector_arg
= arg
;
11707 tree array
= node
->simdclone
->args
[i
].simd_array
;
11708 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
11710 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11711 tree ptr
= build_fold_addr_expr (array
);
11712 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11713 build_int_cst (ptype
, 0));
11714 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11715 gimplify_and_add (t
, &seq
);
11719 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
11720 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11721 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
11723 tree ptr
= build_fold_addr_expr (array
);
11727 arg
= DECL_CHAIN (arg
);
11731 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
11732 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11733 build_int_cst (ptype
, k
* elemsize
));
11734 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11735 gimplify_and_add (t
, &seq
);
11742 /* Callback info for ipa_simd_modify_stmt_ops below. */
11744 struct modify_stmt_info
{
11745 ipa_parm_adjustment_vec adjustments
;
11747 /* True if the parent statement was modified by
11748 ipa_simd_modify_stmt_ops. */
11752 /* Callback for walk_gimple_op.
11754 Adjust operands from a given statement as specified in the
11755 adjustments vector in the callback data. */
11758 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
11760 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
11761 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
11762 tree
*orig_tp
= tp
;
11763 if (TREE_CODE (*tp
) == ADDR_EXPR
)
11764 tp
= &TREE_OPERAND (*tp
, 0);
11765 struct ipa_parm_adjustment
*cand
= NULL
;
11766 if (TREE_CODE (*tp
) == PARM_DECL
)
11767 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
11771 *walk_subtrees
= 0;
11774 tree repl
= NULL_TREE
;
11776 repl
= unshare_expr (cand
->new_decl
);
11781 *walk_subtrees
= 0;
11782 bool modified
= info
->modified
;
11783 info
->modified
= false;
11784 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
11785 if (!info
->modified
)
11787 info
->modified
= modified
;
11790 info
->modified
= modified
;
11799 repl
= build_fold_addr_expr (repl
);
11801 if (is_gimple_debug (info
->stmt
))
11803 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
11804 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
11805 DECL_ARTIFICIAL (vexpr
) = 1;
11806 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
11807 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
11812 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
),
11814 repl
= gimple_assign_lhs (stmt
);
11816 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
11817 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11820 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
11822 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
11828 info
->modified
= true;
11832 /* Traverse the function body and perform all modifications as
11833 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11834 modified such that the replacement/reduction value will now be an
11835 offset into the corresponding simd_array.
11837 This function will replace all function argument uses with their
11838 corresponding simd array elements, and ajust the return values
11842 ipa_simd_modify_function_body (struct cgraph_node
*node
,
11843 ipa_parm_adjustment_vec adjustments
,
11844 tree retval_array
, tree iter
)
11847 unsigned int i
, j
, l
;
11849 /* Re-use the adjustments array, but this time use it to replace
11850 every function argument use to an offset into the corresponding
11852 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
11854 if (!node
->simdclone
->args
[i
].vector_arg
)
11857 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
11858 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
11859 adjustments
[j
].new_decl
11860 = build4 (ARRAY_REF
,
11862 node
->simdclone
->args
[i
].simd_array
,
11864 NULL_TREE
, NULL_TREE
);
11865 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
11866 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
11867 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
11870 l
= adjustments
.length ();
11871 for (i
= 1; i
< num_ssa_names
; i
++)
11873 tree name
= ssa_name (i
);
11875 && SSA_NAME_VAR (name
)
11876 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
11878 for (j
= 0; j
< l
; j
++)
11879 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
11880 && adjustments
[j
].new_decl
)
11883 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
11886 = copy_var_decl (adjustments
[j
].base
,
11887 DECL_NAME (adjustments
[j
].base
),
11888 TREE_TYPE (adjustments
[j
].base
));
11889 adjustments
[j
].new_ssa_base
= base_var
;
11892 base_var
= adjustments
[j
].new_ssa_base
;
11893 if (SSA_NAME_IS_DEFAULT_DEF (name
))
11895 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11896 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
11897 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
11898 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
11899 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11900 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
11901 gimple stmt
= gimple_build_assign (name
, new_decl
);
11902 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11905 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11910 struct modify_stmt_info info
;
11911 info
.adjustments
= adjustments
;
11913 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
11915 gimple_stmt_iterator gsi
;
11917 gsi
= gsi_start_bb (bb
);
11918 while (!gsi_end_p (gsi
))
11920 gimple stmt
= gsi_stmt (gsi
);
11922 struct walk_stmt_info wi
;
11924 memset (&wi
, 0, sizeof (wi
));
11925 info
.modified
= false;
11927 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
11929 if (gimple_code (stmt
) == GIMPLE_RETURN
)
11931 tree retval
= gimple_return_retval (stmt
);
11934 gsi_remove (&gsi
, true);
11938 /* Replace `return foo' with `retval_array[iter] = foo'. */
11939 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
11940 retval_array
, iter
, NULL
, NULL
);
11941 stmt
= gimple_build_assign (ref
, retval
);
11942 gsi_replace (&gsi
, stmt
, true);
11943 info
.modified
= true;
11948 update_stmt (stmt
);
11949 if (maybe_clean_eh_stmt (stmt
))
11950 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
11957 /* Adjust the argument types in NODE to their appropriate vector
11961 simd_clone_adjust (struct cgraph_node
*node
)
11963 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
11965 targetm
.simd_clone
.adjust (node
);
11967 tree retval
= simd_clone_adjust_return_type (node
);
11968 ipa_parm_adjustment_vec adjustments
11969 = simd_clone_adjust_argument_types (node
);
11971 push_gimplify_context ();
11973 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
11975 /* Adjust all uses of vector arguments accordingly. Adjust all
11976 return values accordingly. */
11977 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
11978 tree iter1
= make_ssa_name (iter
, NULL
);
11979 tree iter2
= make_ssa_name (iter
, NULL
);
11980 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
11982 /* Initialize the iteration variable. */
11983 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11984 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
11985 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
11986 /* Insert the SIMD array and iv initialization at function
11988 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
11990 pop_gimplify_context (NULL
);
11992 /* Create a new BB right before the original exit BB, to hold the
11993 iteration increment and the condition/branch. */
11994 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
11995 basic_block incr_bb
= create_empty_bb (orig_exit
);
11996 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
11997 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
11998 flag. Set it now to be a FALLTHRU_EDGE. */
11999 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
12000 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
12001 for (unsigned i
= 0;
12002 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
12004 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
12005 redirect_edge_succ (e
, incr_bb
);
12007 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
12008 e
->probability
= REG_BR_PROB_BASE
;
12009 gsi
= gsi_last_bb (incr_bb
);
12010 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
12011 build_int_cst (unsigned_type_node
,
12013 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12015 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
12016 struct loop
*loop
= alloc_loop ();
12017 cfun
->has_force_vectorize_loops
= true;
12018 loop
->safelen
= node
->simdclone
->simdlen
;
12019 loop
->force_vectorize
= true;
12020 loop
->header
= body_bb
;
12022 /* Branch around the body if the mask applies. */
12023 if (node
->simdclone
->inbranch
)
12025 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
12027 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
12028 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
12029 tree aref
= build4 (ARRAY_REF
,
12030 TREE_TYPE (TREE_TYPE (mask_array
)),
12033 g
= gimple_build_assign (mask
, aref
);
12034 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12035 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
12036 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
12038 aref
= build1 (VIEW_CONVERT_EXPR
,
12039 build_nonstandard_integer_type (bitsize
, 0), mask
);
12040 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
12041 g
= gimple_build_assign (mask
, aref
);
12042 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12045 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
12047 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12048 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
12049 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
12052 /* Generate the condition. */
12053 g
= gimple_build_cond (LT_EXPR
,
12055 build_int_cst (unsigned_type_node
,
12056 node
->simdclone
->simdlen
),
12058 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12059 e
= split_block (incr_bb
, gsi_stmt (gsi
));
12060 basic_block latch_bb
= e
->dest
;
12061 basic_block new_exit_bb
;
12062 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
12063 loop
->latch
= latch_bb
;
12065 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
12067 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
12068 /* The successor of incr_bb is already pointing to latch_bb; just
12070 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
12071 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
12073 gimple phi
= create_phi_node (iter1
, body_bb
);
12074 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
12075 edge latch_edge
= single_succ_edge (latch_bb
);
12076 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
12078 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
12080 /* Generate the new return. */
12081 gsi
= gsi_last_bb (new_exit_bb
);
12083 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
12084 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
12085 retval
= TREE_OPERAND (retval
, 0);
12088 retval
= build1 (VIEW_CONVERT_EXPR
,
12089 TREE_TYPE (TREE_TYPE (node
->decl
)),
12091 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
12092 false, GSI_CONTINUE_LINKING
);
12094 g
= gimple_build_return (retval
);
12095 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12097 /* Handle aligned clauses by replacing default defs of the aligned
12098 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
12099 lhs. Handle linear by adding PHIs. */
12100 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
12101 if (node
->simdclone
->args
[i
].alignment
12102 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
12103 && (node
->simdclone
->args
[i
].alignment
12104 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
12105 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
12108 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
12109 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
12110 tree def
= ssa_default_def (cfun
, orig_arg
);
12111 if (def
&& !has_zero_uses (def
))
12113 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
12114 gimple_seq seq
= NULL
;
12115 bool need_cvt
= false;
12117 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
12119 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
12122 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
12123 gimple_call_set_lhs (g
, t
);
12124 gimple_seq_add_stmt_without_update (&seq
, g
);
12127 t
= make_ssa_name (orig_arg
, NULL
);
12128 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
12129 gimple_call_lhs (g
),
12131 gimple_seq_add_stmt_without_update (&seq
, g
);
12133 gsi_insert_seq_on_edge_immediate
12134 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
12136 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
12137 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
12139 node
->create_edge (cgraph_node::get_create (fn
),
12140 call
, entry_bb
->count
, freq
);
12142 imm_use_iterator iter
;
12143 use_operand_p use_p
;
12145 tree repl
= gimple_get_lhs (g
);
12146 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
12147 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
12150 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
12151 SET_USE (use_p
, repl
);
12154 else if (node
->simdclone
->args
[i
].arg_type
12155 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12157 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
12158 tree def
= ssa_default_def (cfun
, orig_arg
);
12159 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12160 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
12161 if (def
&& !has_zero_uses (def
))
12163 iter1
= make_ssa_name (orig_arg
, NULL
);
12164 iter2
= make_ssa_name (orig_arg
, NULL
);
12165 phi
= create_phi_node (iter1
, body_bb
);
12166 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
12167 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
12168 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12169 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
12170 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12171 ? TREE_TYPE (orig_arg
) : sizetype
;
12173 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
12174 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
12175 gsi
= gsi_last_bb (incr_bb
);
12176 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
12178 imm_use_iterator iter
;
12179 use_operand_p use_p
;
12181 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
12182 if (use_stmt
== phi
)
12185 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
12186 SET_USE (use_p
, iter1
);
12190 calculate_dominance_info (CDI_DOMINATORS
);
12191 add_loop (loop
, loop
->header
->loop_father
);
12192 update_ssa (TODO_update_ssa
);
12197 /* If the function in NODE is tagged as an elemental SIMD function,
12198 create the appropriate SIMD clones. */
12201 expand_simd_clones (struct cgraph_node
*node
)
12203 tree attr
= lookup_attribute ("omp declare simd",
12204 DECL_ATTRIBUTES (node
->decl
));
12205 if (attr
== NULL_TREE
12206 || node
->global
.inlined_to
12207 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
12211 #pragma omp declare simd
12213 in C, there we don't know the argument types at all. */
12214 if (!node
->definition
12215 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
12220 /* Start with parsing the "omp declare simd" attribute(s). */
12221 bool inbranch_clause_specified
;
12222 struct cgraph_simd_clone
*clone_info
12223 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
12224 &inbranch_clause_specified
);
12225 if (clone_info
== NULL
)
12228 int orig_simdlen
= clone_info
->simdlen
;
12229 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
12230 /* The target can return 0 (no simd clones should be created),
12231 1 (just one ISA of simd clones should be created) or higher
12232 count of ISA variants. In that case, clone_info is initialized
12233 for the first ISA variant. */
12235 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
12240 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
12241 also create one inbranch and one !inbranch clone of it. */
12242 for (int i
= 0; i
< count
* 2; i
++)
12244 struct cgraph_simd_clone
*clone
= clone_info
;
12245 if (inbranch_clause_specified
&& (i
& 1) != 0)
12250 clone
= simd_clone_struct_alloc (clone_info
->nargs
12252 simd_clone_struct_copy (clone
, clone_info
);
12253 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
12254 and simd_clone_adjust_argument_types did to the first
12256 clone
->nargs
-= clone_info
->inbranch
;
12257 clone
->simdlen
= orig_simdlen
;
12258 /* And call the target hook again to get the right ISA. */
12259 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
12263 clone
->inbranch
= 1;
12266 /* simd_clone_mangle might fail if such a clone has been created
12268 tree id
= simd_clone_mangle (node
, clone
);
12269 if (id
== NULL_TREE
)
12272 /* Only when we are sure we want to create the clone actually
12273 clone the function (or definitions) or create another
12274 extern FUNCTION_DECL (for prototypes without definitions). */
12275 struct cgraph_node
*n
= simd_clone_create (node
);
12279 n
->simdclone
= clone
;
12280 clone
->origin
= node
;
12281 clone
->next_clone
= NULL
;
12282 if (node
->simd_clones
== NULL
)
12284 clone
->prev_clone
= n
;
12285 node
->simd_clones
= n
;
12289 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
12290 clone
->prev_clone
->simdclone
->next_clone
= n
;
12291 node
->simd_clones
->simdclone
->prev_clone
= n
;
12293 symtab
->change_decl_assembler_name (n
->decl
, id
);
12294 /* And finally adjust the return type, parameters and for
12295 definitions also function body. */
12296 if (node
->definition
)
12297 simd_clone_adjust (n
);
12300 simd_clone_adjust_return_type (n
);
12301 simd_clone_adjust_argument_types (n
);
12305 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
12308 /* Entry point for IPA simd clone creation pass. */
12310 static unsigned int
12311 ipa_omp_simd_clone (void)
12313 struct cgraph_node
*node
;
12314 FOR_EACH_FUNCTION (node
)
12315 expand_simd_clones (node
);
12321 const pass_data pass_data_omp_simd_clone
=
12323 SIMPLE_IPA_PASS
, /* type */
12324 "simdclone", /* name */
12325 OPTGROUP_NONE
, /* optinfo_flags */
12326 TV_NONE
, /* tv_id */
12327 ( PROP_ssa
| PROP_cfg
), /* properties_required */
12328 0, /* properties_provided */
12329 0, /* properties_destroyed */
12330 0, /* todo_flags_start */
12331 0, /* todo_flags_finish */
12334 class pass_omp_simd_clone
: public simple_ipa_opt_pass
12337 pass_omp_simd_clone(gcc::context
*ctxt
)
12338 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
12341 /* opt_pass methods: */
12342 virtual bool gate (function
*);
12343 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
12347 pass_omp_simd_clone::gate (function
*)
12349 return ((flag_openmp
|| flag_openmp_simd
12351 || (in_lto_p
&& !flag_wpa
))
12352 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
12355 } // anon namespace
12357 simple_ipa_opt_pass
*
12358 make_pass_omp_simd_clone (gcc::context
*ctxt
)
12360 return new pass_omp_simd_clone (ctxt
);
12363 #include "gt-omp-low.h"