kernel - support dummy reallocblks in devfs
[dragonfly.git] / contrib / gcc-5.0 / gcc / omp-low.c
blob55afc60ed854eacf214c4c11cc928573b6a1e56e
1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
14 version.
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 for more details.
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
25 #include "config.h"
26 #include "system.h"
27 #include "coretypes.h"
28 #include "tm.h"
29 #include "hash-set.h"
30 #include "machmode.h"
31 #include "vec.h"
32 #include "double-int.h"
33 #include "input.h"
34 #include "alias.h"
35 #include "symtab.h"
36 #include "wide-int.h"
37 #include "inchash.h"
38 #include "tree.h"
39 #include "fold-const.h"
40 #include "stringpool.h"
41 #include "stor-layout.h"
42 #include "rtl.h"
43 #include "predict.h"
44 #include "hard-reg-set.h"
45 #include "function.h"
46 #include "dominance.h"
47 #include "cfg.h"
48 #include "cfganal.h"
49 #include "basic-block.h"
50 #include "tree-ssa-alias.h"
51 #include "internal-fn.h"
52 #include "gimple-fold.h"
53 #include "gimple-expr.h"
54 #include "is-a.h"
55 #include "gimple.h"
56 #include "gimplify.h"
57 #include "gimple-iterator.h"
58 #include "gimplify-me.h"
59 #include "gimple-walk.h"
60 #include "tree-iterator.h"
61 #include "tree-inline.h"
62 #include "langhooks.h"
63 #include "diagnostic-core.h"
64 #include "gimple-ssa.h"
65 #include "hash-map.h"
66 #include "plugin-api.h"
67 #include "ipa-ref.h"
68 #include "cgraph.h"
69 #include "tree-cfg.h"
70 #include "tree-phinodes.h"
71 #include "ssa-iterators.h"
72 #include "tree-ssanames.h"
73 #include "tree-into-ssa.h"
74 #include "hashtab.h"
75 #include "flags.h"
76 #include "statistics.h"
77 #include "real.h"
78 #include "fixed-value.h"
79 #include "insn-config.h"
80 #include "expmed.h"
81 #include "dojump.h"
82 #include "explow.h"
83 #include "calls.h"
84 #include "emit-rtl.h"
85 #include "varasm.h"
86 #include "stmt.h"
87 #include "expr.h"
88 #include "tree-dfa.h"
89 #include "tree-ssa.h"
90 #include "tree-pass.h"
91 #include "except.h"
92 #include "splay-tree.h"
93 #include "insn-codes.h"
94 #include "optabs.h"
95 #include "cfgloop.h"
96 #include "target.h"
97 #include "common/common-target.h"
98 #include "omp-low.h"
99 #include "gimple-low.h"
100 #include "tree-cfgcleanup.h"
101 #include "pretty-print.h"
102 #include "alloc-pool.h"
103 #include "symbol-summary.h"
104 #include "ipa-prop.h"
105 #include "tree-nested.h"
106 #include "tree-eh.h"
107 #include "cilk.h"
108 #include "context.h"
109 #include "lto-section-names.h"
110 #include "gomp-constants.h"
113 /* Lowering of OMP parallel and workshare constructs proceeds in two
114 phases. The first phase scans the function looking for OMP statements
115 and then for variables that must be replaced to satisfy data sharing
116 clauses. The second phase expands code for the constructs, as well as
117 re-gimplifying things when variables have been replaced with complex
118 expressions.
120 Final code generation is done by pass_expand_omp. The flowgraph is
121 scanned for regions which are then moved to a new
122 function, to be invoked by the thread library, or offloaded. */
124 /* OMP region information. Every parallel and workshare
125 directive is enclosed between two markers, the OMP_* directive
126 and a corresponding OMP_RETURN statement. */
128 struct omp_region
130 /* The enclosing region. */
131 struct omp_region *outer;
133 /* First child region. */
134 struct omp_region *inner;
136 /* Next peer region. */
137 struct omp_region *next;
139 /* Block containing the omp directive as its last stmt. */
140 basic_block entry;
142 /* Block containing the OMP_RETURN as its last stmt. */
143 basic_block exit;
145 /* Block containing the OMP_CONTINUE as its last stmt. */
146 basic_block cont;
148 /* If this is a combined parallel+workshare region, this is a list
149 of additional arguments needed by the combined parallel+workshare
150 library call. */
151 vec<tree, va_gc> *ws_args;
153 /* The code for the omp directive of this region. */
154 enum gimple_code type;
156 /* Schedule kind, only used for OMP_FOR type regions. */
157 enum omp_clause_schedule_kind sched_kind;
159 /* True if this is a combined parallel+workshare region. */
160 bool is_combined_parallel;
163 /* Levels of parallelism as defined by OpenACC. Increasing numbers
164 correspond to deeper loop nesting levels. */
165 #define MASK_GANG 1
166 #define MASK_WORKER 2
167 #define MASK_VECTOR 4
169 /* Context structure. Used to store information about each parallel
170 directive in the code. */
172 typedef struct omp_context
174 /* This field must be at the beginning, as we do "inheritance": Some
175 callback functions for tree-inline.c (e.g., omp_copy_decl)
176 receive a copy_body_data pointer that is up-casted to an
177 omp_context pointer. */
178 copy_body_data cb;
180 /* The tree of contexts corresponding to the encountered constructs. */
181 struct omp_context *outer;
182 gimple stmt;
184 /* Map variables to fields in a structure that allows communication
185 between sending and receiving threads. */
186 splay_tree field_map;
187 tree record_type;
188 tree sender_decl;
189 tree receiver_decl;
191 /* These are used just by task contexts, if task firstprivate fn is
192 needed. srecord_type is used to communicate from the thread
193 that encountered the task construct to task firstprivate fn,
194 record_type is allocated by GOMP_task, initialized by task firstprivate
195 fn and passed to the task body fn. */
196 splay_tree sfield_map;
197 tree srecord_type;
199 /* A chain of variables to add to the top-level block surrounding the
200 construct. In the case of a parallel, this is in the child function. */
201 tree block_vars;
203 /* A map of reduction pointer variables. For accelerators, each
204 reduction variable is replaced with an array. Each thread, in turn,
205 is assigned to a slot on that array. */
206 splay_tree reduction_map;
208 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
209 barriers should jump to during omplower pass. */
210 tree cancel_label;
212 /* What to do with variables with implicitly determined sharing
213 attributes. */
214 enum omp_clause_default_kind default_kind;
216 /* Nesting depth of this context. Used to beautify error messages re
217 invalid gotos. The outermost ctx is depth 1, with depth 0 being
218 reserved for the main body of the function. */
219 int depth;
221 /* True if this parallel directive is nested within another. */
222 bool is_nested;
224 /* True if this construct can be cancelled. */
225 bool cancellable;
227 /* For OpenACC loops, a mask of gang, worker and vector used at
228 levels below this one. */
229 int gwv_below;
230 /* For OpenACC loops, a mask of gang, worker and vector used at
231 this level and above. For parallel and kernels clauses, a mask
232 indicating which of num_gangs/num_workers/num_vectors was used. */
233 int gwv_this;
234 } omp_context;
236 /* A structure holding the elements of:
237 for (V = N1; V cond N2; V += STEP) [...] */
239 struct omp_for_data_loop
241 tree v, n1, n2, step;
242 enum tree_code cond_code;
245 /* A structure describing the main elements of a parallel loop. */
247 struct omp_for_data
249 struct omp_for_data_loop loop;
250 tree chunk_size;
251 gomp_for *for_stmt;
252 tree pre, iter_type;
253 int collapse;
254 bool have_nowait, have_ordered;
255 enum omp_clause_schedule_kind sched_kind;
256 struct omp_for_data_loop *loops;
260 static splay_tree all_contexts;
261 static int taskreg_nesting_level;
262 static int target_nesting_level;
263 static struct omp_region *root_omp_region;
264 static bitmap task_shared_vars;
265 static vec<omp_context *> taskreg_contexts;
267 static void scan_omp (gimple_seq *, omp_context *);
268 static tree scan_omp_1_op (tree *, int *, void *);
270 #define WALK_SUBSTMTS \
271 case GIMPLE_BIND: \
272 case GIMPLE_TRY: \
273 case GIMPLE_CATCH: \
274 case GIMPLE_EH_FILTER: \
275 case GIMPLE_TRANSACTION: \
276 /* The sub-statements for these should be walked. */ \
277 *handled_ops_p = false; \
278 break;
280 /* Helper function to get the name of the array containing the partial
281 reductions for OpenACC reductions. */
282 static const char *
283 oacc_get_reduction_array_id (tree node)
285 const char *id = IDENTIFIER_POINTER (DECL_NAME (node));
286 int len = strlen ("OACC") + strlen (id);
287 char *temp_name = XALLOCAVEC (char, len + 1);
288 snprintf (temp_name, len + 1, "OACC%s", id);
289 return IDENTIFIER_POINTER (get_identifier (temp_name));
292 /* Determine the number of threads OpenACC threads used to determine the
293 size of the array of partial reductions. Currently, this is num_gangs
294 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
295 because it is independed of the device used. */
297 static tree
298 oacc_max_threads (omp_context *ctx)
300 tree nthreads, vector_length, gangs, clauses;
302 gangs = fold_convert (sizetype, integer_one_node);
303 vector_length = gangs;
305 /* The reduction clause may be nested inside a loop directive.
306 Scan for the innermost vector_length clause. */
307 for (omp_context *oc = ctx; oc; oc = oc->outer)
309 if (gimple_code (oc->stmt) != GIMPLE_OMP_TARGET
310 || (gimple_omp_target_kind (oc->stmt)
311 != GF_OMP_TARGET_KIND_OACC_PARALLEL))
312 continue;
314 clauses = gimple_omp_target_clauses (oc->stmt);
316 vector_length = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH);
317 if (vector_length)
318 vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length),
319 sizetype,
320 OMP_CLAUSE_VECTOR_LENGTH_EXPR
321 (vector_length));
322 else
323 vector_length = fold_convert (sizetype, integer_one_node);
325 gangs = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS);
326 if (gangs)
327 gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (gangs), sizetype,
328 OMP_CLAUSE_NUM_GANGS_EXPR (gangs));
329 else
330 gangs = fold_convert (sizetype, integer_one_node);
332 break;
335 nthreads = fold_build2 (MULT_EXPR, sizetype, gangs, vector_length);
337 return nthreads;
340 /* Holds offload tables with decls. */
341 vec<tree, va_gc> *offload_funcs, *offload_vars;
343 /* Convenience function for calling scan_omp_1_op on tree operands. */
345 static inline tree
346 scan_omp_op (tree *tp, omp_context *ctx)
348 struct walk_stmt_info wi;
350 memset (&wi, 0, sizeof (wi));
351 wi.info = ctx;
352 wi.want_locations = true;
354 return walk_tree (tp, scan_omp_1_op, &wi, NULL);
357 static void lower_omp (gimple_seq *, omp_context *);
358 static tree lookup_decl_in_outer_ctx (tree, omp_context *);
359 static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
361 /* Find an OMP clause of type KIND within CLAUSES. */
363 tree
364 find_omp_clause (tree clauses, enum omp_clause_code kind)
366 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
367 if (OMP_CLAUSE_CODE (clauses) == kind)
368 return clauses;
370 return NULL_TREE;
373 /* Return true if CTX is for an omp parallel. */
375 static inline bool
376 is_parallel_ctx (omp_context *ctx)
378 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
382 /* Return true if CTX is for an omp task. */
384 static inline bool
385 is_task_ctx (omp_context *ctx)
387 return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
391 /* Return true if CTX is for an omp parallel or omp task. */
393 static inline bool
394 is_taskreg_ctx (omp_context *ctx)
396 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL
397 || gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
401 /* Return true if REGION is a combined parallel+workshare region. */
403 static inline bool
404 is_combined_parallel (struct omp_region *region)
406 return region->is_combined_parallel;
410 /* Extract the header elements of parallel loop FOR_STMT and store
411 them into *FD. */
413 static void
414 extract_omp_for_data (gomp_for *for_stmt, struct omp_for_data *fd,
415 struct omp_for_data_loop *loops)
417 tree t, var, *collapse_iter, *collapse_count;
418 tree count = NULL_TREE, iter_type = long_integer_type_node;
419 struct omp_for_data_loop *loop;
420 int i;
421 struct omp_for_data_loop dummy_loop;
422 location_t loc = gimple_location (for_stmt);
423 bool simd = gimple_omp_for_kind (for_stmt) & GF_OMP_FOR_SIMD;
424 bool distribute = gimple_omp_for_kind (for_stmt)
425 == GF_OMP_FOR_KIND_DISTRIBUTE;
427 fd->for_stmt = for_stmt;
428 fd->pre = NULL;
429 fd->collapse = gimple_omp_for_collapse (for_stmt);
430 if (fd->collapse > 1)
431 fd->loops = loops;
432 else
433 fd->loops = &fd->loop;
435 fd->have_nowait = distribute || simd;
436 fd->have_ordered = false;
437 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
438 fd->chunk_size = NULL_TREE;
439 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_CILKFOR)
440 fd->sched_kind = OMP_CLAUSE_SCHEDULE_CILKFOR;
441 collapse_iter = NULL;
442 collapse_count = NULL;
444 for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t))
445 switch (OMP_CLAUSE_CODE (t))
447 case OMP_CLAUSE_NOWAIT:
448 fd->have_nowait = true;
449 break;
450 case OMP_CLAUSE_ORDERED:
451 fd->have_ordered = true;
452 break;
453 case OMP_CLAUSE_SCHEDULE:
454 gcc_assert (!distribute);
455 fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t);
456 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t);
457 break;
458 case OMP_CLAUSE_DIST_SCHEDULE:
459 gcc_assert (distribute);
460 fd->chunk_size = OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t);
461 break;
462 case OMP_CLAUSE_COLLAPSE:
463 if (fd->collapse > 1)
465 collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t);
466 collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t);
468 break;
469 default:
470 break;
473 /* FIXME: for now map schedule(auto) to schedule(static).
474 There should be analysis to determine whether all iterations
475 are approximately the same amount of work (then schedule(static)
476 is best) or if it varies (then schedule(dynamic,N) is better). */
477 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO)
479 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
480 gcc_assert (fd->chunk_size == NULL);
482 gcc_assert (fd->collapse == 1 || collapse_iter != NULL);
483 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
484 gcc_assert (fd->chunk_size == NULL);
485 else if (fd->chunk_size == NULL)
487 /* We only need to compute a default chunk size for ordered
488 static loops and dynamic loops. */
489 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
490 || fd->have_ordered)
491 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
492 ? integer_zero_node : integer_one_node;
495 for (i = 0; i < fd->collapse; i++)
497 if (fd->collapse == 1)
498 loop = &fd->loop;
499 else if (loops != NULL)
500 loop = loops + i;
501 else
502 loop = &dummy_loop;
504 loop->v = gimple_omp_for_index (for_stmt, i);
505 gcc_assert (SSA_VAR_P (loop->v));
506 gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
507 || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE);
508 var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v;
509 loop->n1 = gimple_omp_for_initial (for_stmt, i);
511 loop->cond_code = gimple_omp_for_cond (for_stmt, i);
512 loop->n2 = gimple_omp_for_final (for_stmt, i);
513 switch (loop->cond_code)
515 case LT_EXPR:
516 case GT_EXPR:
517 break;
518 case NE_EXPR:
519 gcc_assert (gimple_omp_for_kind (for_stmt)
520 == GF_OMP_FOR_KIND_CILKSIMD
521 || (gimple_omp_for_kind (for_stmt)
522 == GF_OMP_FOR_KIND_CILKFOR));
523 break;
524 case LE_EXPR:
525 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
526 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, 1);
527 else
528 loop->n2 = fold_build2_loc (loc,
529 PLUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
530 build_int_cst (TREE_TYPE (loop->n2), 1));
531 loop->cond_code = LT_EXPR;
532 break;
533 case GE_EXPR:
534 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
535 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, -1);
536 else
537 loop->n2 = fold_build2_loc (loc,
538 MINUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
539 build_int_cst (TREE_TYPE (loop->n2), 1));
540 loop->cond_code = GT_EXPR;
541 break;
542 default:
543 gcc_unreachable ();
546 t = gimple_omp_for_incr (for_stmt, i);
547 gcc_assert (TREE_OPERAND (t, 0) == var);
548 switch (TREE_CODE (t))
550 case PLUS_EXPR:
551 loop->step = TREE_OPERAND (t, 1);
552 break;
553 case POINTER_PLUS_EXPR:
554 loop->step = fold_convert (ssizetype, TREE_OPERAND (t, 1));
555 break;
556 case MINUS_EXPR:
557 loop->step = TREE_OPERAND (t, 1);
558 loop->step = fold_build1_loc (loc,
559 NEGATE_EXPR, TREE_TYPE (loop->step),
560 loop->step);
561 break;
562 default:
563 gcc_unreachable ();
566 if (simd
567 || (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
568 && !fd->have_ordered))
570 if (fd->collapse == 1)
571 iter_type = TREE_TYPE (loop->v);
572 else if (i == 0
573 || TYPE_PRECISION (iter_type)
574 < TYPE_PRECISION (TREE_TYPE (loop->v)))
575 iter_type
576 = build_nonstandard_integer_type
577 (TYPE_PRECISION (TREE_TYPE (loop->v)), 1);
579 else if (iter_type != long_long_unsigned_type_node)
581 if (POINTER_TYPE_P (TREE_TYPE (loop->v)))
582 iter_type = long_long_unsigned_type_node;
583 else if (TYPE_UNSIGNED (TREE_TYPE (loop->v))
584 && TYPE_PRECISION (TREE_TYPE (loop->v))
585 >= TYPE_PRECISION (iter_type))
587 tree n;
589 if (loop->cond_code == LT_EXPR)
590 n = fold_build2_loc (loc,
591 PLUS_EXPR, TREE_TYPE (loop->v),
592 loop->n2, loop->step);
593 else
594 n = loop->n1;
595 if (TREE_CODE (n) != INTEGER_CST
596 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n))
597 iter_type = long_long_unsigned_type_node;
599 else if (TYPE_PRECISION (TREE_TYPE (loop->v))
600 > TYPE_PRECISION (iter_type))
602 tree n1, n2;
604 if (loop->cond_code == LT_EXPR)
606 n1 = loop->n1;
607 n2 = fold_build2_loc (loc,
608 PLUS_EXPR, TREE_TYPE (loop->v),
609 loop->n2, loop->step);
611 else
613 n1 = fold_build2_loc (loc,
614 MINUS_EXPR, TREE_TYPE (loop->v),
615 loop->n2, loop->step);
616 n2 = loop->n1;
618 if (TREE_CODE (n1) != INTEGER_CST
619 || TREE_CODE (n2) != INTEGER_CST
620 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1)
621 || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type)))
622 iter_type = long_long_unsigned_type_node;
626 if (collapse_count && *collapse_count == NULL)
628 t = fold_binary (loop->cond_code, boolean_type_node,
629 fold_convert (TREE_TYPE (loop->v), loop->n1),
630 fold_convert (TREE_TYPE (loop->v), loop->n2));
631 if (t && integer_zerop (t))
632 count = build_zero_cst (long_long_unsigned_type_node);
633 else if ((i == 0 || count != NULL_TREE)
634 && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
635 && TREE_CONSTANT (loop->n1)
636 && TREE_CONSTANT (loop->n2)
637 && TREE_CODE (loop->step) == INTEGER_CST)
639 tree itype = TREE_TYPE (loop->v);
641 if (POINTER_TYPE_P (itype))
642 itype = signed_type_for (itype);
643 t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1));
644 t = fold_build2_loc (loc,
645 PLUS_EXPR, itype,
646 fold_convert_loc (loc, itype, loop->step), t);
647 t = fold_build2_loc (loc, PLUS_EXPR, itype, t,
648 fold_convert_loc (loc, itype, loop->n2));
649 t = fold_build2_loc (loc, MINUS_EXPR, itype, t,
650 fold_convert_loc (loc, itype, loop->n1));
651 if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR)
652 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype,
653 fold_build1_loc (loc, NEGATE_EXPR, itype, t),
654 fold_build1_loc (loc, NEGATE_EXPR, itype,
655 fold_convert_loc (loc, itype,
656 loop->step)));
657 else
658 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t,
659 fold_convert_loc (loc, itype, loop->step));
660 t = fold_convert_loc (loc, long_long_unsigned_type_node, t);
661 if (count != NULL_TREE)
662 count = fold_build2_loc (loc,
663 MULT_EXPR, long_long_unsigned_type_node,
664 count, t);
665 else
666 count = t;
667 if (TREE_CODE (count) != INTEGER_CST)
668 count = NULL_TREE;
670 else if (count && !integer_zerop (count))
671 count = NULL_TREE;
675 if (count
676 && !simd
677 && (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
678 || fd->have_ordered))
680 if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node)))
681 iter_type = long_long_unsigned_type_node;
682 else
683 iter_type = long_integer_type_node;
685 else if (collapse_iter && *collapse_iter != NULL)
686 iter_type = TREE_TYPE (*collapse_iter);
687 fd->iter_type = iter_type;
688 if (collapse_iter && *collapse_iter == NULL)
689 *collapse_iter = create_tmp_var (iter_type, ".iter");
690 if (collapse_count && *collapse_count == NULL)
692 if (count)
693 *collapse_count = fold_convert_loc (loc, iter_type, count);
694 else
695 *collapse_count = create_tmp_var (iter_type, ".count");
698 if (fd->collapse > 1)
700 fd->loop.v = *collapse_iter;
701 fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0);
702 fd->loop.n2 = *collapse_count;
703 fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1);
704 fd->loop.cond_code = LT_EXPR;
707 /* For OpenACC loops, force a chunk size of one, as this avoids the default
708 scheduling where several subsequent iterations are being executed by the
709 same thread. */
710 if (gimple_omp_for_kind (for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
712 gcc_assert (fd->chunk_size == NULL_TREE);
713 fd->chunk_size = build_int_cst (TREE_TYPE (fd->loop.v), 1);
718 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
719 is the immediate dominator of PAR_ENTRY_BB, return true if there
720 are no data dependencies that would prevent expanding the parallel
721 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
723 When expanding a combined parallel+workshare region, the call to
724 the child function may need additional arguments in the case of
725 GIMPLE_OMP_FOR regions. In some cases, these arguments are
726 computed out of variables passed in from the parent to the child
727 via 'struct .omp_data_s'. For instance:
729 #pragma omp parallel for schedule (guided, i * 4)
730 for (j ...)
732 Is lowered into:
734 # BLOCK 2 (PAR_ENTRY_BB)
735 .omp_data_o.i = i;
736 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
738 # BLOCK 3 (WS_ENTRY_BB)
739 .omp_data_i = &.omp_data_o;
740 D.1667 = .omp_data_i->i;
741 D.1598 = D.1667 * 4;
742 #pragma omp for schedule (guided, D.1598)
744 When we outline the parallel region, the call to the child function
745 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
746 that value is computed *after* the call site. So, in principle we
747 cannot do the transformation.
749 To see whether the code in WS_ENTRY_BB blocks the combined
750 parallel+workshare call, we collect all the variables used in the
751 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
752 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
753 call.
755 FIXME. If we had the SSA form built at this point, we could merely
756 hoist the code in block 3 into block 2 and be done with it. But at
757 this point we don't have dataflow information and though we could
758 hack something up here, it is really not worth the aggravation. */
760 static bool
761 workshare_safe_to_combine_p (basic_block ws_entry_bb)
763 struct omp_for_data fd;
764 gimple ws_stmt = last_stmt (ws_entry_bb);
766 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
767 return true;
769 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
771 extract_omp_for_data (as_a <gomp_for *> (ws_stmt), &fd, NULL);
773 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
774 return false;
775 if (fd.iter_type != long_integer_type_node)
776 return false;
778 /* FIXME. We give up too easily here. If any of these arguments
779 are not constants, they will likely involve variables that have
780 been mapped into fields of .omp_data_s for sharing with the child
781 function. With appropriate data flow, it would be possible to
782 see through this. */
783 if (!is_gimple_min_invariant (fd.loop.n1)
784 || !is_gimple_min_invariant (fd.loop.n2)
785 || !is_gimple_min_invariant (fd.loop.step)
786 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
787 return false;
789 return true;
793 /* Collect additional arguments needed to emit a combined
794 parallel+workshare call. WS_STMT is the workshare directive being
795 expanded. */
797 static vec<tree, va_gc> *
798 get_ws_args_for (gimple par_stmt, gimple ws_stmt)
800 tree t;
801 location_t loc = gimple_location (ws_stmt);
802 vec<tree, va_gc> *ws_args;
804 if (gomp_for *for_stmt = dyn_cast <gomp_for *> (ws_stmt))
806 struct omp_for_data fd;
807 tree n1, n2;
809 extract_omp_for_data (for_stmt, &fd, NULL);
810 n1 = fd.loop.n1;
811 n2 = fd.loop.n2;
813 if (gimple_omp_for_combined_into_p (for_stmt))
815 tree innerc
816 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt),
817 OMP_CLAUSE__LOOPTEMP_);
818 gcc_assert (innerc);
819 n1 = OMP_CLAUSE_DECL (innerc);
820 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
821 OMP_CLAUSE__LOOPTEMP_);
822 gcc_assert (innerc);
823 n2 = OMP_CLAUSE_DECL (innerc);
826 vec_alloc (ws_args, 3 + (fd.chunk_size != 0));
828 t = fold_convert_loc (loc, long_integer_type_node, n1);
829 ws_args->quick_push (t);
831 t = fold_convert_loc (loc, long_integer_type_node, n2);
832 ws_args->quick_push (t);
834 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step);
835 ws_args->quick_push (t);
837 if (fd.chunk_size)
839 t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size);
840 ws_args->quick_push (t);
843 return ws_args;
845 else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
847 /* Number of sections is equal to the number of edges from the
848 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
849 the exit of the sections region. */
850 basic_block bb = single_succ (gimple_bb (ws_stmt));
851 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
852 vec_alloc (ws_args, 1);
853 ws_args->quick_push (t);
854 return ws_args;
857 gcc_unreachable ();
861 /* Discover whether REGION is a combined parallel+workshare region. */
863 static void
864 determine_parallel_type (struct omp_region *region)
866 basic_block par_entry_bb, par_exit_bb;
867 basic_block ws_entry_bb, ws_exit_bb;
869 if (region == NULL || region->inner == NULL
870 || region->exit == NULL || region->inner->exit == NULL
871 || region->inner->cont == NULL)
872 return;
874 /* We only support parallel+for and parallel+sections. */
875 if (region->type != GIMPLE_OMP_PARALLEL
876 || (region->inner->type != GIMPLE_OMP_FOR
877 && region->inner->type != GIMPLE_OMP_SECTIONS))
878 return;
880 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
881 WS_EXIT_BB -> PAR_EXIT_BB. */
882 par_entry_bb = region->entry;
883 par_exit_bb = region->exit;
884 ws_entry_bb = region->inner->entry;
885 ws_exit_bb = region->inner->exit;
887 if (single_succ (par_entry_bb) == ws_entry_bb
888 && single_succ (ws_exit_bb) == par_exit_bb
889 && workshare_safe_to_combine_p (ws_entry_bb)
890 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb))
891 || (last_and_only_stmt (ws_entry_bb)
892 && last_and_only_stmt (par_exit_bb))))
894 gimple par_stmt = last_stmt (par_entry_bb);
895 gimple ws_stmt = last_stmt (ws_entry_bb);
897 if (region->inner->type == GIMPLE_OMP_FOR)
899 /* If this is a combined parallel loop, we need to determine
900 whether or not to use the combined library calls. There
901 are two cases where we do not apply the transformation:
902 static loops and any kind of ordered loop. In the first
903 case, we already open code the loop so there is no need
904 to do anything else. In the latter case, the combined
905 parallel loop call would still need extra synchronization
906 to implement ordered semantics, so there would not be any
907 gain in using the combined call. */
908 tree clauses = gimple_omp_for_clauses (ws_stmt);
909 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE);
910 if (c == NULL
911 || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC
912 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED))
914 region->is_combined_parallel = false;
915 region->inner->is_combined_parallel = false;
916 return;
920 region->is_combined_parallel = true;
921 region->inner->is_combined_parallel = true;
922 region->ws_args = get_ws_args_for (par_stmt, ws_stmt);
927 /* Return true if EXPR is variable sized. */
929 static inline bool
930 is_variable_sized (const_tree expr)
932 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
935 /* Return true if DECL is a reference type. */
937 static inline bool
938 is_reference (tree decl)
940 return lang_hooks.decls.omp_privatize_by_reference (decl);
943 /* Return the type of a decl. If the decl is reference type,
944 return its base type. */
945 static inline tree
946 get_base_type (tree decl)
948 tree type = TREE_TYPE (decl);
949 if (is_reference (decl))
950 type = TREE_TYPE (type);
951 return type;
954 /* Lookup variables. The "maybe" form
955 allows for the variable form to not have been entered, otherwise we
956 assert that the variable must have been entered. */
958 static inline tree
959 lookup_decl (tree var, omp_context *ctx)
961 tree *n = ctx->cb.decl_map->get (var);
962 return *n;
965 static inline tree
966 maybe_lookup_decl (const_tree var, omp_context *ctx)
968 tree *n = ctx->cb.decl_map->get (const_cast<tree> (var));
969 return n ? *n : NULL_TREE;
972 static inline tree
973 lookup_field (tree var, omp_context *ctx)
975 splay_tree_node n;
976 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
977 return (tree) n->value;
980 static inline tree
981 lookup_sfield (tree var, omp_context *ctx)
983 splay_tree_node n;
984 n = splay_tree_lookup (ctx->sfield_map
985 ? ctx->sfield_map : ctx->field_map,
986 (splay_tree_key) var);
987 return (tree) n->value;
990 static inline tree
991 maybe_lookup_field (tree var, omp_context *ctx)
993 splay_tree_node n;
994 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
995 return n ? (tree) n->value : NULL_TREE;
998 static inline tree
999 lookup_oacc_reduction (const char *id, omp_context *ctx)
1001 splay_tree_node n;
1002 n = splay_tree_lookup (ctx->reduction_map, (splay_tree_key) id);
1003 return (tree) n->value;
1006 static inline tree
1007 maybe_lookup_oacc_reduction (tree var, omp_context *ctx)
1009 splay_tree_node n = NULL;
1010 if (ctx->reduction_map)
1011 n = splay_tree_lookup (ctx->reduction_map, (splay_tree_key) var);
1012 return n ? (tree) n->value : NULL_TREE;
1015 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1016 the parallel context if DECL is to be shared. */
1018 static bool
1019 use_pointer_for_field (tree decl, omp_context *shared_ctx)
1021 if (AGGREGATE_TYPE_P (TREE_TYPE (decl)))
1022 return true;
1024 /* We can only use copy-in/copy-out semantics for shared variables
1025 when we know the value is not accessible from an outer scope. */
1026 if (shared_ctx)
1028 gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt));
1030 /* ??? Trivially accessible from anywhere. But why would we even
1031 be passing an address in this case? Should we simply assert
1032 this to be false, or should we have a cleanup pass that removes
1033 these from the list of mappings? */
1034 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
1035 return true;
1037 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1038 without analyzing the expression whether or not its location
1039 is accessible to anyone else. In the case of nested parallel
1040 regions it certainly may be. */
1041 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
1042 return true;
1044 /* Do not use copy-in/copy-out for variables that have their
1045 address taken. */
1046 if (TREE_ADDRESSABLE (decl))
1047 return true;
1049 /* lower_send_shared_vars only uses copy-in, but not copy-out
1050 for these. */
1051 if (TREE_READONLY (decl)
1052 || ((TREE_CODE (decl) == RESULT_DECL
1053 || TREE_CODE (decl) == PARM_DECL)
1054 && DECL_BY_REFERENCE (decl)))
1055 return false;
1057 /* Disallow copy-in/out in nested parallel if
1058 decl is shared in outer parallel, otherwise
1059 each thread could store the shared variable
1060 in its own copy-in location, making the
1061 variable no longer really shared. */
1062 if (shared_ctx->is_nested)
1064 omp_context *up;
1066 for (up = shared_ctx->outer; up; up = up->outer)
1067 if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up))
1068 break;
1070 if (up)
1072 tree c;
1074 for (c = gimple_omp_taskreg_clauses (up->stmt);
1075 c; c = OMP_CLAUSE_CHAIN (c))
1076 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
1077 && OMP_CLAUSE_DECL (c) == decl)
1078 break;
1080 if (c)
1081 goto maybe_mark_addressable_and_ret;
1085 /* For tasks avoid using copy-in/out. As tasks can be
1086 deferred or executed in different thread, when GOMP_task
1087 returns, the task hasn't necessarily terminated. */
1088 if (is_task_ctx (shared_ctx))
1090 tree outer;
1091 maybe_mark_addressable_and_ret:
1092 outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
1093 if (is_gimple_reg (outer))
1095 /* Taking address of OUTER in lower_send_shared_vars
1096 might need regimplification of everything that uses the
1097 variable. */
1098 if (!task_shared_vars)
1099 task_shared_vars = BITMAP_ALLOC (NULL);
1100 bitmap_set_bit (task_shared_vars, DECL_UID (outer));
1101 TREE_ADDRESSABLE (outer) = 1;
1103 return true;
1107 return false;
1110 /* Construct a new automatic decl similar to VAR. */
1112 static tree
1113 omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
1115 tree copy = copy_var_decl (var, name, type);
1117 DECL_CONTEXT (copy) = current_function_decl;
1118 DECL_CHAIN (copy) = ctx->block_vars;
1119 ctx->block_vars = copy;
1121 return copy;
1124 static tree
1125 omp_copy_decl_1 (tree var, omp_context *ctx)
1127 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
1130 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1131 as appropriate. */
1132 static tree
1133 omp_build_component_ref (tree obj, tree field)
1135 tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL);
1136 if (TREE_THIS_VOLATILE (field))
1137 TREE_THIS_VOLATILE (ret) |= 1;
1138 if (TREE_READONLY (field))
1139 TREE_READONLY (ret) |= 1;
1140 return ret;
1143 /* Build tree nodes to access the field for VAR on the receiver side. */
1145 static tree
1146 build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
1148 tree x, field = lookup_field (var, ctx);
1150 /* If the receiver record type was remapped in the child function,
1151 remap the field into the new record type. */
1152 x = maybe_lookup_field (field, ctx);
1153 if (x != NULL)
1154 field = x;
1156 x = build_simple_mem_ref (ctx->receiver_decl);
1157 x = omp_build_component_ref (x, field);
1158 if (by_ref)
1159 x = build_simple_mem_ref (x);
1161 return x;
1164 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1165 of a parallel, this is a component reference; for workshare constructs
1166 this is some variable. */
1168 static tree
1169 build_outer_var_ref (tree var, omp_context *ctx)
1171 tree x;
1173 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
1174 x = var;
1175 else if (is_variable_sized (var))
1177 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
1178 x = build_outer_var_ref (x, ctx);
1179 x = build_simple_mem_ref (x);
1181 else if (is_taskreg_ctx (ctx))
1183 bool by_ref = use_pointer_for_field (var, NULL);
1184 x = build_receiver_ref (var, by_ref, ctx);
1186 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1187 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
1189 /* #pragma omp simd isn't a worksharing construct, and can reference even
1190 private vars in its linear etc. clauses. */
1191 x = NULL_TREE;
1192 if (ctx->outer && is_taskreg_ctx (ctx))
1193 x = lookup_decl (var, ctx->outer);
1194 else if (ctx->outer)
1195 x = maybe_lookup_decl_in_outer_ctx (var, ctx);
1196 if (x == NULL_TREE)
1197 x = var;
1199 else if (ctx->outer)
1200 x = lookup_decl (var, ctx->outer);
1201 else if (is_reference (var))
1202 /* This can happen with orphaned constructs. If var is reference, it is
1203 possible it is shared and as such valid. */
1204 x = var;
1205 else
1206 gcc_unreachable ();
1208 if (is_reference (var))
1209 x = build_simple_mem_ref (x);
1211 return x;
1214 /* Build tree nodes to access the field for VAR on the sender side. */
1216 static tree
1217 build_sender_ref (tree var, omp_context *ctx)
1219 tree field = lookup_sfield (var, ctx);
1220 return omp_build_component_ref (ctx->sender_decl, field);
1223 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1225 static void
1226 install_var_field (tree var, bool by_ref, int mask, omp_context *ctx)
1228 tree field, type, sfield = NULL_TREE;
1230 gcc_assert ((mask & 1) == 0
1231 || !splay_tree_lookup (ctx->field_map, (splay_tree_key) var));
1232 gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
1233 || !splay_tree_lookup (ctx->sfield_map, (splay_tree_key) var));
1234 gcc_assert ((mask & 3) == 3
1235 || !is_gimple_omp_oacc (ctx->stmt));
1237 type = TREE_TYPE (var);
1238 if (mask & 4)
1240 gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
1241 type = build_pointer_type (build_pointer_type (type));
1243 else if (by_ref)
1244 type = build_pointer_type (type);
1245 else if ((mask & 3) == 1 && is_reference (var))
1246 type = TREE_TYPE (type);
1248 field = build_decl (DECL_SOURCE_LOCATION (var),
1249 FIELD_DECL, DECL_NAME (var), type);
1251 /* Remember what variable this field was created for. This does have a
1252 side effect of making dwarf2out ignore this member, so for helpful
1253 debugging we clear it later in delete_omp_context. */
1254 DECL_ABSTRACT_ORIGIN (field) = var;
1255 if (type == TREE_TYPE (var))
1257 DECL_ALIGN (field) = DECL_ALIGN (var);
1258 DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
1259 TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
1261 else
1262 DECL_ALIGN (field) = TYPE_ALIGN (type);
1264 if ((mask & 3) == 3)
1266 insert_field_into_struct (ctx->record_type, field);
1267 if (ctx->srecord_type)
1269 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1270 FIELD_DECL, DECL_NAME (var), type);
1271 DECL_ABSTRACT_ORIGIN (sfield) = var;
1272 DECL_ALIGN (sfield) = DECL_ALIGN (field);
1273 DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
1274 TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
1275 insert_field_into_struct (ctx->srecord_type, sfield);
1278 else
1280 if (ctx->srecord_type == NULL_TREE)
1282 tree t;
1284 ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
1285 ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1286 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
1288 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1289 FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
1290 DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
1291 insert_field_into_struct (ctx->srecord_type, sfield);
1292 splay_tree_insert (ctx->sfield_map,
1293 (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
1294 (splay_tree_value) sfield);
1297 sfield = field;
1298 insert_field_into_struct ((mask & 1) ? ctx->record_type
1299 : ctx->srecord_type, field);
1302 if (mask & 1)
1303 splay_tree_insert (ctx->field_map, (splay_tree_key) var,
1304 (splay_tree_value) field);
1305 if ((mask & 2) && ctx->sfield_map)
1306 splay_tree_insert (ctx->sfield_map, (splay_tree_key) var,
1307 (splay_tree_value) sfield);
1310 static tree
1311 install_var_local (tree var, omp_context *ctx)
1313 tree new_var = omp_copy_decl_1 (var, ctx);
1314 insert_decl_map (&ctx->cb, var, new_var);
1315 return new_var;
1318 /* Adjust the replacement for DECL in CTX for the new context. This means
1319 copying the DECL_VALUE_EXPR, and fixing up the type. */
1321 static void
1322 fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
1324 tree new_decl, size;
1326 new_decl = lookup_decl (decl, ctx);
1328 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
1330 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
1331 && DECL_HAS_VALUE_EXPR_P (decl))
1333 tree ve = DECL_VALUE_EXPR (decl);
1334 walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
1335 SET_DECL_VALUE_EXPR (new_decl, ve);
1336 DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
1339 if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
1341 size = remap_decl (DECL_SIZE (decl), &ctx->cb);
1342 if (size == error_mark_node)
1343 size = TYPE_SIZE (TREE_TYPE (new_decl));
1344 DECL_SIZE (new_decl) = size;
1346 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
1347 if (size == error_mark_node)
1348 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
1349 DECL_SIZE_UNIT (new_decl) = size;
1353 /* The callback for remap_decl. Search all containing contexts for a
1354 mapping of the variable; this avoids having to duplicate the splay
1355 tree ahead of time. We know a mapping doesn't already exist in the
1356 given context. Create new mappings to implement default semantics. */
1358 static tree
1359 omp_copy_decl (tree var, copy_body_data *cb)
1361 omp_context *ctx = (omp_context *) cb;
1362 tree new_var;
1364 if (TREE_CODE (var) == LABEL_DECL)
1366 new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
1367 DECL_CONTEXT (new_var) = current_function_decl;
1368 insert_decl_map (&ctx->cb, var, new_var);
1369 return new_var;
1372 while (!is_taskreg_ctx (ctx))
1374 ctx = ctx->outer;
1375 if (ctx == NULL)
1376 return var;
1377 new_var = maybe_lookup_decl (var, ctx);
1378 if (new_var)
1379 return new_var;
1382 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
1383 return var;
1385 return error_mark_node;
1389 /* Debugging dumps for parallel regions. */
1390 void dump_omp_region (FILE *, struct omp_region *, int);
1391 void debug_omp_region (struct omp_region *);
1392 void debug_all_omp_regions (void);
1394 /* Dump the parallel region tree rooted at REGION. */
1396 void
1397 dump_omp_region (FILE *file, struct omp_region *region, int indent)
1399 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
1400 gimple_code_name[region->type]);
1402 if (region->inner)
1403 dump_omp_region (file, region->inner, indent + 4);
1405 if (region->cont)
1407 fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "",
1408 region->cont->index);
1411 if (region->exit)
1412 fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "",
1413 region->exit->index);
1414 else
1415 fprintf (file, "%*s[no exit marker]\n", indent, "");
1417 if (region->next)
1418 dump_omp_region (file, region->next, indent);
1421 DEBUG_FUNCTION void
1422 debug_omp_region (struct omp_region *region)
1424 dump_omp_region (stderr, region, 0);
1427 DEBUG_FUNCTION void
1428 debug_all_omp_regions (void)
1430 dump_omp_region (stderr, root_omp_region, 0);
1434 /* Create a new parallel region starting at STMT inside region PARENT. */
1436 static struct omp_region *
1437 new_omp_region (basic_block bb, enum gimple_code type,
1438 struct omp_region *parent)
1440 struct omp_region *region = XCNEW (struct omp_region);
1442 region->outer = parent;
1443 region->entry = bb;
1444 region->type = type;
1446 if (parent)
1448 /* This is a nested region. Add it to the list of inner
1449 regions in PARENT. */
1450 region->next = parent->inner;
1451 parent->inner = region;
1453 else
1455 /* This is a toplevel region. Add it to the list of toplevel
1456 regions in ROOT_OMP_REGION. */
1457 region->next = root_omp_region;
1458 root_omp_region = region;
1461 return region;
1464 /* Release the memory associated with the region tree rooted at REGION. */
1466 static void
1467 free_omp_region_1 (struct omp_region *region)
1469 struct omp_region *i, *n;
1471 for (i = region->inner; i ; i = n)
1473 n = i->next;
1474 free_omp_region_1 (i);
1477 free (region);
1480 /* Release the memory for the entire omp region tree. */
1482 void
1483 free_omp_regions (void)
1485 struct omp_region *r, *n;
1486 for (r = root_omp_region; r ; r = n)
1488 n = r->next;
1489 free_omp_region_1 (r);
1491 root_omp_region = NULL;
1495 /* Create a new context, with OUTER_CTX being the surrounding context. */
1497 static omp_context *
1498 new_omp_context (gimple stmt, omp_context *outer_ctx)
1500 omp_context *ctx = XCNEW (omp_context);
1502 splay_tree_insert (all_contexts, (splay_tree_key) stmt,
1503 (splay_tree_value) ctx);
1504 ctx->stmt = stmt;
1506 if (outer_ctx)
1508 ctx->outer = outer_ctx;
1509 ctx->cb = outer_ctx->cb;
1510 ctx->cb.block = NULL;
1511 ctx->depth = outer_ctx->depth + 1;
1512 ctx->reduction_map = outer_ctx->reduction_map;
1514 else
1516 ctx->cb.src_fn = current_function_decl;
1517 ctx->cb.dst_fn = current_function_decl;
1518 ctx->cb.src_node = cgraph_node::get (current_function_decl);
1519 gcc_checking_assert (ctx->cb.src_node);
1520 ctx->cb.dst_node = ctx->cb.src_node;
1521 ctx->cb.src_cfun = cfun;
1522 ctx->cb.copy_decl = omp_copy_decl;
1523 ctx->cb.eh_lp_nr = 0;
1524 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1525 ctx->depth = 1;
1528 ctx->cb.decl_map = new hash_map<tree, tree>;
1530 return ctx;
1533 static gimple_seq maybe_catch_exception (gimple_seq);
1535 /* Finalize task copyfn. */
1537 static void
1538 finalize_task_copyfn (gomp_task *task_stmt)
1540 struct function *child_cfun;
1541 tree child_fn;
1542 gimple_seq seq = NULL, new_seq;
1543 gbind *bind;
1545 child_fn = gimple_omp_task_copy_fn (task_stmt);
1546 if (child_fn == NULL_TREE)
1547 return;
1549 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1550 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1552 push_cfun (child_cfun);
1553 bind = gimplify_body (child_fn, false);
1554 gimple_seq_add_stmt (&seq, bind);
1555 new_seq = maybe_catch_exception (seq);
1556 if (new_seq != seq)
1558 bind = gimple_build_bind (NULL, new_seq, NULL);
1559 seq = NULL;
1560 gimple_seq_add_stmt (&seq, bind);
1562 gimple_set_body (child_fn, seq);
1563 pop_cfun ();
1565 /* Inform the callgraph about the new function. */
1566 cgraph_node::add_new_function (child_fn, false);
1567 cgraph_node::get (child_fn)->parallelized_function = 1;
1570 /* Destroy a omp_context data structures. Called through the splay tree
1571 value delete callback. */
1573 static void
1574 delete_omp_context (splay_tree_value value)
1576 omp_context *ctx = (omp_context *) value;
1578 delete ctx->cb.decl_map;
1580 if (ctx->field_map)
1581 splay_tree_delete (ctx->field_map);
1582 if (ctx->sfield_map)
1583 splay_tree_delete (ctx->sfield_map);
1584 /* Reduction map is copied to nested contexts, so only delete it in the
1585 owner. */
1586 if (ctx->reduction_map
1587 && gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
1588 && is_gimple_omp_offloaded (ctx->stmt)
1589 && is_gimple_omp_oacc (ctx->stmt))
1590 splay_tree_delete (ctx->reduction_map);
1592 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1593 it produces corrupt debug information. */
1594 if (ctx->record_type)
1596 tree t;
1597 for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1598 DECL_ABSTRACT_ORIGIN (t) = NULL;
1600 if (ctx->srecord_type)
1602 tree t;
1603 for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1604 DECL_ABSTRACT_ORIGIN (t) = NULL;
1607 if (is_task_ctx (ctx))
1608 finalize_task_copyfn (as_a <gomp_task *> (ctx->stmt));
1610 XDELETE (ctx);
1613 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1614 context. */
1616 static void
1617 fixup_child_record_type (omp_context *ctx)
1619 tree f, type = ctx->record_type;
1621 /* ??? It isn't sufficient to just call remap_type here, because
1622 variably_modified_type_p doesn't work the way we expect for
1623 record types. Testing each field for whether it needs remapping
1624 and creating a new record by hand works, however. */
1625 for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1626 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1627 break;
1628 if (f)
1630 tree name, new_fields = NULL;
1632 type = lang_hooks.types.make_type (RECORD_TYPE);
1633 name = DECL_NAME (TYPE_NAME (ctx->record_type));
1634 name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1635 TYPE_DECL, name, type);
1636 TYPE_NAME (type) = name;
1638 for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1640 tree new_f = copy_node (f);
1641 DECL_CONTEXT (new_f) = type;
1642 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1643 DECL_CHAIN (new_f) = new_fields;
1644 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1645 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1646 &ctx->cb, NULL);
1647 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1648 &ctx->cb, NULL);
1649 new_fields = new_f;
1651 /* Arrange to be able to look up the receiver field
1652 given the sender field. */
1653 splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1654 (splay_tree_value) new_f);
1656 TYPE_FIELDS (type) = nreverse (new_fields);
1657 layout_type (type);
1660 TREE_TYPE (ctx->receiver_decl)
1661 = build_qualified_type (build_reference_type (type), TYPE_QUAL_RESTRICT);
1664 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1665 specified by CLAUSES. */
1667 static void
1668 scan_sharing_clauses (tree clauses, omp_context *ctx)
1670 tree c, decl;
1671 bool scan_array_reductions = false;
1673 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1675 bool by_ref;
1677 switch (OMP_CLAUSE_CODE (c))
1679 case OMP_CLAUSE_PRIVATE:
1680 decl = OMP_CLAUSE_DECL (c);
1681 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1682 goto do_private;
1683 else if (!is_variable_sized (decl))
1684 install_var_local (decl, ctx);
1685 break;
1687 case OMP_CLAUSE_SHARED:
1688 decl = OMP_CLAUSE_DECL (c);
1689 /* Ignore shared directives in teams construct. */
1690 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1692 /* Global variables don't need to be copied,
1693 the receiver side will use them directly. */
1694 tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx);
1695 if (is_global_var (odecl))
1696 break;
1697 insert_decl_map (&ctx->cb, decl, odecl);
1698 break;
1700 gcc_assert (is_taskreg_ctx (ctx));
1701 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1702 || !is_variable_sized (decl));
1703 /* Global variables don't need to be copied,
1704 the receiver side will use them directly. */
1705 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1706 break;
1707 by_ref = use_pointer_for_field (decl, ctx);
1708 if (! TREE_READONLY (decl)
1709 || TREE_ADDRESSABLE (decl)
1710 || by_ref
1711 || is_reference (decl))
1713 install_var_field (decl, by_ref, 3, ctx);
1714 install_var_local (decl, ctx);
1715 break;
1717 /* We don't need to copy const scalar vars back. */
1718 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1719 goto do_private;
1721 case OMP_CLAUSE_LASTPRIVATE:
1722 /* Let the corresponding firstprivate clause create
1723 the variable. */
1724 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1725 break;
1726 /* FALLTHRU */
1728 case OMP_CLAUSE_FIRSTPRIVATE:
1729 if (is_gimple_omp_oacc (ctx->stmt))
1731 sorry ("clause not supported yet");
1732 break;
1734 /* FALLTHRU */
1735 case OMP_CLAUSE_REDUCTION:
1736 case OMP_CLAUSE_LINEAR:
1737 decl = OMP_CLAUSE_DECL (c);
1738 do_private:
1739 if (is_variable_sized (decl))
1741 if (is_task_ctx (ctx))
1742 install_var_field (decl, false, 1, ctx);
1743 break;
1745 else if (is_taskreg_ctx (ctx))
1747 bool global
1748 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1749 by_ref = use_pointer_for_field (decl, NULL);
1751 if (is_task_ctx (ctx)
1752 && (global || by_ref || is_reference (decl)))
1754 install_var_field (decl, false, 1, ctx);
1755 if (!global)
1756 install_var_field (decl, by_ref, 2, ctx);
1758 else if (!global)
1759 install_var_field (decl, by_ref, 3, ctx);
1761 install_var_local (decl, ctx);
1762 if (is_gimple_omp_oacc (ctx->stmt)
1763 && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
1765 /* Create a decl for the reduction array. */
1766 tree var = OMP_CLAUSE_DECL (c);
1767 tree type = get_base_type (var);
1768 tree ptype = build_pointer_type (type);
1769 tree array = create_tmp_var (ptype,
1770 oacc_get_reduction_array_id (var));
1771 omp_context *c = (ctx->field_map ? ctx : ctx->outer);
1772 install_var_field (array, true, 3, c);
1773 install_var_local (array, c);
1775 /* Insert it into the current context. */
1776 splay_tree_insert (ctx->reduction_map, (splay_tree_key)
1777 oacc_get_reduction_array_id (var),
1778 (splay_tree_value) array);
1779 splay_tree_insert (ctx->reduction_map,
1780 (splay_tree_key) array,
1781 (splay_tree_value) array);
1783 break;
1785 case OMP_CLAUSE__LOOPTEMP_:
1786 gcc_assert (is_parallel_ctx (ctx));
1787 decl = OMP_CLAUSE_DECL (c);
1788 install_var_field (decl, false, 3, ctx);
1789 install_var_local (decl, ctx);
1790 break;
1792 case OMP_CLAUSE_COPYPRIVATE:
1793 case OMP_CLAUSE_COPYIN:
1794 decl = OMP_CLAUSE_DECL (c);
1795 by_ref = use_pointer_for_field (decl, NULL);
1796 install_var_field (decl, by_ref, 3, ctx);
1797 break;
1799 case OMP_CLAUSE_DEFAULT:
1800 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c);
1801 break;
1803 case OMP_CLAUSE_FINAL:
1804 case OMP_CLAUSE_IF:
1805 case OMP_CLAUSE_NUM_THREADS:
1806 case OMP_CLAUSE_NUM_TEAMS:
1807 case OMP_CLAUSE_THREAD_LIMIT:
1808 case OMP_CLAUSE_DEVICE:
1809 case OMP_CLAUSE_SCHEDULE:
1810 case OMP_CLAUSE_DIST_SCHEDULE:
1811 case OMP_CLAUSE_DEPEND:
1812 case OMP_CLAUSE__CILK_FOR_COUNT_:
1813 case OMP_CLAUSE_NUM_GANGS:
1814 case OMP_CLAUSE_NUM_WORKERS:
1815 case OMP_CLAUSE_VECTOR_LENGTH:
1816 if (ctx->outer)
1817 scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1818 break;
1820 case OMP_CLAUSE_TO:
1821 case OMP_CLAUSE_FROM:
1822 case OMP_CLAUSE_MAP:
1823 if (ctx->outer)
1824 scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1825 decl = OMP_CLAUSE_DECL (c);
1826 /* Global variables with "omp declare target" attribute
1827 don't need to be copied, the receiver side will use them
1828 directly. */
1829 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1830 && DECL_P (decl)
1831 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1832 && varpool_node::get_create (decl)->offloadable)
1833 break;
1834 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1835 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER)
1837 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1838 not offloaded; there is nothing to map for those. */
1839 if (!is_gimple_omp_offloaded (ctx->stmt)
1840 && !POINTER_TYPE_P (TREE_TYPE (decl))
1841 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
1842 break;
1844 if (DECL_P (decl))
1846 if (DECL_SIZE (decl)
1847 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1849 tree decl2 = DECL_VALUE_EXPR (decl);
1850 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
1851 decl2 = TREE_OPERAND (decl2, 0);
1852 gcc_assert (DECL_P (decl2));
1853 install_var_field (decl2, true, 3, ctx);
1854 install_var_local (decl2, ctx);
1855 install_var_local (decl, ctx);
1857 else
1859 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1860 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1861 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1862 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1863 install_var_field (decl, true, 7, ctx);
1864 else
1865 install_var_field (decl, true, 3, ctx);
1866 if (is_gimple_omp_offloaded (ctx->stmt))
1867 install_var_local (decl, ctx);
1870 else
1872 tree base = get_base_address (decl);
1873 tree nc = OMP_CLAUSE_CHAIN (c);
1874 if (DECL_P (base)
1875 && nc != NULL_TREE
1876 && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1877 && OMP_CLAUSE_DECL (nc) == base
1878 && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER
1879 && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1881 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1882 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1884 else
1886 if (ctx->outer)
1888 scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer);
1889 decl = OMP_CLAUSE_DECL (c);
1891 gcc_assert (!splay_tree_lookup (ctx->field_map,
1892 (splay_tree_key) decl));
1893 tree field
1894 = build_decl (OMP_CLAUSE_LOCATION (c),
1895 FIELD_DECL, NULL_TREE, ptr_type_node);
1896 DECL_ALIGN (field) = TYPE_ALIGN (ptr_type_node);
1897 insert_field_into_struct (ctx->record_type, field);
1898 splay_tree_insert (ctx->field_map, (splay_tree_key) decl,
1899 (splay_tree_value) field);
1902 break;
1904 case OMP_CLAUSE_NOWAIT:
1905 case OMP_CLAUSE_ORDERED:
1906 case OMP_CLAUSE_COLLAPSE:
1907 case OMP_CLAUSE_UNTIED:
1908 case OMP_CLAUSE_MERGEABLE:
1909 case OMP_CLAUSE_PROC_BIND:
1910 case OMP_CLAUSE_SAFELEN:
1911 case OMP_CLAUSE_ASYNC:
1912 case OMP_CLAUSE_WAIT:
1913 case OMP_CLAUSE_GANG:
1914 case OMP_CLAUSE_WORKER:
1915 case OMP_CLAUSE_VECTOR:
1916 break;
1918 case OMP_CLAUSE_ALIGNED:
1919 decl = OMP_CLAUSE_DECL (c);
1920 if (is_global_var (decl)
1921 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1922 install_var_local (decl, ctx);
1923 break;
1925 case OMP_CLAUSE_DEVICE_RESIDENT:
1926 case OMP_CLAUSE_USE_DEVICE:
1927 case OMP_CLAUSE__CACHE_:
1928 case OMP_CLAUSE_INDEPENDENT:
1929 case OMP_CLAUSE_AUTO:
1930 case OMP_CLAUSE_SEQ:
1931 sorry ("Clause not supported yet");
1932 break;
1934 default:
1935 gcc_unreachable ();
1939 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1941 switch (OMP_CLAUSE_CODE (c))
1943 case OMP_CLAUSE_LASTPRIVATE:
1944 /* Let the corresponding firstprivate clause create
1945 the variable. */
1946 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1947 scan_array_reductions = true;
1948 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1949 break;
1950 /* FALLTHRU */
1952 case OMP_CLAUSE_FIRSTPRIVATE:
1953 if (is_gimple_omp_oacc (ctx->stmt))
1955 sorry ("clause not supported yet");
1956 break;
1958 /* FALLTHRU */
1959 case OMP_CLAUSE_PRIVATE:
1960 case OMP_CLAUSE_REDUCTION:
1961 case OMP_CLAUSE_LINEAR:
1962 decl = OMP_CLAUSE_DECL (c);
1963 if (is_variable_sized (decl))
1964 install_var_local (decl, ctx);
1965 fixup_remapped_decl (decl, ctx,
1966 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1967 && OMP_CLAUSE_PRIVATE_DEBUG (c));
1968 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1969 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1970 scan_array_reductions = true;
1971 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
1972 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
1973 scan_array_reductions = true;
1974 break;
1976 case OMP_CLAUSE_SHARED:
1977 /* Ignore shared directives in teams construct. */
1978 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1979 break;
1980 decl = OMP_CLAUSE_DECL (c);
1981 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1982 fixup_remapped_decl (decl, ctx, false);
1983 break;
1985 case OMP_CLAUSE_MAP:
1986 if (!is_gimple_omp_offloaded (ctx->stmt))
1987 break;
1988 decl = OMP_CLAUSE_DECL (c);
1989 if (DECL_P (decl)
1990 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1991 && varpool_node::get_create (decl)->offloadable)
1992 break;
1993 if (DECL_P (decl))
1995 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1996 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
1997 && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
1999 tree new_decl = lookup_decl (decl, ctx);
2000 TREE_TYPE (new_decl)
2001 = remap_type (TREE_TYPE (decl), &ctx->cb);
2003 else if (DECL_SIZE (decl)
2004 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
2006 tree decl2 = DECL_VALUE_EXPR (decl);
2007 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
2008 decl2 = TREE_OPERAND (decl2, 0);
2009 gcc_assert (DECL_P (decl2));
2010 fixup_remapped_decl (decl2, ctx, false);
2011 fixup_remapped_decl (decl, ctx, true);
2013 else
2014 fixup_remapped_decl (decl, ctx, false);
2016 break;
2018 case OMP_CLAUSE_COPYPRIVATE:
2019 case OMP_CLAUSE_COPYIN:
2020 case OMP_CLAUSE_DEFAULT:
2021 case OMP_CLAUSE_IF:
2022 case OMP_CLAUSE_NUM_THREADS:
2023 case OMP_CLAUSE_NUM_TEAMS:
2024 case OMP_CLAUSE_THREAD_LIMIT:
2025 case OMP_CLAUSE_DEVICE:
2026 case OMP_CLAUSE_SCHEDULE:
2027 case OMP_CLAUSE_DIST_SCHEDULE:
2028 case OMP_CLAUSE_NOWAIT:
2029 case OMP_CLAUSE_ORDERED:
2030 case OMP_CLAUSE_COLLAPSE:
2031 case OMP_CLAUSE_UNTIED:
2032 case OMP_CLAUSE_FINAL:
2033 case OMP_CLAUSE_MERGEABLE:
2034 case OMP_CLAUSE_PROC_BIND:
2035 case OMP_CLAUSE_SAFELEN:
2036 case OMP_CLAUSE_ALIGNED:
2037 case OMP_CLAUSE_DEPEND:
2038 case OMP_CLAUSE__LOOPTEMP_:
2039 case OMP_CLAUSE_TO:
2040 case OMP_CLAUSE_FROM:
2041 case OMP_CLAUSE__CILK_FOR_COUNT_:
2042 case OMP_CLAUSE_ASYNC:
2043 case OMP_CLAUSE_WAIT:
2044 case OMP_CLAUSE_NUM_GANGS:
2045 case OMP_CLAUSE_NUM_WORKERS:
2046 case OMP_CLAUSE_VECTOR_LENGTH:
2047 case OMP_CLAUSE_GANG:
2048 case OMP_CLAUSE_WORKER:
2049 case OMP_CLAUSE_VECTOR:
2050 break;
2052 case OMP_CLAUSE_DEVICE_RESIDENT:
2053 case OMP_CLAUSE_USE_DEVICE:
2054 case OMP_CLAUSE__CACHE_:
2055 case OMP_CLAUSE_INDEPENDENT:
2056 case OMP_CLAUSE_AUTO:
2057 case OMP_CLAUSE_SEQ:
2058 sorry ("Clause not supported yet");
2059 break;
2061 default:
2062 gcc_unreachable ();
2066 gcc_checking_assert (!scan_array_reductions
2067 || !is_gimple_omp_oacc (ctx->stmt));
2068 if (scan_array_reductions)
2069 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2070 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
2071 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2073 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx);
2074 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
2076 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
2077 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2078 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
2079 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2080 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2081 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
2084 /* Create a new name for omp child function. Returns an identifier. If
2085 IS_CILK_FOR is true then the suffix for the child function is
2086 "_cilk_for_fn." */
2088 static tree
2089 create_omp_child_function_name (bool task_copy, bool is_cilk_for)
2091 if (is_cilk_for)
2092 return clone_function_name (current_function_decl, "_cilk_for_fn");
2093 return clone_function_name (current_function_decl,
2094 task_copy ? "_omp_cpyfn" : "_omp_fn");
2097 /* Returns the type of the induction variable for the child function for
2098 _Cilk_for and the types for _high and _low variables based on TYPE. */
2100 static tree
2101 cilk_for_check_loop_diff_type (tree type)
2103 if (TYPE_PRECISION (type) <= TYPE_PRECISION (uint32_type_node))
2105 if (TYPE_UNSIGNED (type))
2106 return uint32_type_node;
2107 else
2108 return integer_type_node;
2110 else
2112 if (TYPE_UNSIGNED (type))
2113 return uint64_type_node;
2114 else
2115 return long_long_integer_type_node;
2119 /* Build a decl for the omp child function. It'll not contain a body
2120 yet, just the bare decl. */
2122 static void
2123 create_omp_child_function (omp_context *ctx, bool task_copy)
2125 tree decl, type, name, t;
2127 tree cilk_for_count
2128 = (flag_cilkplus && gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2129 ? find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
2130 OMP_CLAUSE__CILK_FOR_COUNT_) : NULL_TREE;
2131 tree cilk_var_type = NULL_TREE;
2133 name = create_omp_child_function_name (task_copy,
2134 cilk_for_count != NULL_TREE);
2135 if (task_copy)
2136 type = build_function_type_list (void_type_node, ptr_type_node,
2137 ptr_type_node, NULL_TREE);
2138 else if (cilk_for_count)
2140 type = TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count, 0));
2141 cilk_var_type = cilk_for_check_loop_diff_type (type);
2142 type = build_function_type_list (void_type_node, ptr_type_node,
2143 cilk_var_type, cilk_var_type, NULL_TREE);
2145 else
2146 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
2148 decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type);
2150 gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt)
2151 || !task_copy);
2152 if (!task_copy)
2153 ctx->cb.dst_fn = decl;
2154 else
2155 gimple_omp_task_set_copy_fn (ctx->stmt, decl);
2157 TREE_STATIC (decl) = 1;
2158 TREE_USED (decl) = 1;
2159 DECL_ARTIFICIAL (decl) = 1;
2160 DECL_IGNORED_P (decl) = 0;
2161 TREE_PUBLIC (decl) = 0;
2162 DECL_UNINLINABLE (decl) = 1;
2163 DECL_EXTERNAL (decl) = 0;
2164 DECL_CONTEXT (decl) = NULL_TREE;
2165 DECL_INITIAL (decl) = make_node (BLOCK);
2166 if (cgraph_node::get (current_function_decl)->offloadable)
2167 cgraph_node::get_create (decl)->offloadable = 1;
2168 else
2170 omp_context *octx;
2171 for (octx = ctx; octx; octx = octx->outer)
2172 if (is_gimple_omp_offloaded (octx->stmt))
2174 cgraph_node::get_create (decl)->offloadable = 1;
2175 #ifdef ENABLE_OFFLOADING
2176 g->have_offload = true;
2177 #endif
2178 break;
2182 if (cgraph_node::get_create (decl)->offloadable
2183 && !lookup_attribute ("omp declare target",
2184 DECL_ATTRIBUTES (current_function_decl)))
2185 DECL_ATTRIBUTES (decl)
2186 = tree_cons (get_identifier ("omp target entrypoint"),
2187 NULL_TREE, DECL_ATTRIBUTES (decl));
2189 t = build_decl (DECL_SOURCE_LOCATION (decl),
2190 RESULT_DECL, NULL_TREE, void_type_node);
2191 DECL_ARTIFICIAL (t) = 1;
2192 DECL_IGNORED_P (t) = 1;
2193 DECL_CONTEXT (t) = decl;
2194 DECL_RESULT (decl) = t;
2196 /* _Cilk_for's child function requires two extra parameters called
2197 __low and __high that are set the by Cilk runtime when it calls this
2198 function. */
2199 if (cilk_for_count)
2201 t = build_decl (DECL_SOURCE_LOCATION (decl),
2202 PARM_DECL, get_identifier ("__high"), cilk_var_type);
2203 DECL_ARTIFICIAL (t) = 1;
2204 DECL_NAMELESS (t) = 1;
2205 DECL_ARG_TYPE (t) = ptr_type_node;
2206 DECL_CONTEXT (t) = current_function_decl;
2207 TREE_USED (t) = 1;
2208 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2209 DECL_ARGUMENTS (decl) = t;
2211 t = build_decl (DECL_SOURCE_LOCATION (decl),
2212 PARM_DECL, get_identifier ("__low"), cilk_var_type);
2213 DECL_ARTIFICIAL (t) = 1;
2214 DECL_NAMELESS (t) = 1;
2215 DECL_ARG_TYPE (t) = ptr_type_node;
2216 DECL_CONTEXT (t) = current_function_decl;
2217 TREE_USED (t) = 1;
2218 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2219 DECL_ARGUMENTS (decl) = t;
2222 tree data_name = get_identifier (".omp_data_i");
2223 t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name,
2224 ptr_type_node);
2225 DECL_ARTIFICIAL (t) = 1;
2226 DECL_NAMELESS (t) = 1;
2227 DECL_ARG_TYPE (t) = ptr_type_node;
2228 DECL_CONTEXT (t) = current_function_decl;
2229 TREE_USED (t) = 1;
2230 if (cilk_for_count)
2231 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2232 DECL_ARGUMENTS (decl) = t;
2233 if (!task_copy)
2234 ctx->receiver_decl = t;
2235 else
2237 t = build_decl (DECL_SOURCE_LOCATION (decl),
2238 PARM_DECL, get_identifier (".omp_data_o"),
2239 ptr_type_node);
2240 DECL_ARTIFICIAL (t) = 1;
2241 DECL_NAMELESS (t) = 1;
2242 DECL_ARG_TYPE (t) = ptr_type_node;
2243 DECL_CONTEXT (t) = current_function_decl;
2244 TREE_USED (t) = 1;
2245 TREE_ADDRESSABLE (t) = 1;
2246 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2247 DECL_ARGUMENTS (decl) = t;
2250 /* Allocate memory for the function structure. The call to
2251 allocate_struct_function clobbers CFUN, so we need to restore
2252 it afterward. */
2253 push_struct_function (decl);
2254 cfun->function_end_locus = gimple_location (ctx->stmt);
2255 pop_cfun ();
2258 /* Callback for walk_gimple_seq. Check if combined parallel
2259 contains gimple_omp_for_combined_into_p OMP_FOR. */
2261 static tree
2262 find_combined_for (gimple_stmt_iterator *gsi_p,
2263 bool *handled_ops_p,
2264 struct walk_stmt_info *wi)
2266 gimple stmt = gsi_stmt (*gsi_p);
2268 *handled_ops_p = true;
2269 switch (gimple_code (stmt))
2271 WALK_SUBSTMTS;
2273 case GIMPLE_OMP_FOR:
2274 if (gimple_omp_for_combined_into_p (stmt)
2275 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
2277 wi->info = stmt;
2278 return integer_zero_node;
2280 break;
2281 default:
2282 break;
2284 return NULL;
2287 /* Scan an OpenMP parallel directive. */
2289 static void
2290 scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2292 omp_context *ctx;
2293 tree name;
2294 gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi));
2296 /* Ignore parallel directives with empty bodies, unless there
2297 are copyin clauses. */
2298 if (optimize > 0
2299 && empty_body_p (gimple_omp_body (stmt))
2300 && find_omp_clause (gimple_omp_parallel_clauses (stmt),
2301 OMP_CLAUSE_COPYIN) == NULL)
2303 gsi_replace (gsi, gimple_build_nop (), false);
2304 return;
2307 if (gimple_omp_parallel_combined_p (stmt))
2309 struct walk_stmt_info wi;
2311 memset (&wi, 0, sizeof (wi));
2312 wi.val_only = true;
2313 walk_gimple_seq (gimple_omp_body (stmt),
2314 find_combined_for, NULL, &wi);
2315 if (wi.info)
2317 gomp_for *for_stmt = as_a <gomp_for *> ((gimple) wi.info);
2318 struct omp_for_data fd;
2319 extract_omp_for_data (for_stmt, &fd, NULL);
2320 /* We need two temporaries with fd.loop.v type (istart/iend)
2321 and then (fd.collapse - 1) temporaries with the same
2322 type for count2 ... countN-1 vars if not constant. */
2323 size_t count = 2, i;
2324 tree type = fd.iter_type;
2325 if (fd.collapse > 1
2326 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
2327 count += fd.collapse - 1;
2328 for (i = 0; i < count; i++)
2330 tree temp = create_tmp_var (type);
2331 tree c = build_omp_clause (UNKNOWN_LOCATION,
2332 OMP_CLAUSE__LOOPTEMP_);
2333 insert_decl_map (&outer_ctx->cb, temp, temp);
2334 OMP_CLAUSE_DECL (c) = temp;
2335 OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
2336 gimple_omp_parallel_set_clauses (stmt, c);
2341 ctx = new_omp_context (stmt, outer_ctx);
2342 taskreg_contexts.safe_push (ctx);
2343 if (taskreg_nesting_level > 1)
2344 ctx->is_nested = true;
2345 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2346 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2347 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2348 name = create_tmp_var_name (".omp_data_s");
2349 name = build_decl (gimple_location (stmt),
2350 TYPE_DECL, name, ctx->record_type);
2351 DECL_ARTIFICIAL (name) = 1;
2352 DECL_NAMELESS (name) = 1;
2353 TYPE_NAME (ctx->record_type) = name;
2354 TYPE_ARTIFICIAL (ctx->record_type) = 1;
2355 create_omp_child_function (ctx, false);
2356 gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2358 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2359 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2361 if (TYPE_FIELDS (ctx->record_type) == NULL)
2362 ctx->record_type = ctx->receiver_decl = NULL;
2365 /* Scan an OpenMP task directive. */
2367 static void
2368 scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2370 omp_context *ctx;
2371 tree name, t;
2372 gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi));
2374 /* Ignore task directives with empty bodies. */
2375 if (optimize > 0
2376 && empty_body_p (gimple_omp_body (stmt)))
2378 gsi_replace (gsi, gimple_build_nop (), false);
2379 return;
2382 ctx = new_omp_context (stmt, outer_ctx);
2383 taskreg_contexts.safe_push (ctx);
2384 if (taskreg_nesting_level > 1)
2385 ctx->is_nested = true;
2386 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2387 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2388 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2389 name = create_tmp_var_name (".omp_data_s");
2390 name = build_decl (gimple_location (stmt),
2391 TYPE_DECL, name, ctx->record_type);
2392 DECL_ARTIFICIAL (name) = 1;
2393 DECL_NAMELESS (name) = 1;
2394 TYPE_NAME (ctx->record_type) = name;
2395 TYPE_ARTIFICIAL (ctx->record_type) = 1;
2396 create_omp_child_function (ctx, false);
2397 gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2399 scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2401 if (ctx->srecord_type)
2403 name = create_tmp_var_name (".omp_data_a");
2404 name = build_decl (gimple_location (stmt),
2405 TYPE_DECL, name, ctx->srecord_type);
2406 DECL_ARTIFICIAL (name) = 1;
2407 DECL_NAMELESS (name) = 1;
2408 TYPE_NAME (ctx->srecord_type) = name;
2409 TYPE_ARTIFICIAL (ctx->srecord_type) = 1;
2410 create_omp_child_function (ctx, true);
2413 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2415 if (TYPE_FIELDS (ctx->record_type) == NULL)
2417 ctx->record_type = ctx->receiver_decl = NULL;
2418 t = build_int_cst (long_integer_type_node, 0);
2419 gimple_omp_task_set_arg_size (stmt, t);
2420 t = build_int_cst (long_integer_type_node, 1);
2421 gimple_omp_task_set_arg_align (stmt, t);
2426 /* If any decls have been made addressable during scan_omp,
2427 adjust their fields if needed, and layout record types
2428 of parallel/task constructs. */
2430 static void
2431 finish_taskreg_scan (omp_context *ctx)
2433 if (ctx->record_type == NULL_TREE)
2434 return;
2436 /* If any task_shared_vars were needed, verify all
2437 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2438 statements if use_pointer_for_field hasn't changed
2439 because of that. If it did, update field types now. */
2440 if (task_shared_vars)
2442 tree c;
2444 for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2445 c; c = OMP_CLAUSE_CHAIN (c))
2446 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED)
2448 tree decl = OMP_CLAUSE_DECL (c);
2450 /* Global variables don't need to be copied,
2451 the receiver side will use them directly. */
2452 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2453 continue;
2454 if (!bitmap_bit_p (task_shared_vars, DECL_UID (decl))
2455 || !use_pointer_for_field (decl, ctx))
2456 continue;
2457 tree field = lookup_field (decl, ctx);
2458 if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE
2459 && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl))
2460 continue;
2461 TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl));
2462 TREE_THIS_VOLATILE (field) = 0;
2463 DECL_USER_ALIGN (field) = 0;
2464 DECL_ALIGN (field) = TYPE_ALIGN (TREE_TYPE (field));
2465 if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field))
2466 TYPE_ALIGN (ctx->record_type) = DECL_ALIGN (field);
2467 if (ctx->srecord_type)
2469 tree sfield = lookup_sfield (decl, ctx);
2470 TREE_TYPE (sfield) = TREE_TYPE (field);
2471 TREE_THIS_VOLATILE (sfield) = 0;
2472 DECL_USER_ALIGN (sfield) = 0;
2473 DECL_ALIGN (sfield) = DECL_ALIGN (field);
2474 if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield))
2475 TYPE_ALIGN (ctx->srecord_type) = DECL_ALIGN (sfield);
2480 if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2482 layout_type (ctx->record_type);
2483 fixup_child_record_type (ctx);
2485 else
2487 location_t loc = gimple_location (ctx->stmt);
2488 tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2489 /* Move VLA fields to the end. */
2490 p = &TYPE_FIELDS (ctx->record_type);
2491 while (*p)
2492 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2493 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2495 *q = *p;
2496 *p = TREE_CHAIN (*p);
2497 TREE_CHAIN (*q) = NULL_TREE;
2498 q = &TREE_CHAIN (*q);
2500 else
2501 p = &DECL_CHAIN (*p);
2502 *p = vla_fields;
2503 layout_type (ctx->record_type);
2504 fixup_child_record_type (ctx);
2505 if (ctx->srecord_type)
2506 layout_type (ctx->srecord_type);
2507 tree t = fold_convert_loc (loc, long_integer_type_node,
2508 TYPE_SIZE_UNIT (ctx->record_type));
2509 gimple_omp_task_set_arg_size (ctx->stmt, t);
2510 t = build_int_cst (long_integer_type_node,
2511 TYPE_ALIGN_UNIT (ctx->record_type));
2512 gimple_omp_task_set_arg_align (ctx->stmt, t);
2517 static omp_context *
2518 enclosing_target_ctx (omp_context *ctx)
2520 while (ctx != NULL
2521 && gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
2522 ctx = ctx->outer;
2523 gcc_assert (ctx != NULL);
2524 return ctx;
2527 static bool
2528 oacc_loop_or_target_p (gimple stmt)
2530 enum gimple_code outer_type = gimple_code (stmt);
2531 return ((outer_type == GIMPLE_OMP_TARGET
2532 && ((gimple_omp_target_kind (stmt)
2533 == GF_OMP_TARGET_KIND_OACC_PARALLEL)
2534 || (gimple_omp_target_kind (stmt)
2535 == GF_OMP_TARGET_KIND_OACC_KERNELS)))
2536 || (outer_type == GIMPLE_OMP_FOR
2537 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP));
2540 /* Scan a GIMPLE_OMP_FOR. */
2542 static void
2543 scan_omp_for (gomp_for *stmt, omp_context *outer_ctx)
2545 enum gimple_code outer_type = GIMPLE_ERROR_MARK;
2546 omp_context *ctx;
2547 size_t i;
2548 tree clauses = gimple_omp_for_clauses (stmt);
2550 if (outer_ctx)
2551 outer_type = gimple_code (outer_ctx->stmt);
2553 ctx = new_omp_context (stmt, outer_ctx);
2555 if (is_gimple_omp_oacc (stmt))
2557 if (outer_ctx && outer_type == GIMPLE_OMP_FOR)
2558 ctx->gwv_this = outer_ctx->gwv_this;
2559 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2561 int val;
2562 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_GANG)
2563 val = MASK_GANG;
2564 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WORKER)
2565 val = MASK_WORKER;
2566 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR)
2567 val = MASK_VECTOR;
2568 else
2569 continue;
2570 ctx->gwv_this |= val;
2571 if (!outer_ctx)
2573 /* Skip; not nested inside a region. */
2574 continue;
2576 if (!oacc_loop_or_target_p (outer_ctx->stmt))
2578 /* Skip; not nested inside an OpenACC region. */
2579 continue;
2581 if (outer_type == GIMPLE_OMP_FOR)
2582 outer_ctx->gwv_below |= val;
2583 if (OMP_CLAUSE_OPERAND (c, 0) != NULL_TREE)
2585 omp_context *enclosing = enclosing_target_ctx (outer_ctx);
2586 if (gimple_omp_target_kind (enclosing->stmt)
2587 == GF_OMP_TARGET_KIND_OACC_PARALLEL)
2588 error_at (gimple_location (stmt),
2589 "no arguments allowed to gang, worker and vector clauses inside parallel");
2594 scan_sharing_clauses (clauses, ctx);
2596 scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
2597 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
2599 scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
2600 scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
2601 scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
2602 scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
2604 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2606 if (is_gimple_omp_oacc (stmt))
2608 if (ctx->gwv_this & ctx->gwv_below)
2609 error_at (gimple_location (stmt),
2610 "gang, worker and vector may occur only once in a loop nest");
2611 else if (ctx->gwv_below != 0
2612 && ctx->gwv_this > ctx->gwv_below)
2613 error_at (gimple_location (stmt),
2614 "gang, worker and vector must occur in this order in a loop nest");
2615 if (outer_ctx && outer_type == GIMPLE_OMP_FOR)
2616 outer_ctx->gwv_below |= ctx->gwv_below;
2620 /* Scan an OpenMP sections directive. */
2622 static void
2623 scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx)
2625 omp_context *ctx;
2627 ctx = new_omp_context (stmt, outer_ctx);
2628 scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
2629 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2632 /* Scan an OpenMP single directive. */
2634 static void
2635 scan_omp_single (gomp_single *stmt, omp_context *outer_ctx)
2637 omp_context *ctx;
2638 tree name;
2640 ctx = new_omp_context (stmt, outer_ctx);
2641 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2642 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2643 name = create_tmp_var_name (".omp_copy_s");
2644 name = build_decl (gimple_location (stmt),
2645 TYPE_DECL, name, ctx->record_type);
2646 TYPE_NAME (ctx->record_type) = name;
2648 scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
2649 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2651 if (TYPE_FIELDS (ctx->record_type) == NULL)
2652 ctx->record_type = NULL;
2653 else
2654 layout_type (ctx->record_type);
2657 /* Scan a GIMPLE_OMP_TARGET. */
2659 static void
2660 scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
2662 omp_context *ctx;
2663 tree name;
2664 bool offloaded = is_gimple_omp_offloaded (stmt);
2665 tree clauses = gimple_omp_target_clauses (stmt);
2667 ctx = new_omp_context (stmt, outer_ctx);
2668 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2669 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2670 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2671 name = create_tmp_var_name (".omp_data_t");
2672 name = build_decl (gimple_location (stmt),
2673 TYPE_DECL, name, ctx->record_type);
2674 DECL_ARTIFICIAL (name) = 1;
2675 DECL_NAMELESS (name) = 1;
2676 TYPE_NAME (ctx->record_type) = name;
2677 TYPE_ARTIFICIAL (ctx->record_type) = 1;
2678 if (offloaded)
2680 if (is_gimple_omp_oacc (stmt))
2681 ctx->reduction_map = splay_tree_new (splay_tree_compare_pointers,
2682 0, 0);
2684 create_omp_child_function (ctx, false);
2685 gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
2688 if (is_gimple_omp_oacc (stmt))
2690 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2692 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_GANGS)
2693 ctx->gwv_this |= MASK_GANG;
2694 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_WORKERS)
2695 ctx->gwv_this |= MASK_WORKER;
2696 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR_LENGTH)
2697 ctx->gwv_this |= MASK_VECTOR;
2701 scan_sharing_clauses (clauses, ctx);
2702 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2704 if (TYPE_FIELDS (ctx->record_type) == NULL)
2705 ctx->record_type = ctx->receiver_decl = NULL;
2706 else
2708 TYPE_FIELDS (ctx->record_type)
2709 = nreverse (TYPE_FIELDS (ctx->record_type));
2710 #ifdef ENABLE_CHECKING
2711 tree field;
2712 unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
2713 for (field = TYPE_FIELDS (ctx->record_type);
2714 field;
2715 field = DECL_CHAIN (field))
2716 gcc_assert (DECL_ALIGN (field) == align);
2717 #endif
2718 layout_type (ctx->record_type);
2719 if (offloaded)
2720 fixup_child_record_type (ctx);
2724 /* Scan an OpenMP teams directive. */
2726 static void
2727 scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
2729 omp_context *ctx = new_omp_context (stmt, outer_ctx);
2730 scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
2731 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2734 /* Check nesting restrictions. */
2735 static bool
2736 check_omp_nesting_restrictions (gimple stmt, omp_context *ctx)
2738 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2739 inside an OpenACC CTX. */
2740 if (!(is_gimple_omp (stmt)
2741 && is_gimple_omp_oacc (stmt)))
2743 for (omp_context *ctx_ = ctx; ctx_ != NULL; ctx_ = ctx_->outer)
2744 if (is_gimple_omp (ctx_->stmt)
2745 && is_gimple_omp_oacc (ctx_->stmt))
2747 error_at (gimple_location (stmt),
2748 "non-OpenACC construct inside of OpenACC region");
2749 return false;
2753 if (ctx != NULL)
2755 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
2756 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
2758 error_at (gimple_location (stmt),
2759 "OpenMP constructs may not be nested inside simd region");
2760 return false;
2762 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2764 if ((gimple_code (stmt) != GIMPLE_OMP_FOR
2765 || (gimple_omp_for_kind (stmt)
2766 != GF_OMP_FOR_KIND_DISTRIBUTE))
2767 && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
2769 error_at (gimple_location (stmt),
2770 "only distribute or parallel constructs are allowed to "
2771 "be closely nested inside teams construct");
2772 return false;
2776 switch (gimple_code (stmt))
2778 case GIMPLE_OMP_FOR:
2779 if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_SIMD)
2780 return true;
2781 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
2783 if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
2785 error_at (gimple_location (stmt),
2786 "distribute construct must be closely nested inside "
2787 "teams construct");
2788 return false;
2790 return true;
2792 /* FALLTHRU */
2793 case GIMPLE_CALL:
2794 if (is_gimple_call (stmt)
2795 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2796 == BUILT_IN_GOMP_CANCEL
2797 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2798 == BUILT_IN_GOMP_CANCELLATION_POINT))
2800 const char *bad = NULL;
2801 const char *kind = NULL;
2802 if (ctx == NULL)
2804 error_at (gimple_location (stmt), "orphaned %qs construct",
2805 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2806 == BUILT_IN_GOMP_CANCEL
2807 ? "#pragma omp cancel"
2808 : "#pragma omp cancellation point");
2809 return false;
2811 switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
2812 ? tree_to_shwi (gimple_call_arg (stmt, 0))
2813 : 0)
2815 case 1:
2816 if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
2817 bad = "#pragma omp parallel";
2818 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2819 == BUILT_IN_GOMP_CANCEL
2820 && !integer_zerop (gimple_call_arg (stmt, 1)))
2821 ctx->cancellable = true;
2822 kind = "parallel";
2823 break;
2824 case 2:
2825 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
2826 || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
2827 bad = "#pragma omp for";
2828 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2829 == BUILT_IN_GOMP_CANCEL
2830 && !integer_zerop (gimple_call_arg (stmt, 1)))
2832 ctx->cancellable = true;
2833 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2834 OMP_CLAUSE_NOWAIT))
2835 warning_at (gimple_location (stmt), 0,
2836 "%<#pragma omp cancel for%> inside "
2837 "%<nowait%> for construct");
2838 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2839 OMP_CLAUSE_ORDERED))
2840 warning_at (gimple_location (stmt), 0,
2841 "%<#pragma omp cancel for%> inside "
2842 "%<ordered%> for construct");
2844 kind = "for";
2845 break;
2846 case 4:
2847 if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
2848 && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
2849 bad = "#pragma omp sections";
2850 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2851 == BUILT_IN_GOMP_CANCEL
2852 && !integer_zerop (gimple_call_arg (stmt, 1)))
2854 if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
2856 ctx->cancellable = true;
2857 if (find_omp_clause (gimple_omp_sections_clauses
2858 (ctx->stmt),
2859 OMP_CLAUSE_NOWAIT))
2860 warning_at (gimple_location (stmt), 0,
2861 "%<#pragma omp cancel sections%> inside "
2862 "%<nowait%> sections construct");
2864 else
2866 gcc_assert (ctx->outer
2867 && gimple_code (ctx->outer->stmt)
2868 == GIMPLE_OMP_SECTIONS);
2869 ctx->outer->cancellable = true;
2870 if (find_omp_clause (gimple_omp_sections_clauses
2871 (ctx->outer->stmt),
2872 OMP_CLAUSE_NOWAIT))
2873 warning_at (gimple_location (stmt), 0,
2874 "%<#pragma omp cancel sections%> inside "
2875 "%<nowait%> sections construct");
2878 kind = "sections";
2879 break;
2880 case 8:
2881 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK)
2882 bad = "#pragma omp task";
2883 else
2884 ctx->cancellable = true;
2885 kind = "taskgroup";
2886 break;
2887 default:
2888 error_at (gimple_location (stmt), "invalid arguments");
2889 return false;
2891 if (bad)
2893 error_at (gimple_location (stmt),
2894 "%<%s %s%> construct not closely nested inside of %qs",
2895 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2896 == BUILT_IN_GOMP_CANCEL
2897 ? "#pragma omp cancel"
2898 : "#pragma omp cancellation point", kind, bad);
2899 return false;
2902 /* FALLTHRU */
2903 case GIMPLE_OMP_SECTIONS:
2904 case GIMPLE_OMP_SINGLE:
2905 for (; ctx != NULL; ctx = ctx->outer)
2906 switch (gimple_code (ctx->stmt))
2908 case GIMPLE_OMP_FOR:
2909 case GIMPLE_OMP_SECTIONS:
2910 case GIMPLE_OMP_SINGLE:
2911 case GIMPLE_OMP_ORDERED:
2912 case GIMPLE_OMP_MASTER:
2913 case GIMPLE_OMP_TASK:
2914 case GIMPLE_OMP_CRITICAL:
2915 if (is_gimple_call (stmt))
2917 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2918 != BUILT_IN_GOMP_BARRIER)
2919 return true;
2920 error_at (gimple_location (stmt),
2921 "barrier region may not be closely nested inside "
2922 "of work-sharing, critical, ordered, master or "
2923 "explicit task region");
2924 return false;
2926 error_at (gimple_location (stmt),
2927 "work-sharing region may not be closely nested inside "
2928 "of work-sharing, critical, ordered, master or explicit "
2929 "task region");
2930 return false;
2931 case GIMPLE_OMP_PARALLEL:
2932 return true;
2933 default:
2934 break;
2936 break;
2937 case GIMPLE_OMP_MASTER:
2938 for (; ctx != NULL; ctx = ctx->outer)
2939 switch (gimple_code (ctx->stmt))
2941 case GIMPLE_OMP_FOR:
2942 case GIMPLE_OMP_SECTIONS:
2943 case GIMPLE_OMP_SINGLE:
2944 case GIMPLE_OMP_TASK:
2945 error_at (gimple_location (stmt),
2946 "master region may not be closely nested inside "
2947 "of work-sharing or explicit task region");
2948 return false;
2949 case GIMPLE_OMP_PARALLEL:
2950 return true;
2951 default:
2952 break;
2954 break;
2955 case GIMPLE_OMP_ORDERED:
2956 for (; ctx != NULL; ctx = ctx->outer)
2957 switch (gimple_code (ctx->stmt))
2959 case GIMPLE_OMP_CRITICAL:
2960 case GIMPLE_OMP_TASK:
2961 error_at (gimple_location (stmt),
2962 "ordered region may not be closely nested inside "
2963 "of critical or explicit task region");
2964 return false;
2965 case GIMPLE_OMP_FOR:
2966 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2967 OMP_CLAUSE_ORDERED) == NULL)
2969 error_at (gimple_location (stmt),
2970 "ordered region must be closely nested inside "
2971 "a loop region with an ordered clause");
2972 return false;
2974 return true;
2975 case GIMPLE_OMP_PARALLEL:
2976 error_at (gimple_location (stmt),
2977 "ordered region must be closely nested inside "
2978 "a loop region with an ordered clause");
2979 return false;
2980 default:
2981 break;
2983 break;
2984 case GIMPLE_OMP_CRITICAL:
2986 tree this_stmt_name
2987 = gimple_omp_critical_name (as_a <gomp_critical *> (stmt));
2988 for (; ctx != NULL; ctx = ctx->outer)
2989 if (gomp_critical *other_crit
2990 = dyn_cast <gomp_critical *> (ctx->stmt))
2991 if (this_stmt_name == gimple_omp_critical_name (other_crit))
2993 error_at (gimple_location (stmt),
2994 "critical region may not be nested inside a critical "
2995 "region with the same name");
2996 return false;
2999 break;
3000 case GIMPLE_OMP_TEAMS:
3001 if (ctx == NULL
3002 || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
3003 || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION)
3005 error_at (gimple_location (stmt),
3006 "teams construct not closely nested inside of target "
3007 "region");
3008 return false;
3010 break;
3011 case GIMPLE_OMP_TARGET:
3012 for (; ctx != NULL; ctx = ctx->outer)
3014 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
3016 if (is_gimple_omp (stmt)
3017 && is_gimple_omp_oacc (stmt)
3018 && is_gimple_omp (ctx->stmt))
3020 error_at (gimple_location (stmt),
3021 "OpenACC construct inside of non-OpenACC region");
3022 return false;
3024 continue;
3027 const char *stmt_name, *ctx_stmt_name;
3028 switch (gimple_omp_target_kind (stmt))
3030 case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break;
3031 case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break;
3032 case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break;
3033 case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break;
3034 case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break;
3035 case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break;
3036 case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break;
3037 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: stmt_name = "enter/exit data"; break;
3038 default: gcc_unreachable ();
3040 switch (gimple_omp_target_kind (ctx->stmt))
3042 case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break;
3043 case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break;
3044 case GF_OMP_TARGET_KIND_OACC_PARALLEL: ctx_stmt_name = "parallel"; break;
3045 case GF_OMP_TARGET_KIND_OACC_KERNELS: ctx_stmt_name = "kernels"; break;
3046 case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break;
3047 default: gcc_unreachable ();
3050 /* OpenACC/OpenMP mismatch? */
3051 if (is_gimple_omp_oacc (stmt)
3052 != is_gimple_omp_oacc (ctx->stmt))
3054 error_at (gimple_location (stmt),
3055 "%s %s construct inside of %s %s region",
3056 (is_gimple_omp_oacc (stmt)
3057 ? "OpenACC" : "OpenMP"), stmt_name,
3058 (is_gimple_omp_oacc (ctx->stmt)
3059 ? "OpenACC" : "OpenMP"), ctx_stmt_name);
3060 return false;
3062 if (is_gimple_omp_offloaded (ctx->stmt))
3064 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3065 if (is_gimple_omp_oacc (ctx->stmt))
3067 error_at (gimple_location (stmt),
3068 "%s construct inside of %s region",
3069 stmt_name, ctx_stmt_name);
3070 return false;
3072 else
3074 gcc_checking_assert (!is_gimple_omp_oacc (stmt));
3075 warning_at (gimple_location (stmt), 0,
3076 "%s construct inside of %s region",
3077 stmt_name, ctx_stmt_name);
3081 break;
3082 default:
3083 break;
3085 return true;
3089 /* Helper function scan_omp.
3091 Callback for walk_tree or operators in walk_gimple_stmt used to
3092 scan for OMP directives in TP. */
3094 static tree
3095 scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
3097 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
3098 omp_context *ctx = (omp_context *) wi->info;
3099 tree t = *tp;
3101 switch (TREE_CODE (t))
3103 case VAR_DECL:
3104 case PARM_DECL:
3105 case LABEL_DECL:
3106 case RESULT_DECL:
3107 if (ctx)
3108 *tp = remap_decl (t, &ctx->cb);
3109 break;
3111 default:
3112 if (ctx && TYPE_P (t))
3113 *tp = remap_type (t, &ctx->cb);
3114 else if (!DECL_P (t))
3116 *walk_subtrees = 1;
3117 if (ctx)
3119 tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
3120 if (tem != TREE_TYPE (t))
3122 if (TREE_CODE (t) == INTEGER_CST)
3123 *tp = wide_int_to_tree (tem, t);
3124 else
3125 TREE_TYPE (t) = tem;
3129 break;
3132 return NULL_TREE;
3135 /* Return true if FNDECL is a setjmp or a longjmp. */
3137 static bool
3138 setjmp_or_longjmp_p (const_tree fndecl)
3140 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
3141 && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP
3142 || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP))
3143 return true;
3145 tree declname = DECL_NAME (fndecl);
3146 if (!declname)
3147 return false;
3148 const char *name = IDENTIFIER_POINTER (declname);
3149 return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
3153 /* Helper function for scan_omp.
3155 Callback for walk_gimple_stmt used to scan for OMP directives in
3156 the current statement in GSI. */
3158 static tree
3159 scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
3160 struct walk_stmt_info *wi)
3162 gimple stmt = gsi_stmt (*gsi);
3163 omp_context *ctx = (omp_context *) wi->info;
3165 if (gimple_has_location (stmt))
3166 input_location = gimple_location (stmt);
3168 /* Check the nesting restrictions. */
3169 bool remove = false;
3170 if (is_gimple_omp (stmt))
3171 remove = !check_omp_nesting_restrictions (stmt, ctx);
3172 else if (is_gimple_call (stmt))
3174 tree fndecl = gimple_call_fndecl (stmt);
3175 if (fndecl)
3177 if (setjmp_or_longjmp_p (fndecl)
3178 && ctx
3179 && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3180 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
3182 remove = true;
3183 error_at (gimple_location (stmt),
3184 "setjmp/longjmp inside simd construct");
3186 else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
3187 switch (DECL_FUNCTION_CODE (fndecl))
3189 case BUILT_IN_GOMP_BARRIER:
3190 case BUILT_IN_GOMP_CANCEL:
3191 case BUILT_IN_GOMP_CANCELLATION_POINT:
3192 case BUILT_IN_GOMP_TASKYIELD:
3193 case BUILT_IN_GOMP_TASKWAIT:
3194 case BUILT_IN_GOMP_TASKGROUP_START:
3195 case BUILT_IN_GOMP_TASKGROUP_END:
3196 remove = !check_omp_nesting_restrictions (stmt, ctx);
3197 break;
3198 default:
3199 break;
3203 if (remove)
3205 stmt = gimple_build_nop ();
3206 gsi_replace (gsi, stmt, false);
3209 *handled_ops_p = true;
3211 switch (gimple_code (stmt))
3213 case GIMPLE_OMP_PARALLEL:
3214 taskreg_nesting_level++;
3215 scan_omp_parallel (gsi, ctx);
3216 taskreg_nesting_level--;
3217 break;
3219 case GIMPLE_OMP_TASK:
3220 taskreg_nesting_level++;
3221 scan_omp_task (gsi, ctx);
3222 taskreg_nesting_level--;
3223 break;
3225 case GIMPLE_OMP_FOR:
3226 scan_omp_for (as_a <gomp_for *> (stmt), ctx);
3227 break;
3229 case GIMPLE_OMP_SECTIONS:
3230 scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
3231 break;
3233 case GIMPLE_OMP_SINGLE:
3234 scan_omp_single (as_a <gomp_single *> (stmt), ctx);
3235 break;
3237 case GIMPLE_OMP_SECTION:
3238 case GIMPLE_OMP_MASTER:
3239 case GIMPLE_OMP_TASKGROUP:
3240 case GIMPLE_OMP_ORDERED:
3241 case GIMPLE_OMP_CRITICAL:
3242 ctx = new_omp_context (stmt, ctx);
3243 scan_omp (gimple_omp_body_ptr (stmt), ctx);
3244 break;
3246 case GIMPLE_OMP_TARGET:
3247 scan_omp_target (as_a <gomp_target *> (stmt), ctx);
3248 break;
3250 case GIMPLE_OMP_TEAMS:
3251 scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
3252 break;
3254 case GIMPLE_BIND:
3256 tree var;
3258 *handled_ops_p = false;
3259 if (ctx)
3260 for (var = gimple_bind_vars (as_a <gbind *> (stmt));
3261 var ;
3262 var = DECL_CHAIN (var))
3263 insert_decl_map (&ctx->cb, var, var);
3265 break;
3266 default:
3267 *handled_ops_p = false;
3268 break;
3271 return NULL_TREE;
3275 /* Scan all the statements starting at the current statement. CTX
3276 contains context information about the OMP directives and
3277 clauses found during the scan. */
3279 static void
3280 scan_omp (gimple_seq *body_p, omp_context *ctx)
3282 location_t saved_location;
3283 struct walk_stmt_info wi;
3285 memset (&wi, 0, sizeof (wi));
3286 wi.info = ctx;
3287 wi.want_locations = true;
3289 saved_location = input_location;
3290 walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
3291 input_location = saved_location;
3294 /* Re-gimplification and code generation routines. */
3296 /* Build a call to GOMP_barrier. */
3298 static gimple
3299 build_omp_barrier (tree lhs)
3301 tree fndecl = builtin_decl_explicit (lhs ? BUILT_IN_GOMP_BARRIER_CANCEL
3302 : BUILT_IN_GOMP_BARRIER);
3303 gcall *g = gimple_build_call (fndecl, 0);
3304 if (lhs)
3305 gimple_call_set_lhs (g, lhs);
3306 return g;
3309 /* If a context was created for STMT when it was scanned, return it. */
3311 static omp_context *
3312 maybe_lookup_ctx (gimple stmt)
3314 splay_tree_node n;
3315 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
3316 return n ? (omp_context *) n->value : NULL;
3320 /* Find the mapping for DECL in CTX or the immediately enclosing
3321 context that has a mapping for DECL.
3323 If CTX is a nested parallel directive, we may have to use the decl
3324 mappings created in CTX's parent context. Suppose that we have the
3325 following parallel nesting (variable UIDs showed for clarity):
3327 iD.1562 = 0;
3328 #omp parallel shared(iD.1562) -> outer parallel
3329 iD.1562 = iD.1562 + 1;
3331 #omp parallel shared (iD.1562) -> inner parallel
3332 iD.1562 = iD.1562 - 1;
3334 Each parallel structure will create a distinct .omp_data_s structure
3335 for copying iD.1562 in/out of the directive:
3337 outer parallel .omp_data_s.1.i -> iD.1562
3338 inner parallel .omp_data_s.2.i -> iD.1562
3340 A shared variable mapping will produce a copy-out operation before
3341 the parallel directive and a copy-in operation after it. So, in
3342 this case we would have:
3344 iD.1562 = 0;
3345 .omp_data_o.1.i = iD.1562;
3346 #omp parallel shared(iD.1562) -> outer parallel
3347 .omp_data_i.1 = &.omp_data_o.1
3348 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3350 .omp_data_o.2.i = iD.1562; -> **
3351 #omp parallel shared(iD.1562) -> inner parallel
3352 .omp_data_i.2 = &.omp_data_o.2
3353 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3356 ** This is a problem. The symbol iD.1562 cannot be referenced
3357 inside the body of the outer parallel region. But since we are
3358 emitting this copy operation while expanding the inner parallel
3359 directive, we need to access the CTX structure of the outer
3360 parallel directive to get the correct mapping:
3362 .omp_data_o.2.i = .omp_data_i.1->i
3364 Since there may be other workshare or parallel directives enclosing
3365 the parallel directive, it may be necessary to walk up the context
3366 parent chain. This is not a problem in general because nested
3367 parallelism happens only rarely. */
3369 static tree
3370 lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
3372 tree t;
3373 omp_context *up;
3375 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
3376 t = maybe_lookup_decl (decl, up);
3378 gcc_assert (!ctx->is_nested || t || is_global_var (decl));
3380 return t ? t : decl;
3384 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3385 in outer contexts. */
3387 static tree
3388 maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
3390 tree t = NULL;
3391 omp_context *up;
3393 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
3394 t = maybe_lookup_decl (decl, up);
3396 return t ? t : decl;
3400 /* Construct the initialization value for reduction CLAUSE. */
3402 tree
3403 omp_reduction_init (tree clause, tree type)
3405 location_t loc = OMP_CLAUSE_LOCATION (clause);
3406 switch (OMP_CLAUSE_REDUCTION_CODE (clause))
3408 case PLUS_EXPR:
3409 case MINUS_EXPR:
3410 case BIT_IOR_EXPR:
3411 case BIT_XOR_EXPR:
3412 case TRUTH_OR_EXPR:
3413 case TRUTH_ORIF_EXPR:
3414 case TRUTH_XOR_EXPR:
3415 case NE_EXPR:
3416 return build_zero_cst (type);
3418 case MULT_EXPR:
3419 case TRUTH_AND_EXPR:
3420 case TRUTH_ANDIF_EXPR:
3421 case EQ_EXPR:
3422 return fold_convert_loc (loc, type, integer_one_node);
3424 case BIT_AND_EXPR:
3425 return fold_convert_loc (loc, type, integer_minus_one_node);
3427 case MAX_EXPR:
3428 if (SCALAR_FLOAT_TYPE_P (type))
3430 REAL_VALUE_TYPE max, min;
3431 if (HONOR_INFINITIES (type))
3433 real_inf (&max);
3434 real_arithmetic (&min, NEGATE_EXPR, &max, NULL);
3436 else
3437 real_maxval (&min, 1, TYPE_MODE (type));
3438 return build_real (type, min);
3440 else
3442 gcc_assert (INTEGRAL_TYPE_P (type));
3443 return TYPE_MIN_VALUE (type);
3446 case MIN_EXPR:
3447 if (SCALAR_FLOAT_TYPE_P (type))
3449 REAL_VALUE_TYPE max;
3450 if (HONOR_INFINITIES (type))
3451 real_inf (&max);
3452 else
3453 real_maxval (&max, 0, TYPE_MODE (type));
3454 return build_real (type, max);
3456 else
3458 gcc_assert (INTEGRAL_TYPE_P (type));
3459 return TYPE_MAX_VALUE (type);
3462 default:
3463 gcc_unreachable ();
3467 /* Return alignment to be assumed for var in CLAUSE, which should be
3468 OMP_CLAUSE_ALIGNED. */
3470 static tree
3471 omp_clause_aligned_alignment (tree clause)
3473 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
3474 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
3476 /* Otherwise return implementation defined alignment. */
3477 unsigned int al = 1;
3478 machine_mode mode, vmode;
3479 int vs = targetm.vectorize.autovectorize_vector_sizes ();
3480 if (vs)
3481 vs = 1 << floor_log2 (vs);
3482 static enum mode_class classes[]
3483 = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
3484 for (int i = 0; i < 4; i += 2)
3485 for (mode = GET_CLASS_NARROWEST_MODE (classes[i]);
3486 mode != VOIDmode;
3487 mode = GET_MODE_WIDER_MODE (mode))
3489 vmode = targetm.vectorize.preferred_simd_mode (mode);
3490 if (GET_MODE_CLASS (vmode) != classes[i + 1])
3491 continue;
3492 while (vs
3493 && GET_MODE_SIZE (vmode) < vs
3494 && GET_MODE_2XWIDER_MODE (vmode) != VOIDmode)
3495 vmode = GET_MODE_2XWIDER_MODE (vmode);
3497 tree type = lang_hooks.types.type_for_mode (mode, 1);
3498 if (type == NULL_TREE || TYPE_MODE (type) != mode)
3499 continue;
3500 type = build_vector_type (type, GET_MODE_SIZE (vmode)
3501 / GET_MODE_SIZE (mode));
3502 if (TYPE_MODE (type) != vmode)
3503 continue;
3504 if (TYPE_ALIGN_UNIT (type) > al)
3505 al = TYPE_ALIGN_UNIT (type);
3507 return build_int_cst (integer_type_node, al);
3510 /* Return maximum possible vectorization factor for the target. */
3512 static int
3513 omp_max_vf (void)
3515 if (!optimize
3516 || optimize_debug
3517 || !flag_tree_loop_optimize
3518 || (!flag_tree_loop_vectorize
3519 && (global_options_set.x_flag_tree_loop_vectorize
3520 || global_options_set.x_flag_tree_vectorize)))
3521 return 1;
3523 int vs = targetm.vectorize.autovectorize_vector_sizes ();
3524 if (vs)
3526 vs = 1 << floor_log2 (vs);
3527 return vs;
3529 machine_mode vqimode = targetm.vectorize.preferred_simd_mode (QImode);
3530 if (GET_MODE_CLASS (vqimode) == MODE_VECTOR_INT)
3531 return GET_MODE_NUNITS (vqimode);
3532 return 1;
3535 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3536 privatization. */
3538 static bool
3539 lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, int &max_vf,
3540 tree &idx, tree &lane, tree &ivar, tree &lvar)
3542 if (max_vf == 0)
3544 max_vf = omp_max_vf ();
3545 if (max_vf > 1)
3547 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3548 OMP_CLAUSE_SAFELEN);
3549 if (c && TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) != INTEGER_CST)
3550 max_vf = 1;
3551 else if (c && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
3552 max_vf) == -1)
3553 max_vf = tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c));
3555 if (max_vf > 1)
3557 idx = create_tmp_var (unsigned_type_node);
3558 lane = create_tmp_var (unsigned_type_node);
3561 if (max_vf == 1)
3562 return false;
3564 tree atype = build_array_type_nelts (TREE_TYPE (new_var), max_vf);
3565 tree avar = create_tmp_var_raw (atype);
3566 if (TREE_ADDRESSABLE (new_var))
3567 TREE_ADDRESSABLE (avar) = 1;
3568 DECL_ATTRIBUTES (avar)
3569 = tree_cons (get_identifier ("omp simd array"), NULL,
3570 DECL_ATTRIBUTES (avar));
3571 gimple_add_tmp_var (avar);
3572 ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, idx,
3573 NULL_TREE, NULL_TREE);
3574 lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, lane,
3575 NULL_TREE, NULL_TREE);
3576 if (DECL_P (new_var))
3578 SET_DECL_VALUE_EXPR (new_var, lvar);
3579 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3581 return true;
3584 /* Helper function of lower_rec_input_clauses. For a reference
3585 in simd reduction, add an underlying variable it will reference. */
3587 static void
3588 handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
3590 tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
3591 if (TREE_CONSTANT (z))
3593 const char *name = NULL;
3594 if (DECL_NAME (new_vard))
3595 name = IDENTIFIER_POINTER (DECL_NAME (new_vard));
3597 z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)), name);
3598 gimple_add_tmp_var (z);
3599 TREE_ADDRESSABLE (z) = 1;
3600 z = build_fold_addr_expr_loc (loc, z);
3601 gimplify_assign (new_vard, z, ilist);
3605 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3606 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3607 private variables. Initialization statements go in ILIST, while calls
3608 to destructors go in DLIST. */
3610 static void
3611 lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
3612 omp_context *ctx, struct omp_for_data *fd)
3614 tree c, dtor, copyin_seq, x, ptr;
3615 bool copyin_by_ref = false;
3616 bool lastprivate_firstprivate = false;
3617 bool reduction_omp_orig_ref = false;
3618 int pass;
3619 bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3620 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD);
3621 int max_vf = 0;
3622 tree lane = NULL_TREE, idx = NULL_TREE;
3623 tree ivar = NULL_TREE, lvar = NULL_TREE;
3624 gimple_seq llist[2] = { NULL, NULL };
3626 copyin_seq = NULL;
3628 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3629 with data sharing clauses referencing variable sized vars. That
3630 is unnecessarily hard to support and very unlikely to result in
3631 vectorized code anyway. */
3632 if (is_simd)
3633 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3634 switch (OMP_CLAUSE_CODE (c))
3636 case OMP_CLAUSE_LINEAR:
3637 if (OMP_CLAUSE_LINEAR_ARRAY (c))
3638 max_vf = 1;
3639 /* FALLTHRU */
3640 case OMP_CLAUSE_REDUCTION:
3641 case OMP_CLAUSE_PRIVATE:
3642 case OMP_CLAUSE_FIRSTPRIVATE:
3643 case OMP_CLAUSE_LASTPRIVATE:
3644 if (is_variable_sized (OMP_CLAUSE_DECL (c)))
3645 max_vf = 1;
3646 break;
3647 default:
3648 continue;
3651 /* Do all the fixed sized types in the first pass, and the variable sized
3652 types in the second pass. This makes sure that the scalar arguments to
3653 the variable sized types are processed before we use them in the
3654 variable sized operations. */
3655 for (pass = 0; pass < 2; ++pass)
3657 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3659 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
3660 tree var, new_var;
3661 bool by_ref;
3662 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3664 switch (c_kind)
3666 case OMP_CLAUSE_PRIVATE:
3667 if (OMP_CLAUSE_PRIVATE_DEBUG (c))
3668 continue;
3669 break;
3670 case OMP_CLAUSE_SHARED:
3671 /* Ignore shared directives in teams construct. */
3672 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3673 continue;
3674 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
3676 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c)));
3677 continue;
3679 case OMP_CLAUSE_FIRSTPRIVATE:
3680 case OMP_CLAUSE_COPYIN:
3681 case OMP_CLAUSE_LINEAR:
3682 break;
3683 case OMP_CLAUSE_REDUCTION:
3684 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
3685 reduction_omp_orig_ref = true;
3686 break;
3687 case OMP_CLAUSE__LOOPTEMP_:
3688 /* Handle _looptemp_ clauses only on parallel. */
3689 if (fd)
3690 continue;
3691 break;
3692 case OMP_CLAUSE_LASTPRIVATE:
3693 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3695 lastprivate_firstprivate = true;
3696 if (pass != 0)
3697 continue;
3699 /* Even without corresponding firstprivate, if
3700 decl is Fortran allocatable, it needs outer var
3701 reference. */
3702 else if (pass == 0
3703 && lang_hooks.decls.omp_private_outer_ref
3704 (OMP_CLAUSE_DECL (c)))
3705 lastprivate_firstprivate = true;
3706 break;
3707 case OMP_CLAUSE_ALIGNED:
3708 if (pass == 0)
3709 continue;
3710 var = OMP_CLAUSE_DECL (c);
3711 if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
3712 && !is_global_var (var))
3714 new_var = maybe_lookup_decl (var, ctx);
3715 if (new_var == NULL_TREE)
3716 new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
3717 x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3718 x = build_call_expr_loc (clause_loc, x, 2, new_var,
3719 omp_clause_aligned_alignment (c));
3720 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3721 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
3722 gimplify_and_add (x, ilist);
3724 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
3725 && is_global_var (var))
3727 tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
3728 new_var = lookup_decl (var, ctx);
3729 t = maybe_lookup_decl_in_outer_ctx (var, ctx);
3730 t = build_fold_addr_expr_loc (clause_loc, t);
3731 t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3732 t = build_call_expr_loc (clause_loc, t2, 2, t,
3733 omp_clause_aligned_alignment (c));
3734 t = fold_convert_loc (clause_loc, ptype, t);
3735 x = create_tmp_var (ptype);
3736 t = build2 (MODIFY_EXPR, ptype, x, t);
3737 gimplify_and_add (t, ilist);
3738 t = build_simple_mem_ref_loc (clause_loc, x);
3739 SET_DECL_VALUE_EXPR (new_var, t);
3740 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3742 continue;
3743 default:
3744 continue;
3747 new_var = var = OMP_CLAUSE_DECL (c);
3748 if (c_kind != OMP_CLAUSE_COPYIN)
3749 new_var = lookup_decl (var, ctx);
3751 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
3753 if (pass != 0)
3754 continue;
3756 else if (is_variable_sized (var))
3758 /* For variable sized types, we need to allocate the
3759 actual storage here. Call alloca and store the
3760 result in the pointer decl that we created elsewhere. */
3761 if (pass == 0)
3762 continue;
3764 if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
3766 gcall *stmt;
3767 tree tmp, atmp;
3769 ptr = DECL_VALUE_EXPR (new_var);
3770 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
3771 ptr = TREE_OPERAND (ptr, 0);
3772 gcc_assert (DECL_P (ptr));
3773 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
3775 /* void *tmp = __builtin_alloca */
3776 atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3777 stmt = gimple_build_call (atmp, 1, x);
3778 tmp = create_tmp_var_raw (ptr_type_node);
3779 gimple_add_tmp_var (tmp);
3780 gimple_call_set_lhs (stmt, tmp);
3782 gimple_seq_add_stmt (ilist, stmt);
3784 x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
3785 gimplify_assign (ptr, x, ilist);
3788 else if (is_reference (var))
3790 /* For references that are being privatized for Fortran,
3791 allocate new backing storage for the new pointer
3792 variable. This allows us to avoid changing all the
3793 code that expects a pointer to something that expects
3794 a direct variable. */
3795 if (pass == 0)
3796 continue;
3798 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
3799 if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
3801 x = build_receiver_ref (var, false, ctx);
3802 x = build_fold_addr_expr_loc (clause_loc, x);
3804 else if (TREE_CONSTANT (x))
3806 /* For reduction in SIMD loop, defer adding the
3807 initialization of the reference, because if we decide
3808 to use SIMD array for it, the initilization could cause
3809 expansion ICE. */
3810 if (c_kind == OMP_CLAUSE_REDUCTION && is_simd)
3811 x = NULL_TREE;
3812 else
3814 const char *name = NULL;
3815 if (DECL_NAME (var))
3816 name = IDENTIFIER_POINTER (DECL_NAME (new_var));
3818 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
3819 name);
3820 gimple_add_tmp_var (x);
3821 TREE_ADDRESSABLE (x) = 1;
3822 x = build_fold_addr_expr_loc (clause_loc, x);
3825 else
3827 tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3828 x = build_call_expr_loc (clause_loc, atmp, 1, x);
3831 if (x)
3833 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3834 gimplify_assign (new_var, x, ilist);
3837 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3839 else if (c_kind == OMP_CLAUSE_REDUCTION
3840 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3842 if (pass == 0)
3843 continue;
3845 else if (pass != 0)
3846 continue;
3848 switch (OMP_CLAUSE_CODE (c))
3850 case OMP_CLAUSE_SHARED:
3851 /* Ignore shared directives in teams construct. */
3852 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3853 continue;
3854 /* Shared global vars are just accessed directly. */
3855 if (is_global_var (new_var))
3856 break;
3857 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3858 needs to be delayed until after fixup_child_record_type so
3859 that we get the correct type during the dereference. */
3860 by_ref = use_pointer_for_field (var, ctx);
3861 x = build_receiver_ref (var, by_ref, ctx);
3862 SET_DECL_VALUE_EXPR (new_var, x);
3863 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3865 /* ??? If VAR is not passed by reference, and the variable
3866 hasn't been initialized yet, then we'll get a warning for
3867 the store into the omp_data_s structure. Ideally, we'd be
3868 able to notice this and not store anything at all, but
3869 we're generating code too early. Suppress the warning. */
3870 if (!by_ref)
3871 TREE_NO_WARNING (var) = 1;
3872 break;
3874 case OMP_CLAUSE_LASTPRIVATE:
3875 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3876 break;
3877 /* FALLTHRU */
3879 case OMP_CLAUSE_PRIVATE:
3880 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
3881 x = build_outer_var_ref (var, ctx);
3882 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
3884 if (is_task_ctx (ctx))
3885 x = build_receiver_ref (var, false, ctx);
3886 else
3887 x = build_outer_var_ref (var, ctx);
3889 else
3890 x = NULL;
3891 do_private:
3892 tree nx;
3893 nx = lang_hooks.decls.omp_clause_default_ctor (c, new_var, x);
3894 if (is_simd)
3896 tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
3897 if ((TREE_ADDRESSABLE (new_var) || nx || y
3898 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
3899 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3900 idx, lane, ivar, lvar))
3902 if (nx)
3903 x = lang_hooks.decls.omp_clause_default_ctor
3904 (c, unshare_expr (ivar), x);
3905 if (nx && x)
3906 gimplify_and_add (x, &llist[0]);
3907 if (y)
3909 y = lang_hooks.decls.omp_clause_dtor (c, ivar);
3910 if (y)
3912 gimple_seq tseq = NULL;
3914 dtor = y;
3915 gimplify_stmt (&dtor, &tseq);
3916 gimple_seq_add_seq (&llist[1], tseq);
3919 break;
3922 if (nx)
3923 gimplify_and_add (nx, ilist);
3924 /* FALLTHRU */
3926 do_dtor:
3927 x = lang_hooks.decls.omp_clause_dtor (c, new_var);
3928 if (x)
3930 gimple_seq tseq = NULL;
3932 dtor = x;
3933 gimplify_stmt (&dtor, &tseq);
3934 gimple_seq_add_seq (dlist, tseq);
3936 break;
3938 case OMP_CLAUSE_LINEAR:
3939 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
3940 goto do_firstprivate;
3941 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
3942 x = NULL;
3943 else
3944 x = build_outer_var_ref (var, ctx);
3945 goto do_private;
3947 case OMP_CLAUSE_FIRSTPRIVATE:
3948 if (is_task_ctx (ctx))
3950 if (is_reference (var) || is_variable_sized (var))
3951 goto do_dtor;
3952 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
3953 ctx))
3954 || use_pointer_for_field (var, NULL))
3956 x = build_receiver_ref (var, false, ctx);
3957 SET_DECL_VALUE_EXPR (new_var, x);
3958 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3959 goto do_dtor;
3962 do_firstprivate:
3963 x = build_outer_var_ref (var, ctx);
3964 if (is_simd)
3966 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
3967 && gimple_omp_for_combined_into_p (ctx->stmt))
3969 tree t = OMP_CLAUSE_LINEAR_STEP (c);
3970 tree stept = TREE_TYPE (t);
3971 tree ct = find_omp_clause (clauses,
3972 OMP_CLAUSE__LOOPTEMP_);
3973 gcc_assert (ct);
3974 tree l = OMP_CLAUSE_DECL (ct);
3975 tree n1 = fd->loop.n1;
3976 tree step = fd->loop.step;
3977 tree itype = TREE_TYPE (l);
3978 if (POINTER_TYPE_P (itype))
3979 itype = signed_type_for (itype);
3980 l = fold_build2 (MINUS_EXPR, itype, l, n1);
3981 if (TYPE_UNSIGNED (itype)
3982 && fd->loop.cond_code == GT_EXPR)
3983 l = fold_build2 (TRUNC_DIV_EXPR, itype,
3984 fold_build1 (NEGATE_EXPR, itype, l),
3985 fold_build1 (NEGATE_EXPR,
3986 itype, step));
3987 else
3988 l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
3989 t = fold_build2 (MULT_EXPR, stept,
3990 fold_convert (stept, l), t);
3992 if (OMP_CLAUSE_LINEAR_ARRAY (c))
3994 x = lang_hooks.decls.omp_clause_linear_ctor
3995 (c, new_var, x, t);
3996 gimplify_and_add (x, ilist);
3997 goto do_dtor;
4000 if (POINTER_TYPE_P (TREE_TYPE (x)))
4001 x = fold_build2 (POINTER_PLUS_EXPR,
4002 TREE_TYPE (x), x, t);
4003 else
4004 x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, t);
4007 if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
4008 || TREE_ADDRESSABLE (new_var))
4009 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
4010 idx, lane, ivar, lvar))
4012 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
4014 tree iv = create_tmp_var (TREE_TYPE (new_var));
4015 x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
4016 gimplify_and_add (x, ilist);
4017 gimple_stmt_iterator gsi
4018 = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
4019 gassign *g
4020 = gimple_build_assign (unshare_expr (lvar), iv);
4021 gsi_insert_before_without_update (&gsi, g,
4022 GSI_SAME_STMT);
4023 tree t = OMP_CLAUSE_LINEAR_STEP (c);
4024 enum tree_code code = PLUS_EXPR;
4025 if (POINTER_TYPE_P (TREE_TYPE (new_var)))
4026 code = POINTER_PLUS_EXPR;
4027 g = gimple_build_assign (iv, code, iv, t);
4028 gsi_insert_before_without_update (&gsi, g,
4029 GSI_SAME_STMT);
4030 break;
4032 x = lang_hooks.decls.omp_clause_copy_ctor
4033 (c, unshare_expr (ivar), x);
4034 gimplify_and_add (x, &llist[0]);
4035 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
4036 if (x)
4038 gimple_seq tseq = NULL;
4040 dtor = x;
4041 gimplify_stmt (&dtor, &tseq);
4042 gimple_seq_add_seq (&llist[1], tseq);
4044 break;
4047 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
4048 gimplify_and_add (x, ilist);
4049 goto do_dtor;
4051 case OMP_CLAUSE__LOOPTEMP_:
4052 gcc_assert (is_parallel_ctx (ctx));
4053 x = build_outer_var_ref (var, ctx);
4054 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
4055 gimplify_and_add (x, ilist);
4056 break;
4058 case OMP_CLAUSE_COPYIN:
4059 by_ref = use_pointer_for_field (var, NULL);
4060 x = build_receiver_ref (var, by_ref, ctx);
4061 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
4062 append_to_statement_list (x, &copyin_seq);
4063 copyin_by_ref |= by_ref;
4064 break;
4066 case OMP_CLAUSE_REDUCTION:
4067 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4069 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
4070 gimple tseq;
4071 x = build_outer_var_ref (var, ctx);
4073 if (is_reference (var)
4074 && !useless_type_conversion_p (TREE_TYPE (placeholder),
4075 TREE_TYPE (x)))
4076 x = build_fold_addr_expr_loc (clause_loc, x);
4077 SET_DECL_VALUE_EXPR (placeholder, x);
4078 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
4079 tree new_vard = new_var;
4080 if (is_reference (var))
4082 gcc_assert (TREE_CODE (new_var) == MEM_REF);
4083 new_vard = TREE_OPERAND (new_var, 0);
4084 gcc_assert (DECL_P (new_vard));
4086 if (is_simd
4087 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
4088 idx, lane, ivar, lvar))
4090 if (new_vard == new_var)
4092 gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
4093 SET_DECL_VALUE_EXPR (new_var, ivar);
4095 else
4097 SET_DECL_VALUE_EXPR (new_vard,
4098 build_fold_addr_expr (ivar));
4099 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
4101 x = lang_hooks.decls.omp_clause_default_ctor
4102 (c, unshare_expr (ivar),
4103 build_outer_var_ref (var, ctx));
4104 if (x)
4105 gimplify_and_add (x, &llist[0]);
4106 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
4108 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
4109 lower_omp (&tseq, ctx);
4110 gimple_seq_add_seq (&llist[0], tseq);
4112 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
4113 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
4114 lower_omp (&tseq, ctx);
4115 gimple_seq_add_seq (&llist[1], tseq);
4116 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
4117 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
4118 if (new_vard == new_var)
4119 SET_DECL_VALUE_EXPR (new_var, lvar);
4120 else
4121 SET_DECL_VALUE_EXPR (new_vard,
4122 build_fold_addr_expr (lvar));
4123 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
4124 if (x)
4126 tseq = NULL;
4127 dtor = x;
4128 gimplify_stmt (&dtor, &tseq);
4129 gimple_seq_add_seq (&llist[1], tseq);
4131 break;
4133 /* If this is a reference to constant size reduction var
4134 with placeholder, we haven't emitted the initializer
4135 for it because it is undesirable if SIMD arrays are used.
4136 But if they aren't used, we need to emit the deferred
4137 initialization now. */
4138 else if (is_reference (var) && is_simd)
4139 handle_simd_reference (clause_loc, new_vard, ilist);
4140 x = lang_hooks.decls.omp_clause_default_ctor
4141 (c, unshare_expr (new_var),
4142 build_outer_var_ref (var, ctx));
4143 if (x)
4144 gimplify_and_add (x, ilist);
4145 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
4147 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
4148 lower_omp (&tseq, ctx);
4149 gimple_seq_add_seq (ilist, tseq);
4151 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
4152 if (is_simd)
4154 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
4155 lower_omp (&tseq, ctx);
4156 gimple_seq_add_seq (dlist, tseq);
4157 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
4159 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
4160 goto do_dtor;
4162 else
4164 x = omp_reduction_init (c, TREE_TYPE (new_var));
4165 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
4166 enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
4168 /* reduction(-:var) sums up the partial results, so it
4169 acts identically to reduction(+:var). */
4170 if (code == MINUS_EXPR)
4171 code = PLUS_EXPR;
4173 tree new_vard = new_var;
4174 if (is_simd && is_reference (var))
4176 gcc_assert (TREE_CODE (new_var) == MEM_REF);
4177 new_vard = TREE_OPERAND (new_var, 0);
4178 gcc_assert (DECL_P (new_vard));
4180 if (is_simd
4181 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
4182 idx, lane, ivar, lvar))
4184 tree ref = build_outer_var_ref (var, ctx);
4186 gimplify_assign (unshare_expr (ivar), x, &llist[0]);
4188 x = build2 (code, TREE_TYPE (ref), ref, ivar);
4189 ref = build_outer_var_ref (var, ctx);
4190 gimplify_assign (ref, x, &llist[1]);
4192 if (new_vard != new_var)
4194 SET_DECL_VALUE_EXPR (new_vard,
4195 build_fold_addr_expr (lvar));
4196 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
4199 else
4201 if (is_reference (var) && is_simd)
4202 handle_simd_reference (clause_loc, new_vard, ilist);
4203 gimplify_assign (new_var, x, ilist);
4204 if (is_simd)
4206 tree ref = build_outer_var_ref (var, ctx);
4208 x = build2 (code, TREE_TYPE (ref), ref, new_var);
4209 ref = build_outer_var_ref (var, ctx);
4210 gimplify_assign (ref, x, dlist);
4214 break;
4216 default:
4217 gcc_unreachable ();
4222 if (lane)
4224 tree uid = create_tmp_var (ptr_type_node, "simduid");
4225 /* Don't want uninit warnings on simduid, it is always uninitialized,
4226 but we use it not for the value, but for the DECL_UID only. */
4227 TREE_NO_WARNING (uid) = 1;
4228 gimple g
4229 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid);
4230 gimple_call_set_lhs (g, lane);
4231 gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
4232 gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
4233 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
4234 OMP_CLAUSE__SIMDUID__DECL (c) = uid;
4235 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
4236 gimple_omp_for_set_clauses (ctx->stmt, c);
4237 g = gimple_build_assign (lane, INTEGER_CST,
4238 build_int_cst (unsigned_type_node, 0));
4239 gimple_seq_add_stmt (ilist, g);
4240 for (int i = 0; i < 2; i++)
4241 if (llist[i])
4243 tree vf = create_tmp_var (unsigned_type_node);
4244 g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
4245 gimple_call_set_lhs (g, vf);
4246 gimple_seq *seq = i == 0 ? ilist : dlist;
4247 gimple_seq_add_stmt (seq, g);
4248 tree t = build_int_cst (unsigned_type_node, 0);
4249 g = gimple_build_assign (idx, INTEGER_CST, t);
4250 gimple_seq_add_stmt (seq, g);
4251 tree body = create_artificial_label (UNKNOWN_LOCATION);
4252 tree header = create_artificial_label (UNKNOWN_LOCATION);
4253 tree end = create_artificial_label (UNKNOWN_LOCATION);
4254 gimple_seq_add_stmt (seq, gimple_build_goto (header));
4255 gimple_seq_add_stmt (seq, gimple_build_label (body));
4256 gimple_seq_add_seq (seq, llist[i]);
4257 t = build_int_cst (unsigned_type_node, 1);
4258 g = gimple_build_assign (idx, PLUS_EXPR, idx, t);
4259 gimple_seq_add_stmt (seq, g);
4260 gimple_seq_add_stmt (seq, gimple_build_label (header));
4261 g = gimple_build_cond (LT_EXPR, idx, vf, body, end);
4262 gimple_seq_add_stmt (seq, g);
4263 gimple_seq_add_stmt (seq, gimple_build_label (end));
4267 /* The copyin sequence is not to be executed by the main thread, since
4268 that would result in self-copies. Perhaps not visible to scalars,
4269 but it certainly is to C++ operator=. */
4270 if (copyin_seq)
4272 x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
4274 x = build2 (NE_EXPR, boolean_type_node, x,
4275 build_int_cst (TREE_TYPE (x), 0));
4276 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
4277 gimplify_and_add (x, ilist);
4280 /* If any copyin variable is passed by reference, we must ensure the
4281 master thread doesn't modify it before it is copied over in all
4282 threads. Similarly for variables in both firstprivate and
4283 lastprivate clauses we need to ensure the lastprivate copying
4284 happens after firstprivate copying in all threads. And similarly
4285 for UDRs if initializer expression refers to omp_orig. */
4286 if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref)
4288 /* Don't add any barrier for #pragma omp simd or
4289 #pragma omp distribute. */
4290 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
4291 || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR)
4292 gimple_seq_add_stmt (ilist, build_omp_barrier (NULL_TREE));
4295 /* If max_vf is non-zero, then we can use only a vectorization factor
4296 up to the max_vf we chose. So stick it into the safelen clause. */
4297 if (max_vf)
4299 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
4300 OMP_CLAUSE_SAFELEN);
4301 if (c == NULL_TREE
4302 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) == INTEGER_CST
4303 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
4304 max_vf) == 1))
4306 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
4307 OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
4308 max_vf);
4309 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
4310 gimple_omp_for_set_clauses (ctx->stmt, c);
4316 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4317 both parallel and workshare constructs. PREDICATE may be NULL if it's
4318 always true. */
4320 static void
4321 lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list,
4322 omp_context *ctx)
4324 tree x, c, label = NULL, orig_clauses = clauses;
4325 bool par_clauses = false;
4326 tree simduid = NULL, lastlane = NULL;
4328 /* Early exit if there are no lastprivate or linear clauses. */
4329 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
4330 if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
4331 || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
4332 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
4333 break;
4334 if (clauses == NULL)
4336 /* If this was a workshare clause, see if it had been combined
4337 with its parallel. In that case, look for the clauses on the
4338 parallel statement itself. */
4339 if (is_parallel_ctx (ctx))
4340 return;
4342 ctx = ctx->outer;
4343 if (ctx == NULL || !is_parallel_ctx (ctx))
4344 return;
4346 clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
4347 OMP_CLAUSE_LASTPRIVATE);
4348 if (clauses == NULL)
4349 return;
4350 par_clauses = true;
4353 if (predicate)
4355 gcond *stmt;
4356 tree label_true, arm1, arm2;
4358 label = create_artificial_label (UNKNOWN_LOCATION);
4359 label_true = create_artificial_label (UNKNOWN_LOCATION);
4360 arm1 = TREE_OPERAND (predicate, 0);
4361 arm2 = TREE_OPERAND (predicate, 1);
4362 gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
4363 gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
4364 stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2,
4365 label_true, label);
4366 gimple_seq_add_stmt (stmt_list, stmt);
4367 gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
4370 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4371 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
4373 simduid = find_omp_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
4374 if (simduid)
4375 simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
4378 for (c = clauses; c ;)
4380 tree var, new_var;
4381 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4383 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
4384 || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
4385 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
4387 var = OMP_CLAUSE_DECL (c);
4388 new_var = lookup_decl (var, ctx);
4390 if (simduid && DECL_HAS_VALUE_EXPR_P (new_var))
4392 tree val = DECL_VALUE_EXPR (new_var);
4393 if (TREE_CODE (val) == ARRAY_REF
4394 && VAR_P (TREE_OPERAND (val, 0))
4395 && lookup_attribute ("omp simd array",
4396 DECL_ATTRIBUTES (TREE_OPERAND (val,
4397 0))))
4399 if (lastlane == NULL)
4401 lastlane = create_tmp_var (unsigned_type_node);
4402 gcall *g
4403 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
4404 2, simduid,
4405 TREE_OPERAND (val, 1));
4406 gimple_call_set_lhs (g, lastlane);
4407 gimple_seq_add_stmt (stmt_list, g);
4409 new_var = build4 (ARRAY_REF, TREE_TYPE (val),
4410 TREE_OPERAND (val, 0), lastlane,
4411 NULL_TREE, NULL_TREE);
4415 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
4416 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
4418 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
4419 gimple_seq_add_seq (stmt_list,
4420 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
4421 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
4423 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
4424 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
4426 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
4427 gimple_seq_add_seq (stmt_list,
4428 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
4429 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
4432 x = build_outer_var_ref (var, ctx);
4433 if (is_reference (var))
4434 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4435 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
4436 gimplify_and_add (x, stmt_list);
4438 c = OMP_CLAUSE_CHAIN (c);
4439 if (c == NULL && !par_clauses)
4441 /* If this was a workshare clause, see if it had been combined
4442 with its parallel. In that case, continue looking for the
4443 clauses also on the parallel statement itself. */
4444 if (is_parallel_ctx (ctx))
4445 break;
4447 ctx = ctx->outer;
4448 if (ctx == NULL || !is_parallel_ctx (ctx))
4449 break;
4451 c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
4452 OMP_CLAUSE_LASTPRIVATE);
4453 par_clauses = true;
4457 if (label)
4458 gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
4461 static void
4462 oacc_lower_reduction_var_helper (gimple_seq *stmt_seqp, omp_context *ctx,
4463 tree tid, tree var, tree new_var)
4465 /* The atomic add at the end of the sum creates unnecessary
4466 write contention on accelerators. To work around this,
4467 create an array to store the partial reductions. Later, in
4468 lower_omp_for (for openacc), the values of array will be
4469 combined. */
4471 tree t = NULL_TREE, array, x;
4472 tree type = get_base_type (var);
4473 gimple stmt;
4475 /* Now insert the partial reductions into the array. */
4477 /* Find the reduction array. */
4479 tree ptype = build_pointer_type (type);
4481 t = lookup_oacc_reduction (oacc_get_reduction_array_id (var), ctx);
4482 t = build_receiver_ref (t, false, ctx->outer);
4484 array = create_tmp_var (ptype);
4485 gimplify_assign (array, t, stmt_seqp);
4487 tree ptr = create_tmp_var (TREE_TYPE (array));
4489 /* Find the reduction array. */
4491 /* testing a unary conversion. */
4492 tree offset = create_tmp_var (sizetype);
4493 gimplify_assign (offset, TYPE_SIZE_UNIT (type),
4494 stmt_seqp);
4495 t = create_tmp_var (sizetype);
4496 gimplify_assign (t, unshare_expr (fold_build1 (NOP_EXPR, sizetype, tid)),
4497 stmt_seqp);
4498 stmt = gimple_build_assign (offset, MULT_EXPR, offset, t);
4499 gimple_seq_add_stmt (stmt_seqp, stmt);
4501 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4502 of adding sizeof(var) to the array? */
4503 ptr = create_tmp_var (ptype);
4504 stmt = gimple_build_assign (unshare_expr (ptr), POINTER_PLUS_EXPR, array,
4505 offset);
4506 gimple_seq_add_stmt (stmt_seqp, stmt);
4508 /* Move the local sum to gfc$sum[i]. */
4509 x = unshare_expr (build_simple_mem_ref (ptr));
4510 stmt = gimplify_assign (x, new_var, stmt_seqp);
4513 /* Generate code to implement the REDUCTION clauses. */
4515 static void
4516 lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx)
4518 gimple_seq sub_seq = NULL;
4519 gimple stmt;
4520 tree x, c, tid = NULL_TREE;
4521 int count = 0;
4523 /* SIMD reductions are handled in lower_rec_input_clauses. */
4524 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4525 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
4526 return;
4528 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4529 update in that case, otherwise use a lock. */
4530 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
4531 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
4533 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4535 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4536 count = -1;
4537 break;
4539 count++;
4542 if (count == 0)
4543 return;
4545 /* Initialize thread info for OpenACC. */
4546 if (is_gimple_omp_oacc (ctx->stmt))
4548 /* Get the current thread id. */
4549 tree call = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
4550 tid = create_tmp_var (TREE_TYPE (TREE_TYPE (call)));
4551 gimple stmt = gimple_build_call (call, 0);
4552 gimple_call_set_lhs (stmt, tid);
4553 gimple_seq_add_stmt (stmt_seqp, stmt);
4556 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4558 tree var, ref, new_var;
4559 enum tree_code code;
4560 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4562 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
4563 continue;
4565 var = OMP_CLAUSE_DECL (c);
4566 new_var = lookup_decl (var, ctx);
4567 if (is_reference (var))
4568 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4569 ref = build_outer_var_ref (var, ctx);
4570 code = OMP_CLAUSE_REDUCTION_CODE (c);
4572 /* reduction(-:var) sums up the partial results, so it acts
4573 identically to reduction(+:var). */
4574 if (code == MINUS_EXPR)
4575 code = PLUS_EXPR;
4577 if (is_gimple_omp_oacc (ctx->stmt))
4579 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c));
4581 oacc_lower_reduction_var_helper (stmt_seqp, ctx, tid, var, new_var);
4583 else if (count == 1)
4585 tree addr = build_fold_addr_expr_loc (clause_loc, ref);
4587 addr = save_expr (addr);
4588 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
4589 x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var);
4590 x = build2 (OMP_ATOMIC, void_type_node, addr, x);
4591 gimplify_and_add (x, stmt_seqp);
4592 return;
4594 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4596 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
4598 if (is_reference (var)
4599 && !useless_type_conversion_p (TREE_TYPE (placeholder),
4600 TREE_TYPE (ref)))
4601 ref = build_fold_addr_expr_loc (clause_loc, ref);
4602 SET_DECL_VALUE_EXPR (placeholder, ref);
4603 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
4604 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
4605 gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
4606 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
4607 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
4609 else
4611 x = build2 (code, TREE_TYPE (ref), ref, new_var);
4612 ref = build_outer_var_ref (var, ctx);
4613 gimplify_assign (ref, x, &sub_seq);
4617 if (is_gimple_omp_oacc (ctx->stmt))
4618 return;
4620 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START),
4622 gimple_seq_add_stmt (stmt_seqp, stmt);
4624 gimple_seq_add_seq (stmt_seqp, sub_seq);
4626 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
4628 gimple_seq_add_stmt (stmt_seqp, stmt);
4632 /* Generate code to implement the COPYPRIVATE clauses. */
4634 static void
4635 lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
4636 omp_context *ctx)
4638 tree c;
4640 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4642 tree var, new_var, ref, x;
4643 bool by_ref;
4644 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4646 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
4647 continue;
4649 var = OMP_CLAUSE_DECL (c);
4650 by_ref = use_pointer_for_field (var, NULL);
4652 ref = build_sender_ref (var, ctx);
4653 x = new_var = lookup_decl_in_outer_ctx (var, ctx);
4654 if (by_ref)
4656 x = build_fold_addr_expr_loc (clause_loc, new_var);
4657 x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
4659 gimplify_assign (ref, x, slist);
4661 ref = build_receiver_ref (var, false, ctx);
4662 if (by_ref)
4664 ref = fold_convert_loc (clause_loc,
4665 build_pointer_type (TREE_TYPE (new_var)),
4666 ref);
4667 ref = build_fold_indirect_ref_loc (clause_loc, ref);
4669 if (is_reference (var))
4671 ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
4672 ref = build_simple_mem_ref_loc (clause_loc, ref);
4673 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4675 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
4676 gimplify_and_add (x, rlist);
4681 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4682 and REDUCTION from the sender (aka parent) side. */
4684 static void
4685 lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
4686 omp_context *ctx)
4688 tree c;
4690 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4692 tree val, ref, x, var;
4693 bool by_ref, do_in = false, do_out = false;
4694 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4696 switch (OMP_CLAUSE_CODE (c))
4698 case OMP_CLAUSE_PRIVATE:
4699 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
4700 break;
4701 continue;
4702 case OMP_CLAUSE_FIRSTPRIVATE:
4703 case OMP_CLAUSE_COPYIN:
4704 case OMP_CLAUSE_LASTPRIVATE:
4705 case OMP_CLAUSE_REDUCTION:
4706 case OMP_CLAUSE__LOOPTEMP_:
4707 break;
4708 default:
4709 continue;
4712 val = OMP_CLAUSE_DECL (c);
4713 var = lookup_decl_in_outer_ctx (val, ctx);
4715 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
4716 && is_global_var (var))
4717 continue;
4718 if (is_variable_sized (val))
4719 continue;
4720 by_ref = use_pointer_for_field (val, NULL);
4722 switch (OMP_CLAUSE_CODE (c))
4724 case OMP_CLAUSE_PRIVATE:
4725 case OMP_CLAUSE_FIRSTPRIVATE:
4726 case OMP_CLAUSE_COPYIN:
4727 case OMP_CLAUSE__LOOPTEMP_:
4728 do_in = true;
4729 break;
4731 case OMP_CLAUSE_LASTPRIVATE:
4732 if (by_ref || is_reference (val))
4734 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
4735 continue;
4736 do_in = true;
4738 else
4740 do_out = true;
4741 if (lang_hooks.decls.omp_private_outer_ref (val))
4742 do_in = true;
4744 break;
4746 case OMP_CLAUSE_REDUCTION:
4747 do_in = true;
4748 do_out = !(by_ref || is_reference (val));
4749 break;
4751 default:
4752 gcc_unreachable ();
4755 if (do_in)
4757 ref = build_sender_ref (val, ctx);
4758 x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
4759 gimplify_assign (ref, x, ilist);
4760 if (is_task_ctx (ctx))
4761 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
4764 if (do_out)
4766 ref = build_sender_ref (val, ctx);
4767 gimplify_assign (var, ref, olist);
4772 /* Generate code to implement SHARED from the sender (aka parent)
4773 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4774 list things that got automatically shared. */
4776 static void
4777 lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
4779 tree var, ovar, nvar, f, x, record_type;
4781 if (ctx->record_type == NULL)
4782 return;
4784 record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
4785 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
4787 ovar = DECL_ABSTRACT_ORIGIN (f);
4788 nvar = maybe_lookup_decl (ovar, ctx);
4789 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar))
4790 continue;
4792 /* If CTX is a nested parallel directive. Find the immediately
4793 enclosing parallel or workshare construct that contains a
4794 mapping for OVAR. */
4795 var = lookup_decl_in_outer_ctx (ovar, ctx);
4797 if (use_pointer_for_field (ovar, ctx))
4799 x = build_sender_ref (ovar, ctx);
4800 var = build_fold_addr_expr (var);
4801 gimplify_assign (x, var, ilist);
4803 else
4805 x = build_sender_ref (ovar, ctx);
4806 gimplify_assign (x, var, ilist);
4808 if (!TREE_READONLY (var)
4809 /* We don't need to receive a new reference to a result
4810 or parm decl. In fact we may not store to it as we will
4811 invalidate any pending RSO and generate wrong gimple
4812 during inlining. */
4813 && !((TREE_CODE (var) == RESULT_DECL
4814 || TREE_CODE (var) == PARM_DECL)
4815 && DECL_BY_REFERENCE (var)))
4817 x = build_sender_ref (ovar, ctx);
4818 gimplify_assign (var, x, olist);
4825 /* A convenience function to build an empty GIMPLE_COND with just the
4826 condition. */
4828 static gcond *
4829 gimple_build_cond_empty (tree cond)
4831 enum tree_code pred_code;
4832 tree lhs, rhs;
4834 gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs);
4835 return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE);
4839 /* Build the function calls to GOMP_parallel_start etc to actually
4840 generate the parallel operation. REGION is the parallel region
4841 being expanded. BB is the block where to insert the code. WS_ARGS
4842 will be set if this is a call to a combined parallel+workshare
4843 construct, it contains the list of additional arguments needed by
4844 the workshare construct. */
4846 static void
4847 expand_parallel_call (struct omp_region *region, basic_block bb,
4848 gomp_parallel *entry_stmt,
4849 vec<tree, va_gc> *ws_args)
4851 tree t, t1, t2, val, cond, c, clauses, flags;
4852 gimple_stmt_iterator gsi;
4853 gimple stmt;
4854 enum built_in_function start_ix;
4855 int start_ix2;
4856 location_t clause_loc;
4857 vec<tree, va_gc> *args;
4859 clauses = gimple_omp_parallel_clauses (entry_stmt);
4861 /* Determine what flavor of GOMP_parallel we will be
4862 emitting. */
4863 start_ix = BUILT_IN_GOMP_PARALLEL;
4864 if (is_combined_parallel (region))
4866 switch (region->inner->type)
4868 case GIMPLE_OMP_FOR:
4869 gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
4870 start_ix2 = ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4871 + (region->inner->sched_kind
4872 == OMP_CLAUSE_SCHEDULE_RUNTIME
4873 ? 3 : region->inner->sched_kind));
4874 start_ix = (enum built_in_function)start_ix2;
4875 break;
4876 case GIMPLE_OMP_SECTIONS:
4877 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS;
4878 break;
4879 default:
4880 gcc_unreachable ();
4884 /* By default, the value of NUM_THREADS is zero (selected at run time)
4885 and there is no conditional. */
4886 cond = NULL_TREE;
4887 val = build_int_cst (unsigned_type_node, 0);
4888 flags = build_int_cst (unsigned_type_node, 0);
4890 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
4891 if (c)
4892 cond = OMP_CLAUSE_IF_EXPR (c);
4894 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS);
4895 if (c)
4897 val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
4898 clause_loc = OMP_CLAUSE_LOCATION (c);
4900 else
4901 clause_loc = gimple_location (entry_stmt);
4903 c = find_omp_clause (clauses, OMP_CLAUSE_PROC_BIND);
4904 if (c)
4905 flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c));
4907 /* Ensure 'val' is of the correct type. */
4908 val = fold_convert_loc (clause_loc, unsigned_type_node, val);
4910 /* If we found the clause 'if (cond)', build either
4911 (cond != 0) or (cond ? val : 1u). */
4912 if (cond)
4914 cond = gimple_boolify (cond);
4916 if (integer_zerop (val))
4917 val = fold_build2_loc (clause_loc,
4918 EQ_EXPR, unsigned_type_node, cond,
4919 build_int_cst (TREE_TYPE (cond), 0));
4920 else
4922 basic_block cond_bb, then_bb, else_bb;
4923 edge e, e_then, e_else;
4924 tree tmp_then, tmp_else, tmp_join, tmp_var;
4926 tmp_var = create_tmp_var (TREE_TYPE (val));
4927 if (gimple_in_ssa_p (cfun))
4929 tmp_then = make_ssa_name (tmp_var);
4930 tmp_else = make_ssa_name (tmp_var);
4931 tmp_join = make_ssa_name (tmp_var);
4933 else
4935 tmp_then = tmp_var;
4936 tmp_else = tmp_var;
4937 tmp_join = tmp_var;
4940 e = split_block (bb, NULL);
4941 cond_bb = e->src;
4942 bb = e->dest;
4943 remove_edge (e);
4945 then_bb = create_empty_bb (cond_bb);
4946 else_bb = create_empty_bb (then_bb);
4947 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
4948 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
4950 stmt = gimple_build_cond_empty (cond);
4951 gsi = gsi_start_bb (cond_bb);
4952 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4954 gsi = gsi_start_bb (then_bb);
4955 stmt = gimple_build_assign (tmp_then, val);
4956 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4958 gsi = gsi_start_bb (else_bb);
4959 stmt = gimple_build_assign
4960 (tmp_else, build_int_cst (unsigned_type_node, 1));
4961 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4963 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
4964 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
4965 add_bb_to_loop (then_bb, cond_bb->loop_father);
4966 add_bb_to_loop (else_bb, cond_bb->loop_father);
4967 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
4968 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
4970 if (gimple_in_ssa_p (cfun))
4972 gphi *phi = create_phi_node (tmp_join, bb);
4973 add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION);
4974 add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION);
4977 val = tmp_join;
4980 gsi = gsi_start_bb (bb);
4981 val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE,
4982 false, GSI_CONTINUE_LINKING);
4985 gsi = gsi_last_bb (bb);
4986 t = gimple_omp_parallel_data_arg (entry_stmt);
4987 if (t == NULL)
4988 t1 = null_pointer_node;
4989 else
4990 t1 = build_fold_addr_expr (t);
4991 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
4993 vec_alloc (args, 4 + vec_safe_length (ws_args));
4994 args->quick_push (t2);
4995 args->quick_push (t1);
4996 args->quick_push (val);
4997 if (ws_args)
4998 args->splice (*ws_args);
4999 args->quick_push (flags);
5001 t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
5002 builtin_decl_explicit (start_ix), args);
5004 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5005 false, GSI_CONTINUE_LINKING);
5008 /* Insert a function call whose name is FUNC_NAME with the information from
5009 ENTRY_STMT into the basic_block BB. */
5011 static void
5012 expand_cilk_for_call (basic_block bb, gomp_parallel *entry_stmt,
5013 vec <tree, va_gc> *ws_args)
5015 tree t, t1, t2;
5016 gimple_stmt_iterator gsi;
5017 vec <tree, va_gc> *args;
5019 gcc_assert (vec_safe_length (ws_args) == 2);
5020 tree func_name = (*ws_args)[0];
5021 tree grain = (*ws_args)[1];
5023 tree clauses = gimple_omp_parallel_clauses (entry_stmt);
5024 tree count = find_omp_clause (clauses, OMP_CLAUSE__CILK_FOR_COUNT_);
5025 gcc_assert (count != NULL_TREE);
5026 count = OMP_CLAUSE_OPERAND (count, 0);
5028 gsi = gsi_last_bb (bb);
5029 t = gimple_omp_parallel_data_arg (entry_stmt);
5030 if (t == NULL)
5031 t1 = null_pointer_node;
5032 else
5033 t1 = build_fold_addr_expr (t);
5034 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
5036 vec_alloc (args, 4);
5037 args->quick_push (t2);
5038 args->quick_push (t1);
5039 args->quick_push (count);
5040 args->quick_push (grain);
5041 t = build_call_expr_loc_vec (UNKNOWN_LOCATION, func_name, args);
5043 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, false,
5044 GSI_CONTINUE_LINKING);
5047 /* Build the function call to GOMP_task to actually
5048 generate the task operation. BB is the block where to insert the code. */
5050 static void
5051 expand_task_call (basic_block bb, gomp_task *entry_stmt)
5053 tree t, t1, t2, t3, flags, cond, c, c2, clauses, depend;
5054 gimple_stmt_iterator gsi;
5055 location_t loc = gimple_location (entry_stmt);
5057 clauses = gimple_omp_task_clauses (entry_stmt);
5059 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
5060 if (c)
5061 cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (c));
5062 else
5063 cond = boolean_true_node;
5065 c = find_omp_clause (clauses, OMP_CLAUSE_UNTIED);
5066 c2 = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE);
5067 depend = find_omp_clause (clauses, OMP_CLAUSE_DEPEND);
5068 flags = build_int_cst (unsigned_type_node,
5069 (c ? 1 : 0) + (c2 ? 4 : 0) + (depend ? 8 : 0));
5071 c = find_omp_clause (clauses, OMP_CLAUSE_FINAL);
5072 if (c)
5074 c = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c));
5075 c = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, c,
5076 build_int_cst (unsigned_type_node, 2),
5077 build_int_cst (unsigned_type_node, 0));
5078 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, c);
5080 if (depend)
5081 depend = OMP_CLAUSE_DECL (depend);
5082 else
5083 depend = build_int_cst (ptr_type_node, 0);
5085 gsi = gsi_last_bb (bb);
5086 t = gimple_omp_task_data_arg (entry_stmt);
5087 if (t == NULL)
5088 t2 = null_pointer_node;
5089 else
5090 t2 = build_fold_addr_expr_loc (loc, t);
5091 t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt));
5092 t = gimple_omp_task_copy_fn (entry_stmt);
5093 if (t == NULL)
5094 t3 = null_pointer_node;
5095 else
5096 t3 = build_fold_addr_expr_loc (loc, t);
5098 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
5099 8, t1, t2, t3,
5100 gimple_omp_task_arg_size (entry_stmt),
5101 gimple_omp_task_arg_align (entry_stmt), cond, flags,
5102 depend);
5104 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5105 false, GSI_CONTINUE_LINKING);
5109 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5110 catch handler and return it. This prevents programs from violating the
5111 structured block semantics with throws. */
5113 static gimple_seq
5114 maybe_catch_exception (gimple_seq body)
5116 gimple g;
5117 tree decl;
5119 if (!flag_exceptions)
5120 return body;
5122 if (lang_hooks.eh_protect_cleanup_actions != NULL)
5123 decl = lang_hooks.eh_protect_cleanup_actions ();
5124 else
5125 decl = builtin_decl_explicit (BUILT_IN_TRAP);
5127 g = gimple_build_eh_must_not_throw (decl);
5128 g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
5129 GIMPLE_TRY_CATCH);
5131 return gimple_seq_alloc_with_stmt (g);
5134 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5136 static tree
5137 vec2chain (vec<tree, va_gc> *v)
5139 tree chain = NULL_TREE, t;
5140 unsigned ix;
5142 FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t)
5144 DECL_CHAIN (t) = chain;
5145 chain = t;
5148 return chain;
5152 /* Remove barriers in REGION->EXIT's block. Note that this is only
5153 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5154 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5155 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5156 removed. */
5158 static void
5159 remove_exit_barrier (struct omp_region *region)
5161 gimple_stmt_iterator gsi;
5162 basic_block exit_bb;
5163 edge_iterator ei;
5164 edge e;
5165 gimple stmt;
5166 int any_addressable_vars = -1;
5168 exit_bb = region->exit;
5170 /* If the parallel region doesn't return, we don't have REGION->EXIT
5171 block at all. */
5172 if (! exit_bb)
5173 return;
5175 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5176 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5177 statements that can appear in between are extremely limited -- no
5178 memory operations at all. Here, we allow nothing at all, so the
5179 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5180 gsi = gsi_last_bb (exit_bb);
5181 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
5182 gsi_prev (&gsi);
5183 if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
5184 return;
5186 FOR_EACH_EDGE (e, ei, exit_bb->preds)
5188 gsi = gsi_last_bb (e->src);
5189 if (gsi_end_p (gsi))
5190 continue;
5191 stmt = gsi_stmt (gsi);
5192 if (gimple_code (stmt) == GIMPLE_OMP_RETURN
5193 && !gimple_omp_return_nowait_p (stmt))
5195 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5196 in many cases. If there could be tasks queued, the barrier
5197 might be needed to let the tasks run before some local
5198 variable of the parallel that the task uses as shared
5199 runs out of scope. The task can be spawned either
5200 from within current function (this would be easy to check)
5201 or from some function it calls and gets passed an address
5202 of such a variable. */
5203 if (any_addressable_vars < 0)
5205 gomp_parallel *parallel_stmt
5206 = as_a <gomp_parallel *> (last_stmt (region->entry));
5207 tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt);
5208 tree local_decls, block, decl;
5209 unsigned ix;
5211 any_addressable_vars = 0;
5212 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl)
5213 if (TREE_ADDRESSABLE (decl))
5215 any_addressable_vars = 1;
5216 break;
5218 for (block = gimple_block (stmt);
5219 !any_addressable_vars
5220 && block
5221 && TREE_CODE (block) == BLOCK;
5222 block = BLOCK_SUPERCONTEXT (block))
5224 for (local_decls = BLOCK_VARS (block);
5225 local_decls;
5226 local_decls = DECL_CHAIN (local_decls))
5227 if (TREE_ADDRESSABLE (local_decls))
5229 any_addressable_vars = 1;
5230 break;
5232 if (block == gimple_block (parallel_stmt))
5233 break;
5236 if (!any_addressable_vars)
5237 gimple_omp_return_set_nowait (stmt);
5242 static void
5243 remove_exit_barriers (struct omp_region *region)
5245 if (region->type == GIMPLE_OMP_PARALLEL)
5246 remove_exit_barrier (region);
5248 if (region->inner)
5250 region = region->inner;
5251 remove_exit_barriers (region);
5252 while (region->next)
5254 region = region->next;
5255 remove_exit_barriers (region);
5260 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5261 calls. These can't be declared as const functions, but
5262 within one parallel body they are constant, so they can be
5263 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5264 which are declared const. Similarly for task body, except
5265 that in untied task omp_get_thread_num () can change at any task
5266 scheduling point. */
5268 static void
5269 optimize_omp_library_calls (gimple entry_stmt)
5271 basic_block bb;
5272 gimple_stmt_iterator gsi;
5273 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5274 tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree);
5275 tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
5276 tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree);
5277 bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
5278 && find_omp_clause (gimple_omp_task_clauses (entry_stmt),
5279 OMP_CLAUSE_UNTIED) != NULL);
5281 FOR_EACH_BB_FN (bb, cfun)
5282 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
5284 gimple call = gsi_stmt (gsi);
5285 tree decl;
5287 if (is_gimple_call (call)
5288 && (decl = gimple_call_fndecl (call))
5289 && DECL_EXTERNAL (decl)
5290 && TREE_PUBLIC (decl)
5291 && DECL_INITIAL (decl) == NULL)
5293 tree built_in;
5295 if (DECL_NAME (decl) == thr_num_id)
5297 /* In #pragma omp task untied omp_get_thread_num () can change
5298 during the execution of the task region. */
5299 if (untied_task)
5300 continue;
5301 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5303 else if (DECL_NAME (decl) == num_thr_id)
5304 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
5305 else
5306 continue;
5308 if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in)
5309 || gimple_call_num_args (call) != 0)
5310 continue;
5312 if (flag_exceptions && !TREE_NOTHROW (decl))
5313 continue;
5315 if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE
5316 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)),
5317 TREE_TYPE (TREE_TYPE (built_in))))
5318 continue;
5320 gimple_call_set_fndecl (call, built_in);
5325 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5326 regimplified. */
5328 static tree
5329 expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *)
5331 tree t = *tp;
5333 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5334 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t))
5335 return t;
5337 if (TREE_CODE (t) == ADDR_EXPR)
5338 recompute_tree_invariant_for_addr_expr (t);
5340 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
5341 return NULL_TREE;
5344 /* Prepend TO = FROM assignment before *GSI_P. */
5346 static void
5347 expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from)
5349 bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to);
5350 from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE,
5351 true, GSI_SAME_STMT);
5352 gimple stmt = gimple_build_assign (to, from);
5353 gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT);
5354 if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL)
5355 || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL))
5357 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5358 gimple_regimplify_operands (stmt, &gsi);
5362 /* Expand the OpenMP parallel or task directive starting at REGION. */
5364 static void
5365 expand_omp_taskreg (struct omp_region *region)
5367 basic_block entry_bb, exit_bb, new_bb;
5368 struct function *child_cfun;
5369 tree child_fn, block, t;
5370 gimple_stmt_iterator gsi;
5371 gimple entry_stmt, stmt;
5372 edge e;
5373 vec<tree, va_gc> *ws_args;
5375 entry_stmt = last_stmt (region->entry);
5376 child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
5377 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
5379 entry_bb = region->entry;
5380 if (gimple_code (entry_stmt) == GIMPLE_OMP_TASK)
5381 exit_bb = region->cont;
5382 else
5383 exit_bb = region->exit;
5385 bool is_cilk_for
5386 = (flag_cilkplus
5387 && gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL
5388 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt),
5389 OMP_CLAUSE__CILK_FOR_COUNT_) != NULL_TREE);
5391 if (is_cilk_for)
5392 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5393 and the inner statement contains the name of the built-in function
5394 and grain. */
5395 ws_args = region->inner->ws_args;
5396 else if (is_combined_parallel (region))
5397 ws_args = region->ws_args;
5398 else
5399 ws_args = NULL;
5401 if (child_cfun->cfg)
5403 /* Due to inlining, it may happen that we have already outlined
5404 the region, in which case all we need to do is make the
5405 sub-graph unreachable and emit the parallel call. */
5406 edge entry_succ_e, exit_succ_e;
5408 entry_succ_e = single_succ_edge (entry_bb);
5410 gsi = gsi_last_bb (entry_bb);
5411 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
5412 || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
5413 gsi_remove (&gsi, true);
5415 new_bb = entry_bb;
5416 if (exit_bb)
5418 exit_succ_e = single_succ_edge (exit_bb);
5419 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
5421 remove_edge_and_dominated_blocks (entry_succ_e);
5423 else
5425 unsigned srcidx, dstidx, num;
5427 /* If the parallel region needs data sent from the parent
5428 function, then the very first statement (except possible
5429 tree profile counter updates) of the parallel body
5430 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5431 &.OMP_DATA_O is passed as an argument to the child function,
5432 we need to replace it with the argument as seen by the child
5433 function.
5435 In most cases, this will end up being the identity assignment
5436 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5437 a function call that has been inlined, the original PARM_DECL
5438 .OMP_DATA_I may have been converted into a different local
5439 variable. In which case, we need to keep the assignment. */
5440 if (gimple_omp_taskreg_data_arg (entry_stmt))
5442 basic_block entry_succ_bb
5443 = single_succ_p (entry_bb) ? single_succ (entry_bb)
5444 : FALLTHRU_EDGE (entry_bb)->dest;
5445 tree arg, narg;
5446 gimple parcopy_stmt = NULL;
5448 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
5450 gimple stmt;
5452 gcc_assert (!gsi_end_p (gsi));
5453 stmt = gsi_stmt (gsi);
5454 if (gimple_code (stmt) != GIMPLE_ASSIGN)
5455 continue;
5457 if (gimple_num_ops (stmt) == 2)
5459 tree arg = gimple_assign_rhs1 (stmt);
5461 /* We're ignore the subcode because we're
5462 effectively doing a STRIP_NOPS. */
5464 if (TREE_CODE (arg) == ADDR_EXPR
5465 && TREE_OPERAND (arg, 0)
5466 == gimple_omp_taskreg_data_arg (entry_stmt))
5468 parcopy_stmt = stmt;
5469 break;
5474 gcc_assert (parcopy_stmt != NULL);
5475 arg = DECL_ARGUMENTS (child_fn);
5477 if (!gimple_in_ssa_p (cfun))
5479 if (gimple_assign_lhs (parcopy_stmt) == arg)
5480 gsi_remove (&gsi, true);
5481 else
5483 /* ?? Is setting the subcode really necessary ?? */
5484 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg));
5485 gimple_assign_set_rhs1 (parcopy_stmt, arg);
5488 else
5490 /* If we are in ssa form, we must load the value from the default
5491 definition of the argument. That should not be defined now,
5492 since the argument is not used uninitialized. */
5493 gcc_assert (ssa_default_def (cfun, arg) == NULL);
5494 narg = make_ssa_name (arg, gimple_build_nop ());
5495 set_ssa_default_def (cfun, arg, narg);
5496 /* ?? Is setting the subcode really necessary ?? */
5497 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (narg));
5498 gimple_assign_set_rhs1 (parcopy_stmt, narg);
5499 update_stmt (parcopy_stmt);
5503 /* Declare local variables needed in CHILD_CFUN. */
5504 block = DECL_INITIAL (child_fn);
5505 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
5506 /* The gimplifier could record temporaries in parallel/task block
5507 rather than in containing function's local_decls chain,
5508 which would mean cgraph missed finalizing them. Do it now. */
5509 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
5510 if (TREE_CODE (t) == VAR_DECL
5511 && TREE_STATIC (t)
5512 && !DECL_EXTERNAL (t))
5513 varpool_node::finalize_decl (t);
5514 DECL_SAVED_TREE (child_fn) = NULL;
5515 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5516 gimple_set_body (child_fn, NULL);
5517 TREE_USED (block) = 1;
5519 /* Reset DECL_CONTEXT on function arguments. */
5520 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
5521 DECL_CONTEXT (t) = child_fn;
5523 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5524 so that it can be moved to the child function. */
5525 gsi = gsi_last_bb (entry_bb);
5526 stmt = gsi_stmt (gsi);
5527 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
5528 || gimple_code (stmt) == GIMPLE_OMP_TASK));
5529 e = split_block (entry_bb, stmt);
5530 gsi_remove (&gsi, true);
5531 entry_bb = e->dest;
5532 edge e2 = NULL;
5533 if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
5534 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
5535 else
5537 e2 = make_edge (e->src, BRANCH_EDGE (entry_bb)->dest, EDGE_ABNORMAL);
5538 gcc_assert (e2->dest == region->exit);
5539 remove_edge (BRANCH_EDGE (entry_bb));
5540 set_immediate_dominator (CDI_DOMINATORS, e2->dest, e->src);
5541 gsi = gsi_last_bb (region->exit);
5542 gcc_assert (!gsi_end_p (gsi)
5543 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
5544 gsi_remove (&gsi, true);
5547 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
5548 if (exit_bb)
5550 gsi = gsi_last_bb (exit_bb);
5551 gcc_assert (!gsi_end_p (gsi)
5552 && (gimple_code (gsi_stmt (gsi))
5553 == (e2 ? GIMPLE_OMP_CONTINUE : GIMPLE_OMP_RETURN)));
5554 stmt = gimple_build_return (NULL);
5555 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
5556 gsi_remove (&gsi, true);
5559 /* Move the parallel region into CHILD_CFUN. */
5561 if (gimple_in_ssa_p (cfun))
5563 init_tree_ssa (child_cfun);
5564 init_ssa_operands (child_cfun);
5565 child_cfun->gimple_df->in_ssa_p = true;
5566 block = NULL_TREE;
5568 else
5569 block = gimple_block (entry_stmt);
5571 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
5572 if (exit_bb)
5573 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
5574 if (e2)
5576 basic_block dest_bb = e2->dest;
5577 if (!exit_bb)
5578 make_edge (new_bb, dest_bb, EDGE_FALLTHRU);
5579 remove_edge (e2);
5580 set_immediate_dominator (CDI_DOMINATORS, dest_bb, new_bb);
5582 /* When the OMP expansion process cannot guarantee an up-to-date
5583 loop tree arrange for the child function to fixup loops. */
5584 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
5585 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
5587 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5588 num = vec_safe_length (child_cfun->local_decls);
5589 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
5591 t = (*child_cfun->local_decls)[srcidx];
5592 if (DECL_CONTEXT (t) == cfun->decl)
5593 continue;
5594 if (srcidx != dstidx)
5595 (*child_cfun->local_decls)[dstidx] = t;
5596 dstidx++;
5598 if (dstidx != num)
5599 vec_safe_truncate (child_cfun->local_decls, dstidx);
5601 /* Inform the callgraph about the new function. */
5602 child_cfun->curr_properties = cfun->curr_properties;
5603 child_cfun->has_simduid_loops |= cfun->has_simduid_loops;
5604 child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops;
5605 cgraph_node::add_new_function (child_fn, true);
5606 cgraph_node::get (child_fn)->parallelized_function = 1;
5608 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5609 fixed in a following pass. */
5610 push_cfun (child_cfun);
5611 if (optimize)
5612 optimize_omp_library_calls (entry_stmt);
5613 cgraph_edge::rebuild_edges ();
5615 /* Some EH regions might become dead, see PR34608. If
5616 pass_cleanup_cfg isn't the first pass to happen with the
5617 new child, these dead EH edges might cause problems.
5618 Clean them up now. */
5619 if (flag_exceptions)
5621 basic_block bb;
5622 bool changed = false;
5624 FOR_EACH_BB_FN (bb, cfun)
5625 changed |= gimple_purge_dead_eh_edges (bb);
5626 if (changed)
5627 cleanup_tree_cfg ();
5629 if (gimple_in_ssa_p (cfun))
5630 update_ssa (TODO_update_ssa);
5631 pop_cfun ();
5634 /* Emit a library call to launch the children threads. */
5635 if (is_cilk_for)
5636 expand_cilk_for_call (new_bb,
5637 as_a <gomp_parallel *> (entry_stmt), ws_args);
5638 else if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
5639 expand_parallel_call (region, new_bb,
5640 as_a <gomp_parallel *> (entry_stmt), ws_args);
5641 else
5642 expand_task_call (new_bb, as_a <gomp_task *> (entry_stmt));
5643 if (gimple_in_ssa_p (cfun))
5644 update_ssa (TODO_update_ssa_only_virtuals);
5648 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5649 of the combined collapse > 1 loop constructs, generate code like:
5650 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5651 if (cond3 is <)
5652 adj = STEP3 - 1;
5653 else
5654 adj = STEP3 + 1;
5655 count3 = (adj + N32 - N31) / STEP3;
5656 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5657 if (cond2 is <)
5658 adj = STEP2 - 1;
5659 else
5660 adj = STEP2 + 1;
5661 count2 = (adj + N22 - N21) / STEP2;
5662 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5663 if (cond1 is <)
5664 adj = STEP1 - 1;
5665 else
5666 adj = STEP1 + 1;
5667 count1 = (adj + N12 - N11) / STEP1;
5668 count = count1 * count2 * count3;
5669 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5670 count = 0;
5671 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5672 of the combined loop constructs, just initialize COUNTS array
5673 from the _looptemp_ clauses. */
5675 /* NOTE: It *could* be better to moosh all of the BBs together,
5676 creating one larger BB with all the computation and the unexpected
5677 jump at the end. I.e.
5679 bool zero3, zero2, zero1, zero;
5681 zero3 = N32 c3 N31;
5682 count3 = (N32 - N31) /[cl] STEP3;
5683 zero2 = N22 c2 N21;
5684 count2 = (N22 - N21) /[cl] STEP2;
5685 zero1 = N12 c1 N11;
5686 count1 = (N12 - N11) /[cl] STEP1;
5687 zero = zero3 || zero2 || zero1;
5688 count = count1 * count2 * count3;
5689 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5691 After all, we expect the zero=false, and thus we expect to have to
5692 evaluate all of the comparison expressions, so short-circuiting
5693 oughtn't be a win. Since the condition isn't protecting a
5694 denominator, we're not concerned about divide-by-zero, so we can
5695 fully evaluate count even if a numerator turned out to be wrong.
5697 It seems like putting this all together would create much better
5698 scheduling opportunities, and less pressure on the chip's branch
5699 predictor. */
5701 static void
5702 expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
5703 basic_block &entry_bb, tree *counts,
5704 basic_block &zero_iter_bb, int &first_zero_iter,
5705 basic_block &l2_dom_bb)
5707 tree t, type = TREE_TYPE (fd->loop.v);
5708 edge e, ne;
5709 int i;
5711 /* Collapsed loops need work for expansion into SSA form. */
5712 gcc_assert (!gimple_in_ssa_p (cfun));
5714 if (gimple_omp_for_combined_into_p (fd->for_stmt)
5715 && TREE_CODE (fd->loop.n2) != INTEGER_CST)
5717 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5718 isn't supposed to be handled, as the inner loop doesn't
5719 use it. */
5720 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
5721 OMP_CLAUSE__LOOPTEMP_);
5722 gcc_assert (innerc);
5723 for (i = 0; i < fd->collapse; i++)
5725 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5726 OMP_CLAUSE__LOOPTEMP_);
5727 gcc_assert (innerc);
5728 if (i)
5729 counts[i] = OMP_CLAUSE_DECL (innerc);
5730 else
5731 counts[0] = NULL_TREE;
5733 return;
5736 for (i = 0; i < fd->collapse; i++)
5738 tree itype = TREE_TYPE (fd->loops[i].v);
5740 if (SSA_VAR_P (fd->loop.n2)
5741 && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
5742 fold_convert (itype, fd->loops[i].n1),
5743 fold_convert (itype, fd->loops[i].n2)))
5744 == NULL_TREE || !integer_onep (t)))
5746 gcond *cond_stmt;
5747 tree n1, n2;
5748 n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1));
5749 n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE,
5750 true, GSI_SAME_STMT);
5751 n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
5752 n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE,
5753 true, GSI_SAME_STMT);
5754 cond_stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
5755 NULL_TREE, NULL_TREE);
5756 gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT);
5757 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
5758 expand_omp_regimplify_p, NULL, NULL)
5759 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
5760 expand_omp_regimplify_p, NULL, NULL))
5762 *gsi = gsi_for_stmt (cond_stmt);
5763 gimple_regimplify_operands (cond_stmt, gsi);
5765 e = split_block (entry_bb, cond_stmt);
5766 if (zero_iter_bb == NULL)
5768 gassign *assign_stmt;
5769 first_zero_iter = i;
5770 zero_iter_bb = create_empty_bb (entry_bb);
5771 add_bb_to_loop (zero_iter_bb, entry_bb->loop_father);
5772 *gsi = gsi_after_labels (zero_iter_bb);
5773 assign_stmt = gimple_build_assign (fd->loop.n2,
5774 build_zero_cst (type));
5775 gsi_insert_before (gsi, assign_stmt, GSI_SAME_STMT);
5776 set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb,
5777 entry_bb);
5779 ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
5780 ne->probability = REG_BR_PROB_BASE / 2000 - 1;
5781 e->flags = EDGE_TRUE_VALUE;
5782 e->probability = REG_BR_PROB_BASE - ne->probability;
5783 if (l2_dom_bb == NULL)
5784 l2_dom_bb = entry_bb;
5785 entry_bb = e->dest;
5786 *gsi = gsi_last_bb (entry_bb);
5789 if (POINTER_TYPE_P (itype))
5790 itype = signed_type_for (itype);
5791 t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
5792 ? -1 : 1));
5793 t = fold_build2 (PLUS_EXPR, itype,
5794 fold_convert (itype, fd->loops[i].step), t);
5795 t = fold_build2 (PLUS_EXPR, itype, t,
5796 fold_convert (itype, fd->loops[i].n2));
5797 t = fold_build2 (MINUS_EXPR, itype, t,
5798 fold_convert (itype, fd->loops[i].n1));
5799 /* ?? We could probably use CEIL_DIV_EXPR instead of
5800 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5801 generate the same code in the end because generically we
5802 don't know that the values involved must be negative for
5803 GT?? */
5804 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
5805 t = fold_build2 (TRUNC_DIV_EXPR, itype,
5806 fold_build1 (NEGATE_EXPR, itype, t),
5807 fold_build1 (NEGATE_EXPR, itype,
5808 fold_convert (itype,
5809 fd->loops[i].step)));
5810 else
5811 t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
5812 fold_convert (itype, fd->loops[i].step));
5813 t = fold_convert (type, t);
5814 if (TREE_CODE (t) == INTEGER_CST)
5815 counts[i] = t;
5816 else
5818 counts[i] = create_tmp_reg (type, ".count");
5819 expand_omp_build_assign (gsi, counts[i], t);
5821 if (SSA_VAR_P (fd->loop.n2))
5823 if (i == 0)
5824 t = counts[0];
5825 else
5826 t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]);
5827 expand_omp_build_assign (gsi, fd->loop.n2, t);
5833 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5834 T = V;
5835 V3 = N31 + (T % count3) * STEP3;
5836 T = T / count3;
5837 V2 = N21 + (T % count2) * STEP2;
5838 T = T / count2;
5839 V1 = N11 + T * STEP1;
5840 if this loop doesn't have an inner loop construct combined with it.
5841 If it does have an inner loop construct combined with it and the
5842 iteration count isn't known constant, store values from counts array
5843 into its _looptemp_ temporaries instead. */
5845 static void
5846 expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
5847 tree *counts, gimple inner_stmt, tree startvar)
5849 int i;
5850 if (gimple_omp_for_combined_p (fd->for_stmt))
5852 /* If fd->loop.n2 is constant, then no propagation of the counts
5853 is needed, they are constant. */
5854 if (TREE_CODE (fd->loop.n2) == INTEGER_CST)
5855 return;
5857 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
5858 ? gimple_omp_parallel_clauses (inner_stmt)
5859 : gimple_omp_for_clauses (inner_stmt);
5860 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5861 isn't supposed to be handled, as the inner loop doesn't
5862 use it. */
5863 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
5864 gcc_assert (innerc);
5865 for (i = 0; i < fd->collapse; i++)
5867 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5868 OMP_CLAUSE__LOOPTEMP_);
5869 gcc_assert (innerc);
5870 if (i)
5872 tree tem = OMP_CLAUSE_DECL (innerc);
5873 tree t = fold_convert (TREE_TYPE (tem), counts[i]);
5874 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
5875 false, GSI_CONTINUE_LINKING);
5876 gassign *stmt = gimple_build_assign (tem, t);
5877 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5880 return;
5883 tree type = TREE_TYPE (fd->loop.v);
5884 tree tem = create_tmp_reg (type, ".tem");
5885 gassign *stmt = gimple_build_assign (tem, startvar);
5886 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5888 for (i = fd->collapse - 1; i >= 0; i--)
5890 tree vtype = TREE_TYPE (fd->loops[i].v), itype, t;
5891 itype = vtype;
5892 if (POINTER_TYPE_P (vtype))
5893 itype = signed_type_for (vtype);
5894 if (i != 0)
5895 t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]);
5896 else
5897 t = tem;
5898 t = fold_convert (itype, t);
5899 t = fold_build2 (MULT_EXPR, itype, t,
5900 fold_convert (itype, fd->loops[i].step));
5901 if (POINTER_TYPE_P (vtype))
5902 t = fold_build_pointer_plus (fd->loops[i].n1, t);
5903 else
5904 t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
5905 t = force_gimple_operand_gsi (gsi, t,
5906 DECL_P (fd->loops[i].v)
5907 && TREE_ADDRESSABLE (fd->loops[i].v),
5908 NULL_TREE, false,
5909 GSI_CONTINUE_LINKING);
5910 stmt = gimple_build_assign (fd->loops[i].v, t);
5911 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5912 if (i != 0)
5914 t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]);
5915 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
5916 false, GSI_CONTINUE_LINKING);
5917 stmt = gimple_build_assign (tem, t);
5918 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5924 /* Helper function for expand_omp_for_*. Generate code like:
5925 L10:
5926 V3 += STEP3;
5927 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5928 L11:
5929 V3 = N31;
5930 V2 += STEP2;
5931 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5932 L12:
5933 V2 = N21;
5934 V1 += STEP1;
5935 goto BODY_BB; */
5937 static basic_block
5938 extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb,
5939 basic_block body_bb)
5941 basic_block last_bb, bb, collapse_bb = NULL;
5942 int i;
5943 gimple_stmt_iterator gsi;
5944 edge e;
5945 tree t;
5946 gimple stmt;
5948 last_bb = cont_bb;
5949 for (i = fd->collapse - 1; i >= 0; i--)
5951 tree vtype = TREE_TYPE (fd->loops[i].v);
5953 bb = create_empty_bb (last_bb);
5954 add_bb_to_loop (bb, last_bb->loop_father);
5955 gsi = gsi_start_bb (bb);
5957 if (i < fd->collapse - 1)
5959 e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
5960 e->probability = REG_BR_PROB_BASE / 8;
5962 t = fd->loops[i + 1].n1;
5963 t = force_gimple_operand_gsi (&gsi, t,
5964 DECL_P (fd->loops[i + 1].v)
5965 && TREE_ADDRESSABLE (fd->loops[i
5966 + 1].v),
5967 NULL_TREE, false,
5968 GSI_CONTINUE_LINKING);
5969 stmt = gimple_build_assign (fd->loops[i + 1].v, t);
5970 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5972 else
5973 collapse_bb = bb;
5975 set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
5977 if (POINTER_TYPE_P (vtype))
5978 t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
5979 else
5980 t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step);
5981 t = force_gimple_operand_gsi (&gsi, t,
5982 DECL_P (fd->loops[i].v)
5983 && TREE_ADDRESSABLE (fd->loops[i].v),
5984 NULL_TREE, false, GSI_CONTINUE_LINKING);
5985 stmt = gimple_build_assign (fd->loops[i].v, t);
5986 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5988 if (i > 0)
5990 t = fd->loops[i].n2;
5991 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5992 false, GSI_CONTINUE_LINKING);
5993 tree v = fd->loops[i].v;
5994 if (DECL_P (v) && TREE_ADDRESSABLE (v))
5995 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
5996 false, GSI_CONTINUE_LINKING);
5997 t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
5998 stmt = gimple_build_cond_empty (t);
5999 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6000 e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
6001 e->probability = REG_BR_PROB_BASE * 7 / 8;
6003 else
6004 make_edge (bb, body_bb, EDGE_FALLTHRU);
6005 last_bb = bb;
6008 return collapse_bb;
6012 /* A subroutine of expand_omp_for. Generate code for a parallel
6013 loop with any schedule. Given parameters:
6015 for (V = N1; V cond N2; V += STEP) BODY;
6017 where COND is "<" or ">", we generate pseudocode
6019 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
6020 if (more) goto L0; else goto L3;
6022 V = istart0;
6023 iend = iend0;
6025 BODY;
6026 V += STEP;
6027 if (V cond iend) goto L1; else goto L2;
6029 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6032 If this is a combined omp parallel loop, instead of the call to
6033 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
6034 If this is gimple_omp_for_combined_p loop, then instead of assigning
6035 V and iend in L0 we assign the first two _looptemp_ clause decls of the
6036 inner GIMPLE_OMP_FOR and V += STEP; and
6037 if (V cond iend) goto L1; else goto L2; are removed.
6039 For collapsed loops, given parameters:
6040 collapse(3)
6041 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6042 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6043 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6044 BODY;
6046 we generate pseudocode
6048 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6049 if (cond3 is <)
6050 adj = STEP3 - 1;
6051 else
6052 adj = STEP3 + 1;
6053 count3 = (adj + N32 - N31) / STEP3;
6054 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6055 if (cond2 is <)
6056 adj = STEP2 - 1;
6057 else
6058 adj = STEP2 + 1;
6059 count2 = (adj + N22 - N21) / STEP2;
6060 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6061 if (cond1 is <)
6062 adj = STEP1 - 1;
6063 else
6064 adj = STEP1 + 1;
6065 count1 = (adj + N12 - N11) / STEP1;
6066 count = count1 * count2 * count3;
6067 goto Z1;
6069 count = 0;
6071 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6072 if (more) goto L0; else goto L3;
6074 V = istart0;
6075 T = V;
6076 V3 = N31 + (T % count3) * STEP3;
6077 T = T / count3;
6078 V2 = N21 + (T % count2) * STEP2;
6079 T = T / count2;
6080 V1 = N11 + T * STEP1;
6081 iend = iend0;
6083 BODY;
6084 V += 1;
6085 if (V < iend) goto L10; else goto L2;
6086 L10:
6087 V3 += STEP3;
6088 if (V3 cond3 N32) goto L1; else goto L11;
6089 L11:
6090 V3 = N31;
6091 V2 += STEP2;
6092 if (V2 cond2 N22) goto L1; else goto L12;
6093 L12:
6094 V2 = N21;
6095 V1 += STEP1;
6096 goto L1;
6098 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6103 static void
6104 expand_omp_for_generic (struct omp_region *region,
6105 struct omp_for_data *fd,
6106 enum built_in_function start_fn,
6107 enum built_in_function next_fn,
6108 gimple inner_stmt)
6110 tree type, istart0, iend0, iend;
6111 tree t, vmain, vback, bias = NULL_TREE;
6112 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
6113 basic_block l2_bb = NULL, l3_bb = NULL;
6114 gimple_stmt_iterator gsi;
6115 gassign *assign_stmt;
6116 bool in_combined_parallel = is_combined_parallel (region);
6117 bool broken_loop = region->cont == NULL;
6118 edge e, ne;
6119 tree *counts = NULL;
6120 int i;
6122 gcc_assert (!broken_loop || !in_combined_parallel);
6123 gcc_assert (fd->iter_type == long_integer_type_node
6124 || !in_combined_parallel);
6126 type = TREE_TYPE (fd->loop.v);
6127 istart0 = create_tmp_var (fd->iter_type, ".istart0");
6128 iend0 = create_tmp_var (fd->iter_type, ".iend0");
6129 TREE_ADDRESSABLE (istart0) = 1;
6130 TREE_ADDRESSABLE (iend0) = 1;
6132 /* See if we need to bias by LLONG_MIN. */
6133 if (fd->iter_type == long_long_unsigned_type_node
6134 && TREE_CODE (type) == INTEGER_TYPE
6135 && !TYPE_UNSIGNED (type))
6137 tree n1, n2;
6139 if (fd->loop.cond_code == LT_EXPR)
6141 n1 = fd->loop.n1;
6142 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
6144 else
6146 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
6147 n2 = fd->loop.n1;
6149 if (TREE_CODE (n1) != INTEGER_CST
6150 || TREE_CODE (n2) != INTEGER_CST
6151 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
6152 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
6155 entry_bb = region->entry;
6156 cont_bb = region->cont;
6157 collapse_bb = NULL;
6158 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6159 gcc_assert (broken_loop
6160 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
6161 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
6162 l1_bb = single_succ (l0_bb);
6163 if (!broken_loop)
6165 l2_bb = create_empty_bb (cont_bb);
6166 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb);
6167 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6169 else
6170 l2_bb = NULL;
6171 l3_bb = BRANCH_EDGE (entry_bb)->dest;
6172 exit_bb = region->exit;
6174 gsi = gsi_last_bb (entry_bb);
6176 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6177 if (fd->collapse > 1)
6179 int first_zero_iter = -1;
6180 basic_block zero_iter_bb = NULL, l2_dom_bb = NULL;
6182 counts = XALLOCAVEC (tree, fd->collapse);
6183 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6184 zero_iter_bb, first_zero_iter,
6185 l2_dom_bb);
6187 if (zero_iter_bb)
6189 /* Some counts[i] vars might be uninitialized if
6190 some loop has zero iterations. But the body shouldn't
6191 be executed in that case, so just avoid uninit warnings. */
6192 for (i = first_zero_iter; i < fd->collapse; i++)
6193 if (SSA_VAR_P (counts[i]))
6194 TREE_NO_WARNING (counts[i]) = 1;
6195 gsi_prev (&gsi);
6196 e = split_block (entry_bb, gsi_stmt (gsi));
6197 entry_bb = e->dest;
6198 make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU);
6199 gsi = gsi_last_bb (entry_bb);
6200 set_immediate_dominator (CDI_DOMINATORS, entry_bb,
6201 get_immediate_dominator (CDI_DOMINATORS,
6202 zero_iter_bb));
6205 if (in_combined_parallel)
6207 /* In a combined parallel loop, emit a call to
6208 GOMP_loop_foo_next. */
6209 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
6210 build_fold_addr_expr (istart0),
6211 build_fold_addr_expr (iend0));
6213 else
6215 tree t0, t1, t2, t3, t4;
6216 /* If this is not a combined parallel loop, emit a call to
6217 GOMP_loop_foo_start in ENTRY_BB. */
6218 t4 = build_fold_addr_expr (iend0);
6219 t3 = build_fold_addr_expr (istart0);
6220 t2 = fold_convert (fd->iter_type, fd->loop.step);
6221 t1 = fd->loop.n2;
6222 t0 = fd->loop.n1;
6223 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6225 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6226 OMP_CLAUSE__LOOPTEMP_);
6227 gcc_assert (innerc);
6228 t0 = OMP_CLAUSE_DECL (innerc);
6229 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6230 OMP_CLAUSE__LOOPTEMP_);
6231 gcc_assert (innerc);
6232 t1 = OMP_CLAUSE_DECL (innerc);
6234 if (POINTER_TYPE_P (TREE_TYPE (t0))
6235 && TYPE_PRECISION (TREE_TYPE (t0))
6236 != TYPE_PRECISION (fd->iter_type))
6238 /* Avoid casting pointers to integer of a different size. */
6239 tree itype = signed_type_for (type);
6240 t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
6241 t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
6243 else
6245 t1 = fold_convert (fd->iter_type, t1);
6246 t0 = fold_convert (fd->iter_type, t0);
6248 if (bias)
6250 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
6251 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
6253 if (fd->iter_type == long_integer_type_node)
6255 if (fd->chunk_size)
6257 t = fold_convert (fd->iter_type, fd->chunk_size);
6258 t = build_call_expr (builtin_decl_explicit (start_fn),
6259 6, t0, t1, t2, t, t3, t4);
6261 else
6262 t = build_call_expr (builtin_decl_explicit (start_fn),
6263 5, t0, t1, t2, t3, t4);
6265 else
6267 tree t5;
6268 tree c_bool_type;
6269 tree bfn_decl;
6271 /* The GOMP_loop_ull_*start functions have additional boolean
6272 argument, true for < loops and false for > loops.
6273 In Fortran, the C bool type can be different from
6274 boolean_type_node. */
6275 bfn_decl = builtin_decl_explicit (start_fn);
6276 c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl));
6277 t5 = build_int_cst (c_bool_type,
6278 fd->loop.cond_code == LT_EXPR ? 1 : 0);
6279 if (fd->chunk_size)
6281 tree bfn_decl = builtin_decl_explicit (start_fn);
6282 t = fold_convert (fd->iter_type, fd->chunk_size);
6283 t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
6285 else
6286 t = build_call_expr (builtin_decl_explicit (start_fn),
6287 6, t5, t0, t1, t2, t3, t4);
6290 if (TREE_TYPE (t) != boolean_type_node)
6291 t = fold_build2 (NE_EXPR, boolean_type_node,
6292 t, build_int_cst (TREE_TYPE (t), 0));
6293 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6294 true, GSI_SAME_STMT);
6295 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6297 /* Remove the GIMPLE_OMP_FOR statement. */
6298 gsi_remove (&gsi, true);
6300 /* Iteration setup for sequential loop goes in L0_BB. */
6301 tree startvar = fd->loop.v;
6302 tree endvar = NULL_TREE;
6304 if (gimple_omp_for_combined_p (fd->for_stmt))
6306 gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR
6307 && gimple_omp_for_kind (inner_stmt)
6308 == GF_OMP_FOR_KIND_SIMD);
6309 tree innerc = find_omp_clause (gimple_omp_for_clauses (inner_stmt),
6310 OMP_CLAUSE__LOOPTEMP_);
6311 gcc_assert (innerc);
6312 startvar = OMP_CLAUSE_DECL (innerc);
6313 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6314 OMP_CLAUSE__LOOPTEMP_);
6315 gcc_assert (innerc);
6316 endvar = OMP_CLAUSE_DECL (innerc);
6319 gsi = gsi_start_bb (l0_bb);
6320 t = istart0;
6321 if (bias)
6322 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
6323 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
6324 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
6325 t = fold_convert (TREE_TYPE (startvar), t);
6326 t = force_gimple_operand_gsi (&gsi, t,
6327 DECL_P (startvar)
6328 && TREE_ADDRESSABLE (startvar),
6329 NULL_TREE, false, GSI_CONTINUE_LINKING);
6330 assign_stmt = gimple_build_assign (startvar, t);
6331 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6333 t = iend0;
6334 if (bias)
6335 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
6336 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
6337 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
6338 t = fold_convert (TREE_TYPE (startvar), t);
6339 iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6340 false, GSI_CONTINUE_LINKING);
6341 if (endvar)
6343 assign_stmt = gimple_build_assign (endvar, iend);
6344 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6345 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (iend)))
6346 assign_stmt = gimple_build_assign (fd->loop.v, iend);
6347 else
6348 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, iend);
6349 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6351 if (fd->collapse > 1)
6352 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
6354 if (!broken_loop)
6356 /* Code to control the increment and predicate for the sequential
6357 loop goes in the CONT_BB. */
6358 gsi = gsi_last_bb (cont_bb);
6359 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
6360 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
6361 vmain = gimple_omp_continue_control_use (cont_stmt);
6362 vback = gimple_omp_continue_control_def (cont_stmt);
6364 if (!gimple_omp_for_combined_p (fd->for_stmt))
6366 if (POINTER_TYPE_P (type))
6367 t = fold_build_pointer_plus (vmain, fd->loop.step);
6368 else
6369 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
6370 t = force_gimple_operand_gsi (&gsi, t,
6371 DECL_P (vback)
6372 && TREE_ADDRESSABLE (vback),
6373 NULL_TREE, true, GSI_SAME_STMT);
6374 assign_stmt = gimple_build_assign (vback, t);
6375 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6377 t = build2 (fd->loop.cond_code, boolean_type_node,
6378 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
6379 iend);
6380 gcond *cond_stmt = gimple_build_cond_empty (t);
6381 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
6384 /* Remove GIMPLE_OMP_CONTINUE. */
6385 gsi_remove (&gsi, true);
6387 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6388 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, l1_bb);
6390 /* Emit code to get the next parallel iteration in L2_BB. */
6391 gsi = gsi_start_bb (l2_bb);
6393 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
6394 build_fold_addr_expr (istart0),
6395 build_fold_addr_expr (iend0));
6396 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6397 false, GSI_CONTINUE_LINKING);
6398 if (TREE_TYPE (t) != boolean_type_node)
6399 t = fold_build2 (NE_EXPR, boolean_type_node,
6400 t, build_int_cst (TREE_TYPE (t), 0));
6401 gcond *cond_stmt = gimple_build_cond_empty (t);
6402 gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
6405 /* Add the loop cleanup function. */
6406 gsi = gsi_last_bb (exit_bb);
6407 if (gimple_omp_return_nowait_p (gsi_stmt (gsi)))
6408 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
6409 else if (gimple_omp_return_lhs (gsi_stmt (gsi)))
6410 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
6411 else
6412 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
6413 gcall *call_stmt = gimple_build_call (t, 0);
6414 if (gimple_omp_return_lhs (gsi_stmt (gsi)))
6415 gimple_call_set_lhs (call_stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
6416 gsi_insert_after (&gsi, call_stmt, GSI_SAME_STMT);
6417 gsi_remove (&gsi, true);
6419 /* Connect the new blocks. */
6420 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
6421 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
6423 if (!broken_loop)
6425 gimple_seq phis;
6427 e = find_edge (cont_bb, l3_bb);
6428 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
6430 phis = phi_nodes (l3_bb);
6431 for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi))
6433 gimple phi = gsi_stmt (gsi);
6434 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
6435 PHI_ARG_DEF_FROM_EDGE (phi, e));
6437 remove_edge (e);
6439 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
6440 add_bb_to_loop (l2_bb, cont_bb->loop_father);
6441 e = find_edge (cont_bb, l1_bb);
6442 if (gimple_omp_for_combined_p (fd->for_stmt))
6444 remove_edge (e);
6445 e = NULL;
6447 else if (fd->collapse > 1)
6449 remove_edge (e);
6450 e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6452 else
6453 e->flags = EDGE_TRUE_VALUE;
6454 if (e)
6456 e->probability = REG_BR_PROB_BASE * 7 / 8;
6457 find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8;
6459 else
6461 e = find_edge (cont_bb, l2_bb);
6462 e->flags = EDGE_FALLTHRU;
6464 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
6466 set_immediate_dominator (CDI_DOMINATORS, l2_bb,
6467 recompute_dominator (CDI_DOMINATORS, l2_bb));
6468 set_immediate_dominator (CDI_DOMINATORS, l3_bb,
6469 recompute_dominator (CDI_DOMINATORS, l3_bb));
6470 set_immediate_dominator (CDI_DOMINATORS, l0_bb,
6471 recompute_dominator (CDI_DOMINATORS, l0_bb));
6472 set_immediate_dominator (CDI_DOMINATORS, l1_bb,
6473 recompute_dominator (CDI_DOMINATORS, l1_bb));
6475 struct loop *outer_loop = alloc_loop ();
6476 outer_loop->header = l0_bb;
6477 outer_loop->latch = l2_bb;
6478 add_loop (outer_loop, l0_bb->loop_father);
6480 if (!gimple_omp_for_combined_p (fd->for_stmt))
6482 struct loop *loop = alloc_loop ();
6483 loop->header = l1_bb;
6484 /* The loop may have multiple latches. */
6485 add_loop (loop, outer_loop);
6491 /* A subroutine of expand_omp_for. Generate code for a parallel
6492 loop with static schedule and no specified chunk size. Given
6493 parameters:
6495 for (V = N1; V cond N2; V += STEP) BODY;
6497 where COND is "<" or ">", we generate pseudocode
6499 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6500 if (cond is <)
6501 adj = STEP - 1;
6502 else
6503 adj = STEP + 1;
6504 if ((__typeof (V)) -1 > 0 && cond is >)
6505 n = -(adj + N2 - N1) / -STEP;
6506 else
6507 n = (adj + N2 - N1) / STEP;
6508 q = n / nthreads;
6509 tt = n % nthreads;
6510 if (threadid < tt) goto L3; else goto L4;
6512 tt = 0;
6513 q = q + 1;
6515 s0 = q * threadid + tt;
6516 e0 = s0 + q;
6517 V = s0 * STEP + N1;
6518 if (s0 >= e0) goto L2; else goto L0;
6520 e = e0 * STEP + N1;
6522 BODY;
6523 V += STEP;
6524 if (V cond e) goto L1;
6528 static void
6529 expand_omp_for_static_nochunk (struct omp_region *region,
6530 struct omp_for_data *fd,
6531 gimple inner_stmt)
6533 tree n, q, s0, e0, e, t, tt, nthreads, threadid;
6534 tree type, itype, vmain, vback;
6535 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
6536 basic_block body_bb, cont_bb, collapse_bb = NULL;
6537 basic_block fin_bb;
6538 gimple_stmt_iterator gsi;
6539 edge ep;
6540 bool broken_loop = region->cont == NULL;
6541 tree *counts = NULL;
6542 tree n1, n2, step;
6544 gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt)
6545 != GF_OMP_FOR_KIND_OACC_LOOP)
6546 || !inner_stmt);
6548 itype = type = TREE_TYPE (fd->loop.v);
6549 if (POINTER_TYPE_P (type))
6550 itype = signed_type_for (type);
6552 entry_bb = region->entry;
6553 cont_bb = region->cont;
6554 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6555 fin_bb = BRANCH_EDGE (entry_bb)->dest;
6556 gcc_assert (broken_loop
6557 || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
6558 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
6559 body_bb = single_succ (seq_start_bb);
6560 if (!broken_loop)
6562 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
6563 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6565 exit_bb = region->exit;
6567 /* Iteration space partitioning goes in ENTRY_BB. */
6568 gsi = gsi_last_bb (entry_bb);
6569 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6571 if (fd->collapse > 1)
6573 int first_zero_iter = -1;
6574 basic_block l2_dom_bb = NULL;
6576 counts = XALLOCAVEC (tree, fd->collapse);
6577 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6578 fin_bb, first_zero_iter,
6579 l2_dom_bb);
6580 t = NULL_TREE;
6582 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
6583 t = integer_one_node;
6584 else
6585 t = fold_binary (fd->loop.cond_code, boolean_type_node,
6586 fold_convert (type, fd->loop.n1),
6587 fold_convert (type, fd->loop.n2));
6588 if (fd->collapse == 1
6589 && TYPE_UNSIGNED (type)
6590 && (t == NULL_TREE || !integer_onep (t)))
6592 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
6593 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
6594 true, GSI_SAME_STMT);
6595 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
6596 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
6597 true, GSI_SAME_STMT);
6598 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
6599 NULL_TREE, NULL_TREE);
6600 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
6601 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
6602 expand_omp_regimplify_p, NULL, NULL)
6603 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
6604 expand_omp_regimplify_p, NULL, NULL))
6606 gsi = gsi_for_stmt (cond_stmt);
6607 gimple_regimplify_operands (cond_stmt, &gsi);
6609 ep = split_block (entry_bb, cond_stmt);
6610 ep->flags = EDGE_TRUE_VALUE;
6611 entry_bb = ep->dest;
6612 ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
6613 ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
6614 ep->probability = REG_BR_PROB_BASE / 2000 - 1;
6615 if (gimple_in_ssa_p (cfun))
6617 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
6618 for (gphi_iterator gpi = gsi_start_phis (fin_bb);
6619 !gsi_end_p (gpi); gsi_next (&gpi))
6621 gphi *phi = gpi.phi ();
6622 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
6623 ep, UNKNOWN_LOCATION);
6626 gsi = gsi_last_bb (entry_bb);
6629 switch (gimple_omp_for_kind (fd->for_stmt))
6631 case GF_OMP_FOR_KIND_FOR:
6632 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
6633 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
6634 break;
6635 case GF_OMP_FOR_KIND_DISTRIBUTE:
6636 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS);
6637 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
6638 break;
6639 case GF_OMP_FOR_KIND_OACC_LOOP:
6640 nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS);
6641 threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
6642 break;
6643 default:
6644 gcc_unreachable ();
6646 nthreads = build_call_expr (nthreads, 0);
6647 nthreads = fold_convert (itype, nthreads);
6648 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE,
6649 true, GSI_SAME_STMT);
6650 threadid = build_call_expr (threadid, 0);
6651 threadid = fold_convert (itype, threadid);
6652 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
6653 true, GSI_SAME_STMT);
6655 n1 = fd->loop.n1;
6656 n2 = fd->loop.n2;
6657 step = fd->loop.step;
6658 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6660 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6661 OMP_CLAUSE__LOOPTEMP_);
6662 gcc_assert (innerc);
6663 n1 = OMP_CLAUSE_DECL (innerc);
6664 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6665 OMP_CLAUSE__LOOPTEMP_);
6666 gcc_assert (innerc);
6667 n2 = OMP_CLAUSE_DECL (innerc);
6669 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
6670 true, NULL_TREE, true, GSI_SAME_STMT);
6671 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
6672 true, NULL_TREE, true, GSI_SAME_STMT);
6673 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
6674 true, NULL_TREE, true, GSI_SAME_STMT);
6676 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
6677 t = fold_build2 (PLUS_EXPR, itype, step, t);
6678 t = fold_build2 (PLUS_EXPR, itype, t, n2);
6679 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
6680 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
6681 t = fold_build2 (TRUNC_DIV_EXPR, itype,
6682 fold_build1 (NEGATE_EXPR, itype, t),
6683 fold_build1 (NEGATE_EXPR, itype, step));
6684 else
6685 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
6686 t = fold_convert (itype, t);
6687 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
6689 q = create_tmp_reg (itype, "q");
6690 t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads);
6691 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
6692 gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT);
6694 tt = create_tmp_reg (itype, "tt");
6695 t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads);
6696 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
6697 gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT);
6699 t = build2 (LT_EXPR, boolean_type_node, threadid, tt);
6700 gcond *cond_stmt = gimple_build_cond_empty (t);
6701 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
6703 second_bb = split_block (entry_bb, cond_stmt)->dest;
6704 gsi = gsi_last_bb (second_bb);
6705 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6707 gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)),
6708 GSI_SAME_STMT);
6709 gassign *assign_stmt
6710 = gimple_build_assign (q, PLUS_EXPR, q, build_int_cst (itype, 1));
6711 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6713 third_bb = split_block (second_bb, assign_stmt)->dest;
6714 gsi = gsi_last_bb (third_bb);
6715 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6717 t = build2 (MULT_EXPR, itype, q, threadid);
6718 t = build2 (PLUS_EXPR, itype, t, tt);
6719 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
6721 t = fold_build2 (PLUS_EXPR, itype, s0, q);
6722 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
6724 t = build2 (GE_EXPR, boolean_type_node, s0, e0);
6725 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6727 /* Remove the GIMPLE_OMP_FOR statement. */
6728 gsi_remove (&gsi, true);
6730 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6731 gsi = gsi_start_bb (seq_start_bb);
6733 tree startvar = fd->loop.v;
6734 tree endvar = NULL_TREE;
6736 if (gimple_omp_for_combined_p (fd->for_stmt))
6738 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
6739 ? gimple_omp_parallel_clauses (inner_stmt)
6740 : gimple_omp_for_clauses (inner_stmt);
6741 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
6742 gcc_assert (innerc);
6743 startvar = OMP_CLAUSE_DECL (innerc);
6744 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6745 OMP_CLAUSE__LOOPTEMP_);
6746 gcc_assert (innerc);
6747 endvar = OMP_CLAUSE_DECL (innerc);
6749 t = fold_convert (itype, s0);
6750 t = fold_build2 (MULT_EXPR, itype, t, step);
6751 if (POINTER_TYPE_P (type))
6752 t = fold_build_pointer_plus (n1, t);
6753 else
6754 t = fold_build2 (PLUS_EXPR, type, t, n1);
6755 t = fold_convert (TREE_TYPE (startvar), t);
6756 t = force_gimple_operand_gsi (&gsi, t,
6757 DECL_P (startvar)
6758 && TREE_ADDRESSABLE (startvar),
6759 NULL_TREE, false, GSI_CONTINUE_LINKING);
6760 assign_stmt = gimple_build_assign (startvar, t);
6761 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6763 t = fold_convert (itype, e0);
6764 t = fold_build2 (MULT_EXPR, itype, t, step);
6765 if (POINTER_TYPE_P (type))
6766 t = fold_build_pointer_plus (n1, t);
6767 else
6768 t = fold_build2 (PLUS_EXPR, type, t, n1);
6769 t = fold_convert (TREE_TYPE (startvar), t);
6770 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6771 false, GSI_CONTINUE_LINKING);
6772 if (endvar)
6774 assign_stmt = gimple_build_assign (endvar, e);
6775 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6776 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
6777 assign_stmt = gimple_build_assign (fd->loop.v, e);
6778 else
6779 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
6780 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6782 if (fd->collapse > 1)
6783 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
6785 if (!broken_loop)
6787 /* The code controlling the sequential loop replaces the
6788 GIMPLE_OMP_CONTINUE. */
6789 gsi = gsi_last_bb (cont_bb);
6790 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
6791 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
6792 vmain = gimple_omp_continue_control_use (cont_stmt);
6793 vback = gimple_omp_continue_control_def (cont_stmt);
6795 if (!gimple_omp_for_combined_p (fd->for_stmt))
6797 if (POINTER_TYPE_P (type))
6798 t = fold_build_pointer_plus (vmain, step);
6799 else
6800 t = fold_build2 (PLUS_EXPR, type, vmain, step);
6801 t = force_gimple_operand_gsi (&gsi, t,
6802 DECL_P (vback)
6803 && TREE_ADDRESSABLE (vback),
6804 NULL_TREE, true, GSI_SAME_STMT);
6805 assign_stmt = gimple_build_assign (vback, t);
6806 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6808 t = build2 (fd->loop.cond_code, boolean_type_node,
6809 DECL_P (vback) && TREE_ADDRESSABLE (vback)
6810 ? t : vback, e);
6811 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6814 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6815 gsi_remove (&gsi, true);
6817 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6818 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
6821 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6822 gsi = gsi_last_bb (exit_bb);
6823 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
6825 t = gimple_omp_return_lhs (gsi_stmt (gsi));
6826 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
6827 gcc_checking_assert (t == NULL_TREE);
6828 else
6829 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
6831 gsi_remove (&gsi, true);
6833 /* Connect all the blocks. */
6834 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
6835 ep->probability = REG_BR_PROB_BASE / 4 * 3;
6836 ep = find_edge (entry_bb, second_bb);
6837 ep->flags = EDGE_TRUE_VALUE;
6838 ep->probability = REG_BR_PROB_BASE / 4;
6839 find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
6840 find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
6842 if (!broken_loop)
6844 ep = find_edge (cont_bb, body_bb);
6845 if (gimple_omp_for_combined_p (fd->for_stmt))
6847 remove_edge (ep);
6848 ep = NULL;
6850 else if (fd->collapse > 1)
6852 remove_edge (ep);
6853 ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6855 else
6856 ep->flags = EDGE_TRUE_VALUE;
6857 find_edge (cont_bb, fin_bb)->flags
6858 = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
6861 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
6862 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
6863 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb);
6865 set_immediate_dominator (CDI_DOMINATORS, body_bb,
6866 recompute_dominator (CDI_DOMINATORS, body_bb));
6867 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
6868 recompute_dominator (CDI_DOMINATORS, fin_bb));
6870 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
6872 struct loop *loop = alloc_loop ();
6873 loop->header = body_bb;
6874 if (collapse_bb == NULL)
6875 loop->latch = cont_bb;
6876 add_loop (loop, body_bb->loop_father);
6881 /* A subroutine of expand_omp_for. Generate code for a parallel
6882 loop with static schedule and a specified chunk size. Given
6883 parameters:
6885 for (V = N1; V cond N2; V += STEP) BODY;
6887 where COND is "<" or ">", we generate pseudocode
6889 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6890 if (cond is <)
6891 adj = STEP - 1;
6892 else
6893 adj = STEP + 1;
6894 if ((__typeof (V)) -1 > 0 && cond is >)
6895 n = -(adj + N2 - N1) / -STEP;
6896 else
6897 n = (adj + N2 - N1) / STEP;
6898 trip = 0;
6899 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6900 here so that V is defined
6901 if the loop is not entered
6903 s0 = (trip * nthreads + threadid) * CHUNK;
6904 e0 = min(s0 + CHUNK, n);
6905 if (s0 < n) goto L1; else goto L4;
6907 V = s0 * STEP + N1;
6908 e = e0 * STEP + N1;
6910 BODY;
6911 V += STEP;
6912 if (V cond e) goto L2; else goto L3;
6914 trip += 1;
6915 goto L0;
6919 static void
6920 expand_omp_for_static_chunk (struct omp_region *region,
6921 struct omp_for_data *fd, gimple inner_stmt)
6923 tree n, s0, e0, e, t;
6924 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
6925 tree type, itype, vmain, vback, vextra;
6926 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
6927 basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb;
6928 gimple_stmt_iterator gsi;
6929 edge se;
6930 bool broken_loop = region->cont == NULL;
6931 tree *counts = NULL;
6932 tree n1, n2, step;
6934 gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt)
6935 != GF_OMP_FOR_KIND_OACC_LOOP)
6936 || !inner_stmt);
6938 itype = type = TREE_TYPE (fd->loop.v);
6939 if (POINTER_TYPE_P (type))
6940 itype = signed_type_for (type);
6942 entry_bb = region->entry;
6943 se = split_block (entry_bb, last_stmt (entry_bb));
6944 entry_bb = se->src;
6945 iter_part_bb = se->dest;
6946 cont_bb = region->cont;
6947 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
6948 fin_bb = BRANCH_EDGE (iter_part_bb)->dest;
6949 gcc_assert (broken_loop
6950 || fin_bb == FALLTHRU_EDGE (cont_bb)->dest);
6951 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
6952 body_bb = single_succ (seq_start_bb);
6953 if (!broken_loop)
6955 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
6956 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6957 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
6959 exit_bb = region->exit;
6961 /* Trip and adjustment setup goes in ENTRY_BB. */
6962 gsi = gsi_last_bb (entry_bb);
6963 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6965 if (fd->collapse > 1)
6967 int first_zero_iter = -1;
6968 basic_block l2_dom_bb = NULL;
6970 counts = XALLOCAVEC (tree, fd->collapse);
6971 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6972 fin_bb, first_zero_iter,
6973 l2_dom_bb);
6974 t = NULL_TREE;
6976 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
6977 t = integer_one_node;
6978 else
6979 t = fold_binary (fd->loop.cond_code, boolean_type_node,
6980 fold_convert (type, fd->loop.n1),
6981 fold_convert (type, fd->loop.n2));
6982 if (fd->collapse == 1
6983 && TYPE_UNSIGNED (type)
6984 && (t == NULL_TREE || !integer_onep (t)))
6986 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
6987 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
6988 true, GSI_SAME_STMT);
6989 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
6990 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
6991 true, GSI_SAME_STMT);
6992 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
6993 NULL_TREE, NULL_TREE);
6994 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
6995 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
6996 expand_omp_regimplify_p, NULL, NULL)
6997 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
6998 expand_omp_regimplify_p, NULL, NULL))
7000 gsi = gsi_for_stmt (cond_stmt);
7001 gimple_regimplify_operands (cond_stmt, &gsi);
7003 se = split_block (entry_bb, cond_stmt);
7004 se->flags = EDGE_TRUE_VALUE;
7005 entry_bb = se->dest;
7006 se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
7007 se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
7008 se->probability = REG_BR_PROB_BASE / 2000 - 1;
7009 if (gimple_in_ssa_p (cfun))
7011 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
7012 for (gphi_iterator gpi = gsi_start_phis (fin_bb);
7013 !gsi_end_p (gpi); gsi_next (&gpi))
7015 gphi *phi = gpi.phi ();
7016 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
7017 se, UNKNOWN_LOCATION);
7020 gsi = gsi_last_bb (entry_bb);
7023 switch (gimple_omp_for_kind (fd->for_stmt))
7025 case GF_OMP_FOR_KIND_FOR:
7026 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
7027 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
7028 break;
7029 case GF_OMP_FOR_KIND_DISTRIBUTE:
7030 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS);
7031 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
7032 break;
7033 case GF_OMP_FOR_KIND_OACC_LOOP:
7034 nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS);
7035 threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
7036 break;
7037 default:
7038 gcc_unreachable ();
7040 nthreads = build_call_expr (nthreads, 0);
7041 nthreads = fold_convert (itype, nthreads);
7042 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE,
7043 true, GSI_SAME_STMT);
7044 threadid = build_call_expr (threadid, 0);
7045 threadid = fold_convert (itype, threadid);
7046 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
7047 true, GSI_SAME_STMT);
7049 n1 = fd->loop.n1;
7050 n2 = fd->loop.n2;
7051 step = fd->loop.step;
7052 if (gimple_omp_for_combined_into_p (fd->for_stmt))
7054 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7055 OMP_CLAUSE__LOOPTEMP_);
7056 gcc_assert (innerc);
7057 n1 = OMP_CLAUSE_DECL (innerc);
7058 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7059 OMP_CLAUSE__LOOPTEMP_);
7060 gcc_assert (innerc);
7061 n2 = OMP_CLAUSE_DECL (innerc);
7063 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
7064 true, NULL_TREE, true, GSI_SAME_STMT);
7065 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
7066 true, NULL_TREE, true, GSI_SAME_STMT);
7067 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
7068 true, NULL_TREE, true, GSI_SAME_STMT);
7069 fd->chunk_size
7070 = force_gimple_operand_gsi (&gsi, fold_convert (itype, fd->chunk_size),
7071 true, NULL_TREE, true, GSI_SAME_STMT);
7073 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
7074 t = fold_build2 (PLUS_EXPR, itype, step, t);
7075 t = fold_build2 (PLUS_EXPR, itype, t, n2);
7076 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
7077 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
7078 t = fold_build2 (TRUNC_DIV_EXPR, itype,
7079 fold_build1 (NEGATE_EXPR, itype, t),
7080 fold_build1 (NEGATE_EXPR, itype, step));
7081 else
7082 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
7083 t = fold_convert (itype, t);
7084 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7085 true, GSI_SAME_STMT);
7087 trip_var = create_tmp_reg (itype, ".trip");
7088 if (gimple_in_ssa_p (cfun))
7090 trip_init = make_ssa_name (trip_var);
7091 trip_main = make_ssa_name (trip_var);
7092 trip_back = make_ssa_name (trip_var);
7094 else
7096 trip_init = trip_var;
7097 trip_main = trip_var;
7098 trip_back = trip_var;
7101 gassign *assign_stmt
7102 = gimple_build_assign (trip_init, build_int_cst (itype, 0));
7103 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
7105 t = fold_build2 (MULT_EXPR, itype, threadid, fd->chunk_size);
7106 t = fold_build2 (MULT_EXPR, itype, t, step);
7107 if (POINTER_TYPE_P (type))
7108 t = fold_build_pointer_plus (n1, t);
7109 else
7110 t = fold_build2 (PLUS_EXPR, type, t, n1);
7111 vextra = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7112 true, GSI_SAME_STMT);
7114 /* Remove the GIMPLE_OMP_FOR. */
7115 gsi_remove (&gsi, true);
7117 /* Iteration space partitioning goes in ITER_PART_BB. */
7118 gsi = gsi_last_bb (iter_part_bb);
7120 t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads);
7121 t = fold_build2 (PLUS_EXPR, itype, t, threadid);
7122 t = fold_build2 (MULT_EXPR, itype, t, fd->chunk_size);
7123 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7124 false, GSI_CONTINUE_LINKING);
7126 t = fold_build2 (PLUS_EXPR, itype, s0, fd->chunk_size);
7127 t = fold_build2 (MIN_EXPR, itype, t, n);
7128 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7129 false, GSI_CONTINUE_LINKING);
7131 t = build2 (LT_EXPR, boolean_type_node, s0, n);
7132 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING);
7134 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7135 gsi = gsi_start_bb (seq_start_bb);
7137 tree startvar = fd->loop.v;
7138 tree endvar = NULL_TREE;
7140 if (gimple_omp_for_combined_p (fd->for_stmt))
7142 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
7143 ? gimple_omp_parallel_clauses (inner_stmt)
7144 : gimple_omp_for_clauses (inner_stmt);
7145 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
7146 gcc_assert (innerc);
7147 startvar = OMP_CLAUSE_DECL (innerc);
7148 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7149 OMP_CLAUSE__LOOPTEMP_);
7150 gcc_assert (innerc);
7151 endvar = OMP_CLAUSE_DECL (innerc);
7154 t = fold_convert (itype, s0);
7155 t = fold_build2 (MULT_EXPR, itype, t, step);
7156 if (POINTER_TYPE_P (type))
7157 t = fold_build_pointer_plus (n1, t);
7158 else
7159 t = fold_build2 (PLUS_EXPR, type, t, n1);
7160 t = fold_convert (TREE_TYPE (startvar), t);
7161 t = force_gimple_operand_gsi (&gsi, t,
7162 DECL_P (startvar)
7163 && TREE_ADDRESSABLE (startvar),
7164 NULL_TREE, false, GSI_CONTINUE_LINKING);
7165 assign_stmt = gimple_build_assign (startvar, t);
7166 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7168 t = fold_convert (itype, e0);
7169 t = fold_build2 (MULT_EXPR, itype, t, step);
7170 if (POINTER_TYPE_P (type))
7171 t = fold_build_pointer_plus (n1, t);
7172 else
7173 t = fold_build2 (PLUS_EXPR, type, t, n1);
7174 t = fold_convert (TREE_TYPE (startvar), t);
7175 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7176 false, GSI_CONTINUE_LINKING);
7177 if (endvar)
7179 assign_stmt = gimple_build_assign (endvar, e);
7180 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7181 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
7182 assign_stmt = gimple_build_assign (fd->loop.v, e);
7183 else
7184 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
7185 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7187 if (fd->collapse > 1)
7188 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
7190 if (!broken_loop)
7192 /* The code controlling the sequential loop goes in CONT_BB,
7193 replacing the GIMPLE_OMP_CONTINUE. */
7194 gsi = gsi_last_bb (cont_bb);
7195 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
7196 vmain = gimple_omp_continue_control_use (cont_stmt);
7197 vback = gimple_omp_continue_control_def (cont_stmt);
7199 if (!gimple_omp_for_combined_p (fd->for_stmt))
7201 if (POINTER_TYPE_P (type))
7202 t = fold_build_pointer_plus (vmain, step);
7203 else
7204 t = fold_build2 (PLUS_EXPR, type, vmain, step);
7205 if (DECL_P (vback) && TREE_ADDRESSABLE (vback))
7206 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7207 true, GSI_SAME_STMT);
7208 assign_stmt = gimple_build_assign (vback, t);
7209 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
7211 t = build2 (fd->loop.cond_code, boolean_type_node,
7212 DECL_P (vback) && TREE_ADDRESSABLE (vback)
7213 ? t : vback, e);
7214 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
7217 /* Remove GIMPLE_OMP_CONTINUE. */
7218 gsi_remove (&gsi, true);
7220 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
7221 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
7223 /* Trip update code goes into TRIP_UPDATE_BB. */
7224 gsi = gsi_start_bb (trip_update_bb);
7226 t = build_int_cst (itype, 1);
7227 t = build2 (PLUS_EXPR, itype, trip_main, t);
7228 assign_stmt = gimple_build_assign (trip_back, t);
7229 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7232 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7233 gsi = gsi_last_bb (exit_bb);
7234 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
7236 t = gimple_omp_return_lhs (gsi_stmt (gsi));
7237 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
7238 gcc_checking_assert (t == NULL_TREE);
7239 else
7240 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
7242 gsi_remove (&gsi, true);
7244 /* Connect the new blocks. */
7245 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
7246 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
7248 if (!broken_loop)
7250 se = find_edge (cont_bb, body_bb);
7251 if (gimple_omp_for_combined_p (fd->for_stmt))
7253 remove_edge (se);
7254 se = NULL;
7256 else if (fd->collapse > 1)
7258 remove_edge (se);
7259 se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
7261 else
7262 se->flags = EDGE_TRUE_VALUE;
7263 find_edge (cont_bb, trip_update_bb)->flags
7264 = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
7266 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb);
7269 if (gimple_in_ssa_p (cfun))
7271 gphi_iterator psi;
7272 gphi *phi;
7273 edge re, ene;
7274 edge_var_map *vm;
7275 size_t i;
7277 gcc_assert (fd->collapse == 1 && !broken_loop);
7279 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7280 remove arguments of the phi nodes in fin_bb. We need to create
7281 appropriate phi nodes in iter_part_bb instead. */
7282 se = single_pred_edge (fin_bb);
7283 re = single_succ_edge (trip_update_bb);
7284 vec<edge_var_map> *head = redirect_edge_var_map_vector (re);
7285 ene = single_succ_edge (entry_bb);
7287 psi = gsi_start_phis (fin_bb);
7288 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
7289 gsi_next (&psi), ++i)
7291 gphi *nphi;
7292 source_location locus;
7294 phi = psi.phi ();
7295 t = gimple_phi_result (phi);
7296 gcc_assert (t == redirect_edge_var_map_result (vm));
7297 nphi = create_phi_node (t, iter_part_bb);
7299 t = PHI_ARG_DEF_FROM_EDGE (phi, se);
7300 locus = gimple_phi_arg_location_from_edge (phi, se);
7302 /* A special case -- fd->loop.v is not yet computed in
7303 iter_part_bb, we need to use vextra instead. */
7304 if (t == fd->loop.v)
7305 t = vextra;
7306 add_phi_arg (nphi, t, ene, locus);
7307 locus = redirect_edge_var_map_location (vm);
7308 add_phi_arg (nphi, redirect_edge_var_map_def (vm), re, locus);
7310 gcc_assert (gsi_end_p (psi) && i == head->length ());
7311 redirect_edge_var_map_clear (re);
7312 while (1)
7314 psi = gsi_start_phis (fin_bb);
7315 if (gsi_end_p (psi))
7316 break;
7317 remove_phi_node (&psi, false);
7320 /* Make phi node for trip. */
7321 phi = create_phi_node (trip_main, iter_part_bb);
7322 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb),
7323 UNKNOWN_LOCATION);
7324 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb),
7325 UNKNOWN_LOCATION);
7328 if (!broken_loop)
7329 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
7330 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
7331 recompute_dominator (CDI_DOMINATORS, iter_part_bb));
7332 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
7333 recompute_dominator (CDI_DOMINATORS, fin_bb));
7334 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
7335 recompute_dominator (CDI_DOMINATORS, seq_start_bb));
7336 set_immediate_dominator (CDI_DOMINATORS, body_bb,
7337 recompute_dominator (CDI_DOMINATORS, body_bb));
7339 if (!broken_loop)
7341 struct loop *trip_loop = alloc_loop ();
7342 trip_loop->header = iter_part_bb;
7343 trip_loop->latch = trip_update_bb;
7344 add_loop (trip_loop, iter_part_bb->loop_father);
7346 if (!gimple_omp_for_combined_p (fd->for_stmt))
7348 struct loop *loop = alloc_loop ();
7349 loop->header = body_bb;
7350 if (collapse_bb == NULL)
7351 loop->latch = cont_bb;
7352 add_loop (loop, trip_loop);
7357 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7358 Given parameters:
7359 for (V = N1; V cond N2; V += STEP) BODY;
7361 where COND is "<" or ">" or "!=", we generate pseudocode
7363 for (ind_var = low; ind_var < high; ind_var++)
7365 V = n1 + (ind_var * STEP)
7367 <BODY>
7370 In the above pseudocode, low and high are function parameters of the
7371 child function. In the function below, we are inserting a temp.
7372 variable that will be making a call to two OMP functions that will not be
7373 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7374 with _Cilk_for). These functions are replaced with low and high
7375 by the function that handles taskreg. */
7378 static void
7379 expand_cilk_for (struct omp_region *region, struct omp_for_data *fd)
7381 bool broken_loop = region->cont == NULL;
7382 basic_block entry_bb = region->entry;
7383 basic_block cont_bb = region->cont;
7385 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7386 gcc_assert (broken_loop
7387 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
7388 basic_block l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
7389 basic_block l1_bb, l2_bb;
7391 if (!broken_loop)
7393 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
7394 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
7395 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
7396 l2_bb = BRANCH_EDGE (entry_bb)->dest;
7398 else
7400 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
7401 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
7402 l2_bb = single_succ (l1_bb);
7404 basic_block exit_bb = region->exit;
7405 basic_block l2_dom_bb = NULL;
7407 gimple_stmt_iterator gsi = gsi_last_bb (entry_bb);
7409 /* Below statements until the "tree high_val = ..." are pseudo statements
7410 used to pass information to be used by expand_omp_taskreg.
7411 low_val and high_val will be replaced by the __low and __high
7412 parameter from the child function.
7414 The call_exprs part is a place-holder, it is mainly used
7415 to distinctly identify to the top-level part that this is
7416 where we should put low and high (reasoning given in header
7417 comment). */
7419 tree child_fndecl
7420 = gimple_omp_parallel_child_fn (
7421 as_a <gomp_parallel *> (last_stmt (region->outer->entry)));
7422 tree t, low_val = NULL_TREE, high_val = NULL_TREE;
7423 for (t = DECL_ARGUMENTS (child_fndecl); t; t = TREE_CHAIN (t))
7425 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__high"))
7426 high_val = t;
7427 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__low"))
7428 low_val = t;
7430 gcc_assert (low_val && high_val);
7432 tree type = TREE_TYPE (low_val);
7433 tree ind_var = create_tmp_reg (type, "__cilk_ind_var");
7434 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
7436 /* Not needed in SSA form right now. */
7437 gcc_assert (!gimple_in_ssa_p (cfun));
7438 if (l2_dom_bb == NULL)
7439 l2_dom_bb = l1_bb;
7441 tree n1 = low_val;
7442 tree n2 = high_val;
7444 gimple stmt = gimple_build_assign (ind_var, n1);
7446 /* Replace the GIMPLE_OMP_FOR statement. */
7447 gsi_replace (&gsi, stmt, true);
7449 if (!broken_loop)
7451 /* Code to control the increment goes in the CONT_BB. */
7452 gsi = gsi_last_bb (cont_bb);
7453 stmt = gsi_stmt (gsi);
7454 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
7455 stmt = gimple_build_assign (ind_var, PLUS_EXPR, ind_var,
7456 build_one_cst (type));
7458 /* Replace GIMPLE_OMP_CONTINUE. */
7459 gsi_replace (&gsi, stmt, true);
7462 /* Emit the condition in L1_BB. */
7463 gsi = gsi_after_labels (l1_bb);
7464 t = fold_build2 (MULT_EXPR, TREE_TYPE (fd->loop.step),
7465 fold_convert (TREE_TYPE (fd->loop.step), ind_var),
7466 fd->loop.step);
7467 if (POINTER_TYPE_P (TREE_TYPE (fd->loop.n1)))
7468 t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (fd->loop.n1),
7469 fd->loop.n1, fold_convert (sizetype, t));
7470 else
7471 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loop.n1),
7472 fd->loop.n1, fold_convert (TREE_TYPE (fd->loop.n1), t));
7473 t = fold_convert (TREE_TYPE (fd->loop.v), t);
7474 expand_omp_build_assign (&gsi, fd->loop.v, t);
7476 /* The condition is always '<' since the runtime will fill in the low
7477 and high values. */
7478 stmt = gimple_build_cond (LT_EXPR, ind_var, n2, NULL_TREE, NULL_TREE);
7479 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
7481 /* Remove GIMPLE_OMP_RETURN. */
7482 gsi = gsi_last_bb (exit_bb);
7483 gsi_remove (&gsi, true);
7485 /* Connect the new blocks. */
7486 remove_edge (FALLTHRU_EDGE (entry_bb));
7488 edge e, ne;
7489 if (!broken_loop)
7491 remove_edge (BRANCH_EDGE (entry_bb));
7492 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
7494 e = BRANCH_EDGE (l1_bb);
7495 ne = FALLTHRU_EDGE (l1_bb);
7496 e->flags = EDGE_TRUE_VALUE;
7498 else
7500 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7502 ne = single_succ_edge (l1_bb);
7503 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
7506 ne->flags = EDGE_FALSE_VALUE;
7507 e->probability = REG_BR_PROB_BASE * 7 / 8;
7508 ne->probability = REG_BR_PROB_BASE / 8;
7510 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
7511 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
7512 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
7514 if (!broken_loop)
7516 struct loop *loop = alloc_loop ();
7517 loop->header = l1_bb;
7518 loop->latch = cont_bb;
7519 add_loop (loop, l1_bb->loop_father);
7520 loop->safelen = INT_MAX;
7523 /* Pick the correct library function based on the precision of the
7524 induction variable type. */
7525 tree lib_fun = NULL_TREE;
7526 if (TYPE_PRECISION (type) == 32)
7527 lib_fun = cilk_for_32_fndecl;
7528 else if (TYPE_PRECISION (type) == 64)
7529 lib_fun = cilk_for_64_fndecl;
7530 else
7531 gcc_unreachable ();
7533 gcc_assert (fd->sched_kind == OMP_CLAUSE_SCHEDULE_CILKFOR);
7535 /* WS_ARGS contains the library function flavor to call:
7536 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7537 user-defined grain value. If the user does not define one, then zero
7538 is passed in by the parser. */
7539 vec_alloc (region->ws_args, 2);
7540 region->ws_args->quick_push (lib_fun);
7541 region->ws_args->quick_push (fd->chunk_size);
7544 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7545 loop. Given parameters:
7547 for (V = N1; V cond N2; V += STEP) BODY;
7549 where COND is "<" or ">", we generate pseudocode
7551 V = N1;
7552 goto L1;
7554 BODY;
7555 V += STEP;
7557 if (V cond N2) goto L0; else goto L2;
7560 For collapsed loops, given parameters:
7561 collapse(3)
7562 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7563 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7564 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7565 BODY;
7567 we generate pseudocode
7569 if (cond3 is <)
7570 adj = STEP3 - 1;
7571 else
7572 adj = STEP3 + 1;
7573 count3 = (adj + N32 - N31) / STEP3;
7574 if (cond2 is <)
7575 adj = STEP2 - 1;
7576 else
7577 adj = STEP2 + 1;
7578 count2 = (adj + N22 - N21) / STEP2;
7579 if (cond1 is <)
7580 adj = STEP1 - 1;
7581 else
7582 adj = STEP1 + 1;
7583 count1 = (adj + N12 - N11) / STEP1;
7584 count = count1 * count2 * count3;
7585 V = 0;
7586 V1 = N11;
7587 V2 = N21;
7588 V3 = N31;
7589 goto L1;
7591 BODY;
7592 V += 1;
7593 V3 += STEP3;
7594 V2 += (V3 cond3 N32) ? 0 : STEP2;
7595 V3 = (V3 cond3 N32) ? V3 : N31;
7596 V1 += (V2 cond2 N22) ? 0 : STEP1;
7597 V2 = (V2 cond2 N22) ? V2 : N21;
7599 if (V < count) goto L0; else goto L2;
7604 static void
7605 expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
7607 tree type, t;
7608 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb;
7609 gimple_stmt_iterator gsi;
7610 gimple stmt;
7611 gcond *cond_stmt;
7612 bool broken_loop = region->cont == NULL;
7613 edge e, ne;
7614 tree *counts = NULL;
7615 int i;
7616 tree safelen = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7617 OMP_CLAUSE_SAFELEN);
7618 tree simduid = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7619 OMP_CLAUSE__SIMDUID_);
7620 tree n1, n2;
7622 type = TREE_TYPE (fd->loop.v);
7623 entry_bb = region->entry;
7624 cont_bb = region->cont;
7625 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7626 gcc_assert (broken_loop
7627 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
7628 l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
7629 if (!broken_loop)
7631 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
7632 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
7633 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
7634 l2_bb = BRANCH_EDGE (entry_bb)->dest;
7636 else
7638 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
7639 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
7640 l2_bb = single_succ (l1_bb);
7642 exit_bb = region->exit;
7643 l2_dom_bb = NULL;
7645 gsi = gsi_last_bb (entry_bb);
7647 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
7648 /* Not needed in SSA form right now. */
7649 gcc_assert (!gimple_in_ssa_p (cfun));
7650 if (fd->collapse > 1)
7652 int first_zero_iter = -1;
7653 basic_block zero_iter_bb = l2_bb;
7655 counts = XALLOCAVEC (tree, fd->collapse);
7656 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
7657 zero_iter_bb, first_zero_iter,
7658 l2_dom_bb);
7660 if (l2_dom_bb == NULL)
7661 l2_dom_bb = l1_bb;
7663 n1 = fd->loop.n1;
7664 n2 = fd->loop.n2;
7665 if (gimple_omp_for_combined_into_p (fd->for_stmt))
7667 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7668 OMP_CLAUSE__LOOPTEMP_);
7669 gcc_assert (innerc);
7670 n1 = OMP_CLAUSE_DECL (innerc);
7671 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7672 OMP_CLAUSE__LOOPTEMP_);
7673 gcc_assert (innerc);
7674 n2 = OMP_CLAUSE_DECL (innerc);
7675 expand_omp_build_assign (&gsi, fd->loop.v,
7676 fold_convert (type, n1));
7677 if (fd->collapse > 1)
7679 gsi_prev (&gsi);
7680 expand_omp_for_init_vars (fd, &gsi, counts, NULL, n1);
7681 gsi_next (&gsi);
7684 else
7686 expand_omp_build_assign (&gsi, fd->loop.v,
7687 fold_convert (type, fd->loop.n1));
7688 if (fd->collapse > 1)
7689 for (i = 0; i < fd->collapse; i++)
7691 tree itype = TREE_TYPE (fd->loops[i].v);
7692 if (POINTER_TYPE_P (itype))
7693 itype = signed_type_for (itype);
7694 t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1);
7695 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
7699 /* Remove the GIMPLE_OMP_FOR statement. */
7700 gsi_remove (&gsi, true);
7702 if (!broken_loop)
7704 /* Code to control the increment goes in the CONT_BB. */
7705 gsi = gsi_last_bb (cont_bb);
7706 stmt = gsi_stmt (gsi);
7707 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
7709 if (POINTER_TYPE_P (type))
7710 t = fold_build_pointer_plus (fd->loop.v, fd->loop.step);
7711 else
7712 t = fold_build2 (PLUS_EXPR, type, fd->loop.v, fd->loop.step);
7713 expand_omp_build_assign (&gsi, fd->loop.v, t);
7715 if (fd->collapse > 1)
7717 i = fd->collapse - 1;
7718 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
7720 t = fold_convert (sizetype, fd->loops[i].step);
7721 t = fold_build_pointer_plus (fd->loops[i].v, t);
7723 else
7725 t = fold_convert (TREE_TYPE (fd->loops[i].v),
7726 fd->loops[i].step);
7727 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
7728 fd->loops[i].v, t);
7730 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
7732 for (i = fd->collapse - 1; i > 0; i--)
7734 tree itype = TREE_TYPE (fd->loops[i].v);
7735 tree itype2 = TREE_TYPE (fd->loops[i - 1].v);
7736 if (POINTER_TYPE_P (itype2))
7737 itype2 = signed_type_for (itype2);
7738 t = build3 (COND_EXPR, itype2,
7739 build2 (fd->loops[i].cond_code, boolean_type_node,
7740 fd->loops[i].v,
7741 fold_convert (itype, fd->loops[i].n2)),
7742 build_int_cst (itype2, 0),
7743 fold_convert (itype2, fd->loops[i - 1].step));
7744 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i - 1].v)))
7745 t = fold_build_pointer_plus (fd->loops[i - 1].v, t);
7746 else
7747 t = fold_build2 (PLUS_EXPR, itype2, fd->loops[i - 1].v, t);
7748 expand_omp_build_assign (&gsi, fd->loops[i - 1].v, t);
7750 t = build3 (COND_EXPR, itype,
7751 build2 (fd->loops[i].cond_code, boolean_type_node,
7752 fd->loops[i].v,
7753 fold_convert (itype, fd->loops[i].n2)),
7754 fd->loops[i].v,
7755 fold_convert (itype, fd->loops[i].n1));
7756 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
7760 /* Remove GIMPLE_OMP_CONTINUE. */
7761 gsi_remove (&gsi, true);
7764 /* Emit the condition in L1_BB. */
7765 gsi = gsi_start_bb (l1_bb);
7767 t = fold_convert (type, n2);
7768 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7769 false, GSI_CONTINUE_LINKING);
7770 t = build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t);
7771 cond_stmt = gimple_build_cond_empty (t);
7772 gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
7773 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p,
7774 NULL, NULL)
7775 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p,
7776 NULL, NULL))
7778 gsi = gsi_for_stmt (cond_stmt);
7779 gimple_regimplify_operands (cond_stmt, &gsi);
7782 /* Remove GIMPLE_OMP_RETURN. */
7783 gsi = gsi_last_bb (exit_bb);
7784 gsi_remove (&gsi, true);
7786 /* Connect the new blocks. */
7787 remove_edge (FALLTHRU_EDGE (entry_bb));
7789 if (!broken_loop)
7791 remove_edge (BRANCH_EDGE (entry_bb));
7792 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
7794 e = BRANCH_EDGE (l1_bb);
7795 ne = FALLTHRU_EDGE (l1_bb);
7796 e->flags = EDGE_TRUE_VALUE;
7798 else
7800 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7802 ne = single_succ_edge (l1_bb);
7803 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
7806 ne->flags = EDGE_FALSE_VALUE;
7807 e->probability = REG_BR_PROB_BASE * 7 / 8;
7808 ne->probability = REG_BR_PROB_BASE / 8;
7810 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
7811 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
7812 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
7814 if (!broken_loop)
7816 struct loop *loop = alloc_loop ();
7817 loop->header = l1_bb;
7818 loop->latch = cont_bb;
7819 add_loop (loop, l1_bb->loop_father);
7820 if (safelen == NULL_TREE)
7821 loop->safelen = INT_MAX;
7822 else
7824 safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen);
7825 if (TREE_CODE (safelen) != INTEGER_CST)
7826 loop->safelen = 0;
7827 else if (!tree_fits_uhwi_p (safelen)
7828 || tree_to_uhwi (safelen) > INT_MAX)
7829 loop->safelen = INT_MAX;
7830 else
7831 loop->safelen = tree_to_uhwi (safelen);
7832 if (loop->safelen == 1)
7833 loop->safelen = 0;
7835 if (simduid)
7837 loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
7838 cfun->has_simduid_loops = true;
7840 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7841 the loop. */
7842 if ((flag_tree_loop_vectorize
7843 || (!global_options_set.x_flag_tree_loop_vectorize
7844 && !global_options_set.x_flag_tree_vectorize))
7845 && flag_tree_loop_optimize
7846 && loop->safelen > 1)
7848 loop->force_vectorize = true;
7849 cfun->has_force_vectorize_loops = true;
7852 else if (simduid)
7853 cfun->has_simduid_loops = true;
7857 /* Expand the OMP loop defined by REGION. */
7859 static void
7860 expand_omp_for (struct omp_region *region, gimple inner_stmt)
7862 struct omp_for_data fd;
7863 struct omp_for_data_loop *loops;
7865 loops
7866 = (struct omp_for_data_loop *)
7867 alloca (gimple_omp_for_collapse (last_stmt (region->entry))
7868 * sizeof (struct omp_for_data_loop));
7869 extract_omp_for_data (as_a <gomp_for *> (last_stmt (region->entry)),
7870 &fd, loops);
7871 region->sched_kind = fd.sched_kind;
7873 gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
7874 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
7875 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
7876 if (region->cont)
7878 gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
7879 BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
7880 FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
7882 else
7883 /* If there isn't a continue then this is a degerate case where
7884 the introduction of abnormal edges during lowering will prevent
7885 original loops from being detected. Fix that up. */
7886 loops_state_set (LOOPS_NEED_FIXUP);
7888 if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD)
7889 expand_omp_simd (region, &fd);
7890 else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_CILKFOR)
7891 expand_cilk_for (region, &fd);
7892 else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
7893 && !fd.have_ordered)
7895 if (fd.chunk_size == NULL)
7896 expand_omp_for_static_nochunk (region, &fd, inner_stmt);
7897 else
7898 expand_omp_for_static_chunk (region, &fd, inner_stmt);
7900 else
7902 int fn_index, start_ix, next_ix;
7904 gcc_assert (gimple_omp_for_kind (fd.for_stmt)
7905 == GF_OMP_FOR_KIND_FOR);
7906 if (fd.chunk_size == NULL
7907 && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
7908 fd.chunk_size = integer_zero_node;
7909 gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
7910 fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
7911 ? 3 : fd.sched_kind;
7912 fn_index += fd.have_ordered * 4;
7913 start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
7914 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
7915 if (fd.iter_type == long_long_unsigned_type_node)
7917 start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7918 - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
7919 next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7920 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
7922 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
7923 (enum built_in_function) next_ix, inner_stmt);
7926 if (gimple_in_ssa_p (cfun))
7927 update_ssa (TODO_update_ssa_only_virtuals);
7931 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7933 v = GOMP_sections_start (n);
7935 switch (v)
7937 case 0:
7938 goto L2;
7939 case 1:
7940 section 1;
7941 goto L1;
7942 case 2:
7944 case n:
7946 default:
7947 abort ();
7950 v = GOMP_sections_next ();
7951 goto L0;
7953 reduction;
7955 If this is a combined parallel sections, replace the call to
7956 GOMP_sections_start with call to GOMP_sections_next. */
7958 static void
7959 expand_omp_sections (struct omp_region *region)
7961 tree t, u, vin = NULL, vmain, vnext, l2;
7962 unsigned len;
7963 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
7964 gimple_stmt_iterator si, switch_si;
7965 gomp_sections *sections_stmt;
7966 gimple stmt;
7967 gomp_continue *cont;
7968 edge_iterator ei;
7969 edge e;
7970 struct omp_region *inner;
7971 unsigned i, casei;
7972 bool exit_reachable = region->cont != NULL;
7974 gcc_assert (region->exit != NULL);
7975 entry_bb = region->entry;
7976 l0_bb = single_succ (entry_bb);
7977 l1_bb = region->cont;
7978 l2_bb = region->exit;
7979 if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb)
7980 l2 = gimple_block_label (l2_bb);
7981 else
7983 /* This can happen if there are reductions. */
7984 len = EDGE_COUNT (l0_bb->succs);
7985 gcc_assert (len > 0);
7986 e = EDGE_SUCC (l0_bb, len - 1);
7987 si = gsi_last_bb (e->dest);
7988 l2 = NULL_TREE;
7989 if (gsi_end_p (si)
7990 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
7991 l2 = gimple_block_label (e->dest);
7992 else
7993 FOR_EACH_EDGE (e, ei, l0_bb->succs)
7995 si = gsi_last_bb (e->dest);
7996 if (gsi_end_p (si)
7997 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
7999 l2 = gimple_block_label (e->dest);
8000 break;
8004 if (exit_reachable)
8005 default_bb = create_empty_bb (l1_bb->prev_bb);
8006 else
8007 default_bb = create_empty_bb (l0_bb);
8009 /* We will build a switch() with enough cases for all the
8010 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
8011 and a default case to abort if something goes wrong. */
8012 len = EDGE_COUNT (l0_bb->succs);
8014 /* Use vec::quick_push on label_vec throughout, since we know the size
8015 in advance. */
8016 auto_vec<tree> label_vec (len);
8018 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
8019 GIMPLE_OMP_SECTIONS statement. */
8020 si = gsi_last_bb (entry_bb);
8021 sections_stmt = as_a <gomp_sections *> (gsi_stmt (si));
8022 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
8023 vin = gimple_omp_sections_control (sections_stmt);
8024 if (!is_combined_parallel (region))
8026 /* If we are not inside a combined parallel+sections region,
8027 call GOMP_sections_start. */
8028 t = build_int_cst (unsigned_type_node, len - 1);
8029 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
8030 stmt = gimple_build_call (u, 1, t);
8032 else
8034 /* Otherwise, call GOMP_sections_next. */
8035 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
8036 stmt = gimple_build_call (u, 0);
8038 gimple_call_set_lhs (stmt, vin);
8039 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8040 gsi_remove (&si, true);
8042 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8043 L0_BB. */
8044 switch_si = gsi_last_bb (l0_bb);
8045 gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH);
8046 if (exit_reachable)
8048 cont = as_a <gomp_continue *> (last_stmt (l1_bb));
8049 gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE);
8050 vmain = gimple_omp_continue_control_use (cont);
8051 vnext = gimple_omp_continue_control_def (cont);
8053 else
8055 vmain = vin;
8056 vnext = NULL_TREE;
8059 t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2);
8060 label_vec.quick_push (t);
8061 i = 1;
8063 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8064 for (inner = region->inner, casei = 1;
8065 inner;
8066 inner = inner->next, i++, casei++)
8068 basic_block s_entry_bb, s_exit_bb;
8070 /* Skip optional reduction region. */
8071 if (inner->type == GIMPLE_OMP_ATOMIC_LOAD)
8073 --i;
8074 --casei;
8075 continue;
8078 s_entry_bb = inner->entry;
8079 s_exit_bb = inner->exit;
8081 t = gimple_block_label (s_entry_bb);
8082 u = build_int_cst (unsigned_type_node, casei);
8083 u = build_case_label (u, NULL, t);
8084 label_vec.quick_push (u);
8086 si = gsi_last_bb (s_entry_bb);
8087 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION);
8088 gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si)));
8089 gsi_remove (&si, true);
8090 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
8092 if (s_exit_bb == NULL)
8093 continue;
8095 si = gsi_last_bb (s_exit_bb);
8096 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
8097 gsi_remove (&si, true);
8099 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
8102 /* Error handling code goes in DEFAULT_BB. */
8103 t = gimple_block_label (default_bb);
8104 u = build_case_label (NULL, NULL, t);
8105 make_edge (l0_bb, default_bb, 0);
8106 add_bb_to_loop (default_bb, current_loops->tree_root);
8108 stmt = gimple_build_switch (vmain, u, label_vec);
8109 gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT);
8110 gsi_remove (&switch_si, true);
8112 si = gsi_start_bb (default_bb);
8113 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
8114 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
8116 if (exit_reachable)
8118 tree bfn_decl;
8120 /* Code to get the next section goes in L1_BB. */
8121 si = gsi_last_bb (l1_bb);
8122 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
8124 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
8125 stmt = gimple_build_call (bfn_decl, 0);
8126 gimple_call_set_lhs (stmt, vnext);
8127 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8128 gsi_remove (&si, true);
8130 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
8133 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8134 si = gsi_last_bb (l2_bb);
8135 if (gimple_omp_return_nowait_p (gsi_stmt (si)))
8136 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT);
8137 else if (gimple_omp_return_lhs (gsi_stmt (si)))
8138 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL);
8139 else
8140 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END);
8141 stmt = gimple_build_call (t, 0);
8142 if (gimple_omp_return_lhs (gsi_stmt (si)))
8143 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si)));
8144 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8145 gsi_remove (&si, true);
8147 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
8151 /* Expand code for an OpenMP single directive. We've already expanded
8152 much of the code, here we simply place the GOMP_barrier call. */
8154 static void
8155 expand_omp_single (struct omp_region *region)
8157 basic_block entry_bb, exit_bb;
8158 gimple_stmt_iterator si;
8160 entry_bb = region->entry;
8161 exit_bb = region->exit;
8163 si = gsi_last_bb (entry_bb);
8164 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE);
8165 gsi_remove (&si, true);
8166 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8168 si = gsi_last_bb (exit_bb);
8169 if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
8171 tree t = gimple_omp_return_lhs (gsi_stmt (si));
8172 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT);
8174 gsi_remove (&si, true);
8175 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
8179 /* Generic expansion for OpenMP synchronization directives: master,
8180 ordered and critical. All we need to do here is remove the entry
8181 and exit markers for REGION. */
8183 static void
8184 expand_omp_synch (struct omp_region *region)
8186 basic_block entry_bb, exit_bb;
8187 gimple_stmt_iterator si;
8189 entry_bb = region->entry;
8190 exit_bb = region->exit;
8192 si = gsi_last_bb (entry_bb);
8193 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
8194 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
8195 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
8196 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
8197 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
8198 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS);
8199 gsi_remove (&si, true);
8200 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8202 if (exit_bb)
8204 si = gsi_last_bb (exit_bb);
8205 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
8206 gsi_remove (&si, true);
8207 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
8211 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8212 operation as a normal volatile load. */
8214 static bool
8215 expand_omp_atomic_load (basic_block load_bb, tree addr,
8216 tree loaded_val, int index)
8218 enum built_in_function tmpbase;
8219 gimple_stmt_iterator gsi;
8220 basic_block store_bb;
8221 location_t loc;
8222 gimple stmt;
8223 tree decl, call, type, itype;
8225 gsi = gsi_last_bb (load_bb);
8226 stmt = gsi_stmt (gsi);
8227 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
8228 loc = gimple_location (stmt);
8230 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8231 is smaller than word size, then expand_atomic_load assumes that the load
8232 is atomic. We could avoid the builtin entirely in this case. */
8234 tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
8235 decl = builtin_decl_explicit (tmpbase);
8236 if (decl == NULL_TREE)
8237 return false;
8239 type = TREE_TYPE (loaded_val);
8240 itype = TREE_TYPE (TREE_TYPE (decl));
8242 call = build_call_expr_loc (loc, decl, 2, addr,
8243 build_int_cst (NULL,
8244 gimple_omp_atomic_seq_cst_p (stmt)
8245 ? MEMMODEL_SEQ_CST
8246 : MEMMODEL_RELAXED));
8247 if (!useless_type_conversion_p (type, itype))
8248 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
8249 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
8251 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
8252 gsi_remove (&gsi, true);
8254 store_bb = single_succ (load_bb);
8255 gsi = gsi_last_bb (store_bb);
8256 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
8257 gsi_remove (&gsi, true);
8259 if (gimple_in_ssa_p (cfun))
8260 update_ssa (TODO_update_ssa_no_phi);
8262 return true;
8265 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8266 operation as a normal volatile store. */
8268 static bool
8269 expand_omp_atomic_store (basic_block load_bb, tree addr,
8270 tree loaded_val, tree stored_val, int index)
8272 enum built_in_function tmpbase;
8273 gimple_stmt_iterator gsi;
8274 basic_block store_bb = single_succ (load_bb);
8275 location_t loc;
8276 gimple stmt;
8277 tree decl, call, type, itype;
8278 machine_mode imode;
8279 bool exchange;
8281 gsi = gsi_last_bb (load_bb);
8282 stmt = gsi_stmt (gsi);
8283 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
8285 /* If the load value is needed, then this isn't a store but an exchange. */
8286 exchange = gimple_omp_atomic_need_value_p (stmt);
8288 gsi = gsi_last_bb (store_bb);
8289 stmt = gsi_stmt (gsi);
8290 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE);
8291 loc = gimple_location (stmt);
8293 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8294 is smaller than word size, then expand_atomic_store assumes that the store
8295 is atomic. We could avoid the builtin entirely in this case. */
8297 tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N);
8298 tmpbase = (enum built_in_function) ((int) tmpbase + index + 1);
8299 decl = builtin_decl_explicit (tmpbase);
8300 if (decl == NULL_TREE)
8301 return false;
8303 type = TREE_TYPE (stored_val);
8305 /* Dig out the type of the function's second argument. */
8306 itype = TREE_TYPE (decl);
8307 itype = TYPE_ARG_TYPES (itype);
8308 itype = TREE_CHAIN (itype);
8309 itype = TREE_VALUE (itype);
8310 imode = TYPE_MODE (itype);
8312 if (exchange && !can_atomic_exchange_p (imode, true))
8313 return false;
8315 if (!useless_type_conversion_p (itype, type))
8316 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
8317 call = build_call_expr_loc (loc, decl, 3, addr, stored_val,
8318 build_int_cst (NULL,
8319 gimple_omp_atomic_seq_cst_p (stmt)
8320 ? MEMMODEL_SEQ_CST
8321 : MEMMODEL_RELAXED));
8322 if (exchange)
8324 if (!useless_type_conversion_p (type, itype))
8325 call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
8326 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
8329 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
8330 gsi_remove (&gsi, true);
8332 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8333 gsi = gsi_last_bb (load_bb);
8334 gsi_remove (&gsi, true);
8336 if (gimple_in_ssa_p (cfun))
8337 update_ssa (TODO_update_ssa_no_phi);
8339 return true;
8342 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8343 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8344 size of the data type, and thus usable to find the index of the builtin
8345 decl. Returns false if the expression is not of the proper form. */
8347 static bool
8348 expand_omp_atomic_fetch_op (basic_block load_bb,
8349 tree addr, tree loaded_val,
8350 tree stored_val, int index)
8352 enum built_in_function oldbase, newbase, tmpbase;
8353 tree decl, itype, call;
8354 tree lhs, rhs;
8355 basic_block store_bb = single_succ (load_bb);
8356 gimple_stmt_iterator gsi;
8357 gimple stmt;
8358 location_t loc;
8359 enum tree_code code;
8360 bool need_old, need_new;
8361 machine_mode imode;
8362 bool seq_cst;
8364 /* We expect to find the following sequences:
8366 load_bb:
8367 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8369 store_bb:
8370 val = tmp OP something; (or: something OP tmp)
8371 GIMPLE_OMP_STORE (val)
8373 ???FIXME: Allow a more flexible sequence.
8374 Perhaps use data flow to pick the statements.
8378 gsi = gsi_after_labels (store_bb);
8379 stmt = gsi_stmt (gsi);
8380 loc = gimple_location (stmt);
8381 if (!is_gimple_assign (stmt))
8382 return false;
8383 gsi_next (&gsi);
8384 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
8385 return false;
8386 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
8387 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb));
8388 seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb));
8389 gcc_checking_assert (!need_old || !need_new);
8391 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
8392 return false;
8394 /* Check for one of the supported fetch-op operations. */
8395 code = gimple_assign_rhs_code (stmt);
8396 switch (code)
8398 case PLUS_EXPR:
8399 case POINTER_PLUS_EXPR:
8400 oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N;
8401 newbase = BUILT_IN_ATOMIC_ADD_FETCH_N;
8402 break;
8403 case MINUS_EXPR:
8404 oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N;
8405 newbase = BUILT_IN_ATOMIC_SUB_FETCH_N;
8406 break;
8407 case BIT_AND_EXPR:
8408 oldbase = BUILT_IN_ATOMIC_FETCH_AND_N;
8409 newbase = BUILT_IN_ATOMIC_AND_FETCH_N;
8410 break;
8411 case BIT_IOR_EXPR:
8412 oldbase = BUILT_IN_ATOMIC_FETCH_OR_N;
8413 newbase = BUILT_IN_ATOMIC_OR_FETCH_N;
8414 break;
8415 case BIT_XOR_EXPR:
8416 oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N;
8417 newbase = BUILT_IN_ATOMIC_XOR_FETCH_N;
8418 break;
8419 default:
8420 return false;
8423 /* Make sure the expression is of the proper form. */
8424 if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
8425 rhs = gimple_assign_rhs2 (stmt);
8426 else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
8427 && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
8428 rhs = gimple_assign_rhs1 (stmt);
8429 else
8430 return false;
8432 tmpbase = ((enum built_in_function)
8433 ((need_new ? newbase : oldbase) + index + 1));
8434 decl = builtin_decl_explicit (tmpbase);
8435 if (decl == NULL_TREE)
8436 return false;
8437 itype = TREE_TYPE (TREE_TYPE (decl));
8438 imode = TYPE_MODE (itype);
8440 /* We could test all of the various optabs involved, but the fact of the
8441 matter is that (with the exception of i486 vs i586 and xadd) all targets
8442 that support any atomic operaton optab also implements compare-and-swap.
8443 Let optabs.c take care of expanding any compare-and-swap loop. */
8444 if (!can_compare_and_swap_p (imode, true))
8445 return false;
8447 gsi = gsi_last_bb (load_bb);
8448 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
8450 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8451 It only requires that the operation happen atomically. Thus we can
8452 use the RELAXED memory model. */
8453 call = build_call_expr_loc (loc, decl, 3, addr,
8454 fold_convert_loc (loc, itype, rhs),
8455 build_int_cst (NULL,
8456 seq_cst ? MEMMODEL_SEQ_CST
8457 : MEMMODEL_RELAXED));
8459 if (need_old || need_new)
8461 lhs = need_old ? loaded_val : stored_val;
8462 call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
8463 call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call);
8465 else
8466 call = fold_convert_loc (loc, void_type_node, call);
8467 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
8468 gsi_remove (&gsi, true);
8470 gsi = gsi_last_bb (store_bb);
8471 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
8472 gsi_remove (&gsi, true);
8473 gsi = gsi_last_bb (store_bb);
8474 gsi_remove (&gsi, true);
8476 if (gimple_in_ssa_p (cfun))
8477 update_ssa (TODO_update_ssa_no_phi);
8479 return true;
8482 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8484 oldval = *addr;
8485 repeat:
8486 newval = rhs; // with oldval replacing *addr in rhs
8487 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8488 if (oldval != newval)
8489 goto repeat;
8491 INDEX is log2 of the size of the data type, and thus usable to find the
8492 index of the builtin decl. */
8494 static bool
8495 expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
8496 tree addr, tree loaded_val, tree stored_val,
8497 int index)
8499 tree loadedi, storedi, initial, new_storedi, old_vali;
8500 tree type, itype, cmpxchg, iaddr;
8501 gimple_stmt_iterator si;
8502 basic_block loop_header = single_succ (load_bb);
8503 gimple phi, stmt;
8504 edge e;
8505 enum built_in_function fncode;
8507 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8508 order to use the RELAXED memory model effectively. */
8509 fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8510 + index + 1);
8511 cmpxchg = builtin_decl_explicit (fncode);
8512 if (cmpxchg == NULL_TREE)
8513 return false;
8514 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
8515 itype = TREE_TYPE (TREE_TYPE (cmpxchg));
8517 if (!can_compare_and_swap_p (TYPE_MODE (itype), true))
8518 return false;
8520 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8521 si = gsi_last_bb (load_bb);
8522 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
8524 /* For floating-point values, we'll need to view-convert them to integers
8525 so that we can perform the atomic compare and swap. Simplify the
8526 following code by always setting up the "i"ntegral variables. */
8527 if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
8529 tree iaddr_val;
8531 iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode,
8532 true));
8533 iaddr_val
8534 = force_gimple_operand_gsi (&si,
8535 fold_convert (TREE_TYPE (iaddr), addr),
8536 false, NULL_TREE, true, GSI_SAME_STMT);
8537 stmt = gimple_build_assign (iaddr, iaddr_val);
8538 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8539 loadedi = create_tmp_var (itype);
8540 if (gimple_in_ssa_p (cfun))
8541 loadedi = make_ssa_name (loadedi);
8543 else
8545 iaddr = addr;
8546 loadedi = loaded_val;
8549 fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
8550 tree loaddecl = builtin_decl_explicit (fncode);
8551 if (loaddecl)
8552 initial
8553 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr)),
8554 build_call_expr (loaddecl, 2, iaddr,
8555 build_int_cst (NULL_TREE,
8556 MEMMODEL_RELAXED)));
8557 else
8558 initial = build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)), iaddr,
8559 build_int_cst (TREE_TYPE (iaddr), 0));
8561 initial
8562 = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true,
8563 GSI_SAME_STMT);
8565 /* Move the value to the LOADEDI temporary. */
8566 if (gimple_in_ssa_p (cfun))
8568 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
8569 phi = create_phi_node (loadedi, loop_header);
8570 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
8571 initial);
8573 else
8574 gsi_insert_before (&si,
8575 gimple_build_assign (loadedi, initial),
8576 GSI_SAME_STMT);
8577 if (loadedi != loaded_val)
8579 gimple_stmt_iterator gsi2;
8580 tree x;
8582 x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
8583 gsi2 = gsi_start_bb (loop_header);
8584 if (gimple_in_ssa_p (cfun))
8586 gassign *stmt;
8587 x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
8588 true, GSI_SAME_STMT);
8589 stmt = gimple_build_assign (loaded_val, x);
8590 gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
8592 else
8594 x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
8595 force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
8596 true, GSI_SAME_STMT);
8599 gsi_remove (&si, true);
8601 si = gsi_last_bb (store_bb);
8602 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
8604 if (iaddr == addr)
8605 storedi = stored_val;
8606 else
8607 storedi =
8608 force_gimple_operand_gsi (&si,
8609 build1 (VIEW_CONVERT_EXPR, itype,
8610 stored_val), true, NULL_TREE, true,
8611 GSI_SAME_STMT);
8613 /* Build the compare&swap statement. */
8614 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
8615 new_storedi = force_gimple_operand_gsi (&si,
8616 fold_convert (TREE_TYPE (loadedi),
8617 new_storedi),
8618 true, NULL_TREE,
8619 true, GSI_SAME_STMT);
8621 if (gimple_in_ssa_p (cfun))
8622 old_vali = loadedi;
8623 else
8625 old_vali = create_tmp_var (TREE_TYPE (loadedi));
8626 stmt = gimple_build_assign (old_vali, loadedi);
8627 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8629 stmt = gimple_build_assign (loadedi, new_storedi);
8630 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8633 /* Note that we always perform the comparison as an integer, even for
8634 floating point. This allows the atomic operation to properly
8635 succeed even with NaNs and -0.0. */
8636 stmt = gimple_build_cond_empty
8637 (build2 (NE_EXPR, boolean_type_node,
8638 new_storedi, old_vali));
8639 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8641 /* Update cfg. */
8642 e = single_succ_edge (store_bb);
8643 e->flags &= ~EDGE_FALLTHRU;
8644 e->flags |= EDGE_FALSE_VALUE;
8646 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
8648 /* Copy the new value to loadedi (we already did that before the condition
8649 if we are not in SSA). */
8650 if (gimple_in_ssa_p (cfun))
8652 phi = gimple_seq_first_stmt (phi_nodes (loop_header));
8653 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
8656 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8657 gsi_remove (&si, true);
8659 struct loop *loop = alloc_loop ();
8660 loop->header = loop_header;
8661 loop->latch = store_bb;
8662 add_loop (loop, loop_header->loop_father);
8664 if (gimple_in_ssa_p (cfun))
8665 update_ssa (TODO_update_ssa_no_phi);
8667 return true;
8670 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8672 GOMP_atomic_start ();
8673 *addr = rhs;
8674 GOMP_atomic_end ();
8676 The result is not globally atomic, but works so long as all parallel
8677 references are within #pragma omp atomic directives. According to
8678 responses received from omp@openmp.org, appears to be within spec.
8679 Which makes sense, since that's how several other compilers handle
8680 this situation as well.
8681 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8682 expanding. STORED_VAL is the operand of the matching
8683 GIMPLE_OMP_ATOMIC_STORE.
8685 We replace
8686 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8687 loaded_val = *addr;
8689 and replace
8690 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8691 *addr = stored_val;
8694 static bool
8695 expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
8696 tree addr, tree loaded_val, tree stored_val)
8698 gimple_stmt_iterator si;
8699 gassign *stmt;
8700 tree t;
8702 si = gsi_last_bb (load_bb);
8703 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
8705 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
8706 t = build_call_expr (t, 0);
8707 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
8709 stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr));
8710 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8711 gsi_remove (&si, true);
8713 si = gsi_last_bb (store_bb);
8714 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
8716 stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)),
8717 stored_val);
8718 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8720 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
8721 t = build_call_expr (t, 0);
8722 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
8723 gsi_remove (&si, true);
8725 if (gimple_in_ssa_p (cfun))
8726 update_ssa (TODO_update_ssa_no_phi);
8727 return true;
8730 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8731 using expand_omp_atomic_fetch_op. If it failed, we try to
8732 call expand_omp_atomic_pipeline, and if it fails too, the
8733 ultimate fallback is wrapping the operation in a mutex
8734 (expand_omp_atomic_mutex). REGION is the atomic region built
8735 by build_omp_regions_1(). */
8737 static void
8738 expand_omp_atomic (struct omp_region *region)
8740 basic_block load_bb = region->entry, store_bb = region->exit;
8741 gomp_atomic_load *load = as_a <gomp_atomic_load *> (last_stmt (load_bb));
8742 gomp_atomic_store *store = as_a <gomp_atomic_store *> (last_stmt (store_bb));
8743 tree loaded_val = gimple_omp_atomic_load_lhs (load);
8744 tree addr = gimple_omp_atomic_load_rhs (load);
8745 tree stored_val = gimple_omp_atomic_store_val (store);
8746 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
8747 HOST_WIDE_INT index;
8749 /* Make sure the type is one of the supported sizes. */
8750 index = tree_to_uhwi (TYPE_SIZE_UNIT (type));
8751 index = exact_log2 (index);
8752 if (index >= 0 && index <= 4)
8754 unsigned int align = TYPE_ALIGN_UNIT (type);
8756 /* __sync builtins require strict data alignment. */
8757 if (exact_log2 (align) >= index)
8759 /* Atomic load. */
8760 if (loaded_val == stored_val
8761 && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
8762 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
8763 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
8764 && expand_omp_atomic_load (load_bb, addr, loaded_val, index))
8765 return;
8767 /* Atomic store. */
8768 if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
8769 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
8770 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
8771 && store_bb == single_succ (load_bb)
8772 && first_stmt (store_bb) == store
8773 && expand_omp_atomic_store (load_bb, addr, loaded_val,
8774 stored_val, index))
8775 return;
8777 /* When possible, use specialized atomic update functions. */
8778 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
8779 && store_bb == single_succ (load_bb)
8780 && expand_omp_atomic_fetch_op (load_bb, addr,
8781 loaded_val, stored_val, index))
8782 return;
8784 /* If we don't have specialized __sync builtins, try and implement
8785 as a compare and swap loop. */
8786 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
8787 loaded_val, stored_val, index))
8788 return;
8792 /* The ultimate fallback is wrapping the operation in a mutex. */
8793 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
8797 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8799 static void
8800 expand_omp_target (struct omp_region *region)
8802 basic_block entry_bb, exit_bb, new_bb;
8803 struct function *child_cfun;
8804 tree child_fn, block, t;
8805 gimple_stmt_iterator gsi;
8806 gomp_target *entry_stmt;
8807 gimple stmt;
8808 edge e;
8809 bool offloaded, data_region;
8811 entry_stmt = as_a <gomp_target *> (last_stmt (region->entry));
8812 new_bb = region->entry;
8814 offloaded = is_gimple_omp_offloaded (entry_stmt);
8815 switch (gimple_omp_target_kind (entry_stmt))
8817 case GF_OMP_TARGET_KIND_REGION:
8818 case GF_OMP_TARGET_KIND_UPDATE:
8819 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
8820 case GF_OMP_TARGET_KIND_OACC_KERNELS:
8821 case GF_OMP_TARGET_KIND_OACC_UPDATE:
8822 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
8823 data_region = false;
8824 break;
8825 case GF_OMP_TARGET_KIND_DATA:
8826 case GF_OMP_TARGET_KIND_OACC_DATA:
8827 data_region = true;
8828 break;
8829 default:
8830 gcc_unreachable ();
8833 child_fn = NULL_TREE;
8834 child_cfun = NULL;
8835 if (offloaded)
8837 child_fn = gimple_omp_target_child_fn (entry_stmt);
8838 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
8841 /* Supported by expand_omp_taskreg, but not here. */
8842 if (child_cfun != NULL)
8843 gcc_checking_assert (!child_cfun->cfg);
8844 gcc_checking_assert (!gimple_in_ssa_p (cfun));
8846 entry_bb = region->entry;
8847 exit_bb = region->exit;
8849 if (offloaded)
8851 unsigned srcidx, dstidx, num;
8853 /* If the offloading region needs data sent from the parent
8854 function, then the very first statement (except possible
8855 tree profile counter updates) of the offloading body
8856 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8857 &.OMP_DATA_O is passed as an argument to the child function,
8858 we need to replace it with the argument as seen by the child
8859 function.
8861 In most cases, this will end up being the identity assignment
8862 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8863 a function call that has been inlined, the original PARM_DECL
8864 .OMP_DATA_I may have been converted into a different local
8865 variable. In which case, we need to keep the assignment. */
8866 tree data_arg = gimple_omp_target_data_arg (entry_stmt);
8867 if (data_arg)
8869 basic_block entry_succ_bb = single_succ (entry_bb);
8870 gimple_stmt_iterator gsi;
8871 tree arg;
8872 gimple tgtcopy_stmt = NULL;
8873 tree sender = TREE_VEC_ELT (data_arg, 0);
8875 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
8877 gcc_assert (!gsi_end_p (gsi));
8878 stmt = gsi_stmt (gsi);
8879 if (gimple_code (stmt) != GIMPLE_ASSIGN)
8880 continue;
8882 if (gimple_num_ops (stmt) == 2)
8884 tree arg = gimple_assign_rhs1 (stmt);
8886 /* We're ignoring the subcode because we're
8887 effectively doing a STRIP_NOPS. */
8889 if (TREE_CODE (arg) == ADDR_EXPR
8890 && TREE_OPERAND (arg, 0) == sender)
8892 tgtcopy_stmt = stmt;
8893 break;
8898 gcc_assert (tgtcopy_stmt != NULL);
8899 arg = DECL_ARGUMENTS (child_fn);
8901 gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg);
8902 gsi_remove (&gsi, true);
8905 /* Declare local variables needed in CHILD_CFUN. */
8906 block = DECL_INITIAL (child_fn);
8907 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
8908 /* The gimplifier could record temporaries in the offloading block
8909 rather than in containing function's local_decls chain,
8910 which would mean cgraph missed finalizing them. Do it now. */
8911 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
8912 if (TREE_CODE (t) == VAR_DECL
8913 && TREE_STATIC (t)
8914 && !DECL_EXTERNAL (t))
8915 varpool_node::finalize_decl (t);
8916 DECL_SAVED_TREE (child_fn) = NULL;
8917 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8918 gimple_set_body (child_fn, NULL);
8919 TREE_USED (block) = 1;
8921 /* Reset DECL_CONTEXT on function arguments. */
8922 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
8923 DECL_CONTEXT (t) = child_fn;
8925 /* Split ENTRY_BB at GIMPLE_*,
8926 so that it can be moved to the child function. */
8927 gsi = gsi_last_bb (entry_bb);
8928 stmt = gsi_stmt (gsi);
8929 gcc_assert (stmt
8930 && gimple_code (stmt) == gimple_code (entry_stmt));
8931 e = split_block (entry_bb, stmt);
8932 gsi_remove (&gsi, true);
8933 entry_bb = e->dest;
8934 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8936 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8937 if (exit_bb)
8939 gsi = gsi_last_bb (exit_bb);
8940 gcc_assert (!gsi_end_p (gsi)
8941 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
8942 stmt = gimple_build_return (NULL);
8943 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
8944 gsi_remove (&gsi, true);
8947 /* Move the offloading region into CHILD_CFUN. */
8949 block = gimple_block (entry_stmt);
8951 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
8952 if (exit_bb)
8953 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
8954 /* When the OMP expansion process cannot guarantee an up-to-date
8955 loop tree arrange for the child function to fixup loops. */
8956 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
8957 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
8959 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8960 num = vec_safe_length (child_cfun->local_decls);
8961 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
8963 t = (*child_cfun->local_decls)[srcidx];
8964 if (DECL_CONTEXT (t) == cfun->decl)
8965 continue;
8966 if (srcidx != dstidx)
8967 (*child_cfun->local_decls)[dstidx] = t;
8968 dstidx++;
8970 if (dstidx != num)
8971 vec_safe_truncate (child_cfun->local_decls, dstidx);
8973 /* Inform the callgraph about the new function. */
8974 child_cfun->curr_properties = cfun->curr_properties;
8975 child_cfun->has_simduid_loops |= cfun->has_simduid_loops;
8976 child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops;
8977 cgraph_node::add_new_function (child_fn, true);
8979 #ifdef ENABLE_OFFLOADING
8980 /* Add the new function to the offload table. */
8981 vec_safe_push (offload_funcs, child_fn);
8982 #endif
8984 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8985 fixed in a following pass. */
8986 push_cfun (child_cfun);
8987 cgraph_edge::rebuild_edges ();
8989 #ifdef ENABLE_OFFLOADING
8990 /* Prevent IPA from removing child_fn as unreachable, since there are no
8991 refs from the parent function to child_fn in offload LTO mode. */
8992 struct cgraph_node *node = cgraph_node::get (child_fn);
8993 node->mark_force_output ();
8994 #endif
8996 /* Some EH regions might become dead, see PR34608. If
8997 pass_cleanup_cfg isn't the first pass to happen with the
8998 new child, these dead EH edges might cause problems.
8999 Clean them up now. */
9000 if (flag_exceptions)
9002 basic_block bb;
9003 bool changed = false;
9005 FOR_EACH_BB_FN (bb, cfun)
9006 changed |= gimple_purge_dead_eh_edges (bb);
9007 if (changed)
9008 cleanup_tree_cfg ();
9010 pop_cfun ();
9013 /* Emit a library call to launch the offloading region, or do data
9014 transfers. */
9015 tree t1, t2, t3, t4, device, cond, c, clauses;
9016 enum built_in_function start_ix;
9017 location_t clause_loc;
9019 switch (gimple_omp_target_kind (entry_stmt))
9021 case GF_OMP_TARGET_KIND_REGION:
9022 start_ix = BUILT_IN_GOMP_TARGET;
9023 break;
9024 case GF_OMP_TARGET_KIND_DATA:
9025 start_ix = BUILT_IN_GOMP_TARGET_DATA;
9026 break;
9027 case GF_OMP_TARGET_KIND_UPDATE:
9028 start_ix = BUILT_IN_GOMP_TARGET_UPDATE;
9029 break;
9030 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
9031 case GF_OMP_TARGET_KIND_OACC_KERNELS:
9032 start_ix = BUILT_IN_GOACC_PARALLEL;
9033 break;
9034 case GF_OMP_TARGET_KIND_OACC_DATA:
9035 start_ix = BUILT_IN_GOACC_DATA_START;
9036 break;
9037 case GF_OMP_TARGET_KIND_OACC_UPDATE:
9038 start_ix = BUILT_IN_GOACC_UPDATE;
9039 break;
9040 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
9041 start_ix = BUILT_IN_GOACC_ENTER_EXIT_DATA;
9042 break;
9043 default:
9044 gcc_unreachable ();
9047 clauses = gimple_omp_target_clauses (entry_stmt);
9049 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
9050 library choose) and there is no conditional. */
9051 cond = NULL_TREE;
9052 device = build_int_cst (integer_type_node, GOMP_DEVICE_ICV);
9054 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
9055 if (c)
9056 cond = OMP_CLAUSE_IF_EXPR (c);
9058 c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE);
9059 if (c)
9061 /* Even if we pass it to all library function calls, it is currently only
9062 defined/used for the OpenMP target ones. */
9063 gcc_checking_assert (start_ix == BUILT_IN_GOMP_TARGET
9064 || start_ix == BUILT_IN_GOMP_TARGET_DATA
9065 || start_ix == BUILT_IN_GOMP_TARGET_UPDATE);
9067 device = OMP_CLAUSE_DEVICE_ID (c);
9068 clause_loc = OMP_CLAUSE_LOCATION (c);
9070 else
9071 clause_loc = gimple_location (entry_stmt);
9073 /* Ensure 'device' is of the correct type. */
9074 device = fold_convert_loc (clause_loc, integer_type_node, device);
9076 /* If we found the clause 'if (cond)', build
9077 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
9078 if (cond)
9080 cond = gimple_boolify (cond);
9082 basic_block cond_bb, then_bb, else_bb;
9083 edge e;
9084 tree tmp_var;
9086 tmp_var = create_tmp_var (TREE_TYPE (device));
9087 if (offloaded)
9088 e = split_block (new_bb, NULL);
9089 else
9091 gsi = gsi_last_bb (new_bb);
9092 gsi_prev (&gsi);
9093 e = split_block (new_bb, gsi_stmt (gsi));
9095 cond_bb = e->src;
9096 new_bb = e->dest;
9097 remove_edge (e);
9099 then_bb = create_empty_bb (cond_bb);
9100 else_bb = create_empty_bb (then_bb);
9101 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
9102 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
9104 stmt = gimple_build_cond_empty (cond);
9105 gsi = gsi_last_bb (cond_bb);
9106 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
9108 gsi = gsi_start_bb (then_bb);
9109 stmt = gimple_build_assign (tmp_var, device);
9110 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
9112 gsi = gsi_start_bb (else_bb);
9113 stmt = gimple_build_assign (tmp_var,
9114 build_int_cst (integer_type_node,
9115 GOMP_DEVICE_HOST_FALLBACK));
9116 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
9118 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
9119 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
9120 add_bb_to_loop (then_bb, cond_bb->loop_father);
9121 add_bb_to_loop (else_bb, cond_bb->loop_father);
9122 make_edge (then_bb, new_bb, EDGE_FALLTHRU);
9123 make_edge (else_bb, new_bb, EDGE_FALLTHRU);
9125 device = tmp_var;
9126 gsi = gsi_last_bb (new_bb);
9128 else
9130 gsi = gsi_last_bb (new_bb);
9131 device = force_gimple_operand_gsi (&gsi, device, true, NULL_TREE,
9132 true, GSI_SAME_STMT);
9135 t = gimple_omp_target_data_arg (entry_stmt);
9136 if (t == NULL)
9138 t1 = size_zero_node;
9139 t2 = build_zero_cst (ptr_type_node);
9140 t3 = t2;
9141 t4 = t2;
9143 else
9145 t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1))));
9146 t1 = size_binop (PLUS_EXPR, t1, size_int (1));
9147 t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0));
9148 t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1));
9149 t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2));
9152 gimple g;
9153 /* The maximum number used by any start_ix, without varargs. */
9154 auto_vec<tree, 11> args;
9155 args.quick_push (device);
9156 if (offloaded)
9157 args.quick_push (build_fold_addr_expr (child_fn));
9158 switch (start_ix)
9160 case BUILT_IN_GOMP_TARGET:
9161 case BUILT_IN_GOMP_TARGET_DATA:
9162 case BUILT_IN_GOMP_TARGET_UPDATE:
9163 /* This const void * is part of the current ABI, but we're not actually
9164 using it. */
9165 args.quick_push (build_zero_cst (ptr_type_node));
9166 break;
9167 case BUILT_IN_GOACC_DATA_START:
9168 case BUILT_IN_GOACC_ENTER_EXIT_DATA:
9169 case BUILT_IN_GOACC_PARALLEL:
9170 case BUILT_IN_GOACC_UPDATE:
9171 break;
9172 default:
9173 gcc_unreachable ();
9175 args.quick_push (t1);
9176 args.quick_push (t2);
9177 args.quick_push (t3);
9178 args.quick_push (t4);
9179 switch (start_ix)
9181 case BUILT_IN_GOACC_DATA_START:
9182 case BUILT_IN_GOMP_TARGET:
9183 case BUILT_IN_GOMP_TARGET_DATA:
9184 case BUILT_IN_GOMP_TARGET_UPDATE:
9185 break;
9186 case BUILT_IN_GOACC_PARALLEL:
9188 tree t_num_gangs, t_num_workers, t_vector_length;
9190 /* Default values for num_gangs, num_workers, and vector_length. */
9191 t_num_gangs = t_num_workers = t_vector_length
9192 = fold_convert_loc (gimple_location (entry_stmt),
9193 integer_type_node, integer_one_node);
9194 /* ..., but if present, use the value specified by the respective
9195 clause, making sure that are of the correct type. */
9196 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS);
9197 if (c)
9198 t_num_gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
9199 integer_type_node,
9200 OMP_CLAUSE_NUM_GANGS_EXPR (c));
9201 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_WORKERS);
9202 if (c)
9203 t_num_workers = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
9204 integer_type_node,
9205 OMP_CLAUSE_NUM_WORKERS_EXPR (c));
9206 c = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH);
9207 if (c)
9208 t_vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
9209 integer_type_node,
9210 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c));
9211 args.quick_push (t_num_gangs);
9212 args.quick_push (t_num_workers);
9213 args.quick_push (t_vector_length);
9215 /* FALLTHRU */
9216 case BUILT_IN_GOACC_ENTER_EXIT_DATA:
9217 case BUILT_IN_GOACC_UPDATE:
9219 tree t_async;
9220 int t_wait_idx;
9222 /* Default values for t_async. */
9223 t_async = fold_convert_loc (gimple_location (entry_stmt),
9224 integer_type_node,
9225 build_int_cst (integer_type_node,
9226 GOMP_ASYNC_SYNC));
9227 /* ..., but if present, use the value specified by the respective
9228 clause, making sure that is of the correct type. */
9229 c = find_omp_clause (clauses, OMP_CLAUSE_ASYNC);
9230 if (c)
9231 t_async = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
9232 integer_type_node,
9233 OMP_CLAUSE_ASYNC_EXPR (c));
9235 args.quick_push (t_async);
9236 /* Save the index, and... */
9237 t_wait_idx = args.length ();
9238 /* ... push a default value. */
9239 args.quick_push (fold_convert_loc (gimple_location (entry_stmt),
9240 integer_type_node,
9241 integer_zero_node));
9242 c = find_omp_clause (clauses, OMP_CLAUSE_WAIT);
9243 if (c)
9245 int n = 0;
9247 for (; c; c = OMP_CLAUSE_CHAIN (c))
9249 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WAIT)
9251 args.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c),
9252 integer_type_node,
9253 OMP_CLAUSE_WAIT_EXPR (c)));
9254 n++;
9258 /* Now that we know the number, replace the default value. */
9259 args.ordered_remove (t_wait_idx);
9260 args.quick_insert (t_wait_idx,
9261 fold_convert_loc (gimple_location (entry_stmt),
9262 integer_type_node,
9263 build_int_cst (integer_type_node, n)));
9266 break;
9267 default:
9268 gcc_unreachable ();
9271 g = gimple_build_call_vec (builtin_decl_explicit (start_ix), args);
9272 gimple_set_location (g, gimple_location (entry_stmt));
9273 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9274 if (!offloaded)
9276 g = gsi_stmt (gsi);
9277 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET);
9278 gsi_remove (&gsi, true);
9280 if (data_region
9281 && region->exit)
9283 gsi = gsi_last_bb (region->exit);
9284 g = gsi_stmt (gsi);
9285 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN);
9286 gsi_remove (&gsi, true);
9291 /* Expand the parallel region tree rooted at REGION. Expansion
9292 proceeds in depth-first order. Innermost regions are expanded
9293 first. This way, parallel regions that require a new function to
9294 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9295 internal dependencies in their body. */
9297 static void
9298 expand_omp (struct omp_region *region)
9300 while (region)
9302 location_t saved_location;
9303 gimple inner_stmt = NULL;
9305 /* First, determine whether this is a combined parallel+workshare
9306 region. */
9307 if (region->type == GIMPLE_OMP_PARALLEL)
9308 determine_parallel_type (region);
9310 if (region->type == GIMPLE_OMP_FOR
9311 && gimple_omp_for_combined_p (last_stmt (region->entry)))
9312 inner_stmt = last_stmt (region->inner->entry);
9314 if (region->inner)
9315 expand_omp (region->inner);
9317 saved_location = input_location;
9318 if (gimple_has_location (last_stmt (region->entry)))
9319 input_location = gimple_location (last_stmt (region->entry));
9321 switch (region->type)
9323 case GIMPLE_OMP_PARALLEL:
9324 case GIMPLE_OMP_TASK:
9325 expand_omp_taskreg (region);
9326 break;
9328 case GIMPLE_OMP_FOR:
9329 expand_omp_for (region, inner_stmt);
9330 break;
9332 case GIMPLE_OMP_SECTIONS:
9333 expand_omp_sections (region);
9334 break;
9336 case GIMPLE_OMP_SECTION:
9337 /* Individual omp sections are handled together with their
9338 parent GIMPLE_OMP_SECTIONS region. */
9339 break;
9341 case GIMPLE_OMP_SINGLE:
9342 expand_omp_single (region);
9343 break;
9345 case GIMPLE_OMP_MASTER:
9346 case GIMPLE_OMP_TASKGROUP:
9347 case GIMPLE_OMP_ORDERED:
9348 case GIMPLE_OMP_CRITICAL:
9349 case GIMPLE_OMP_TEAMS:
9350 expand_omp_synch (region);
9351 break;
9353 case GIMPLE_OMP_ATOMIC_LOAD:
9354 expand_omp_atomic (region);
9355 break;
9357 case GIMPLE_OMP_TARGET:
9358 expand_omp_target (region);
9359 break;
9361 default:
9362 gcc_unreachable ();
9365 input_location = saved_location;
9366 region = region->next;
9371 /* Helper for build_omp_regions. Scan the dominator tree starting at
9372 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9373 true, the function ends once a single tree is built (otherwise, whole
9374 forest of OMP constructs may be built). */
9376 static void
9377 build_omp_regions_1 (basic_block bb, struct omp_region *parent,
9378 bool single_tree)
9380 gimple_stmt_iterator gsi;
9381 gimple stmt;
9382 basic_block son;
9384 gsi = gsi_last_bb (bb);
9385 if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi)))
9387 struct omp_region *region;
9388 enum gimple_code code;
9390 stmt = gsi_stmt (gsi);
9391 code = gimple_code (stmt);
9392 if (code == GIMPLE_OMP_RETURN)
9394 /* STMT is the return point out of region PARENT. Mark it
9395 as the exit point and make PARENT the immediately
9396 enclosing region. */
9397 gcc_assert (parent);
9398 region = parent;
9399 region->exit = bb;
9400 parent = parent->outer;
9402 else if (code == GIMPLE_OMP_ATOMIC_STORE)
9404 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9405 GIMPLE_OMP_RETURN, but matches with
9406 GIMPLE_OMP_ATOMIC_LOAD. */
9407 gcc_assert (parent);
9408 gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD);
9409 region = parent;
9410 region->exit = bb;
9411 parent = parent->outer;
9413 else if (code == GIMPLE_OMP_CONTINUE)
9415 gcc_assert (parent);
9416 parent->cont = bb;
9418 else if (code == GIMPLE_OMP_SECTIONS_SWITCH)
9420 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9421 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9423 else
9425 region = new_omp_region (bb, code, parent);
9426 /* Otherwise... */
9427 if (code == GIMPLE_OMP_TARGET)
9429 switch (gimple_omp_target_kind (stmt))
9431 case GF_OMP_TARGET_KIND_REGION:
9432 case GF_OMP_TARGET_KIND_DATA:
9433 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
9434 case GF_OMP_TARGET_KIND_OACC_KERNELS:
9435 case GF_OMP_TARGET_KIND_OACC_DATA:
9436 break;
9437 case GF_OMP_TARGET_KIND_UPDATE:
9438 case GF_OMP_TARGET_KIND_OACC_UPDATE:
9439 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
9440 /* ..., other than for those stand-alone directives... */
9441 region = NULL;
9442 break;
9443 default:
9444 gcc_unreachable ();
9447 /* ..., this directive becomes the parent for a new region. */
9448 if (region)
9449 parent = region;
9453 if (single_tree && !parent)
9454 return;
9456 for (son = first_dom_son (CDI_DOMINATORS, bb);
9457 son;
9458 son = next_dom_son (CDI_DOMINATORS, son))
9459 build_omp_regions_1 (son, parent, single_tree);
9462 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9463 root_omp_region. */
9465 static void
9466 build_omp_regions_root (basic_block root)
9468 gcc_assert (root_omp_region == NULL);
9469 build_omp_regions_1 (root, NULL, true);
9470 gcc_assert (root_omp_region != NULL);
9473 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9475 void
9476 omp_expand_local (basic_block head)
9478 build_omp_regions_root (head);
9479 if (dump_file && (dump_flags & TDF_DETAILS))
9481 fprintf (dump_file, "\nOMP region tree\n\n");
9482 dump_omp_region (dump_file, root_omp_region, 0);
9483 fprintf (dump_file, "\n");
9486 remove_exit_barriers (root_omp_region);
9487 expand_omp (root_omp_region);
9489 free_omp_regions ();
9492 /* Scan the CFG and build a tree of OMP regions. Return the root of
9493 the OMP region tree. */
9495 static void
9496 build_omp_regions (void)
9498 gcc_assert (root_omp_region == NULL);
9499 calculate_dominance_info (CDI_DOMINATORS);
9500 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false);
9503 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9505 static unsigned int
9506 execute_expand_omp (void)
9508 build_omp_regions ();
9510 if (!root_omp_region)
9511 return 0;
9513 if (dump_file)
9515 fprintf (dump_file, "\nOMP region tree\n\n");
9516 dump_omp_region (dump_file, root_omp_region, 0);
9517 fprintf (dump_file, "\n");
9520 remove_exit_barriers (root_omp_region);
9522 expand_omp (root_omp_region);
9524 cleanup_tree_cfg ();
9526 free_omp_regions ();
9528 return 0;
9531 /* OMP expansion -- the default pass, run before creation of SSA form. */
9533 namespace {
9535 const pass_data pass_data_expand_omp =
9537 GIMPLE_PASS, /* type */
9538 "ompexp", /* name */
9539 OPTGROUP_NONE, /* optinfo_flags */
9540 TV_NONE, /* tv_id */
9541 PROP_gimple_any, /* properties_required */
9542 PROP_gimple_eomp, /* properties_provided */
9543 0, /* properties_destroyed */
9544 0, /* todo_flags_start */
9545 0, /* todo_flags_finish */
9548 class pass_expand_omp : public gimple_opt_pass
9550 public:
9551 pass_expand_omp (gcc::context *ctxt)
9552 : gimple_opt_pass (pass_data_expand_omp, ctxt)
9555 /* opt_pass methods: */
9556 virtual unsigned int execute (function *)
9558 bool gate = ((flag_cilkplus != 0 || flag_openacc != 0 || flag_openmp != 0
9559 || flag_openmp_simd != 0)
9560 && !seen_error ());
9562 /* This pass always runs, to provide PROP_gimple_eomp.
9563 But often, there is nothing to do. */
9564 if (!gate)
9565 return 0;
9567 return execute_expand_omp ();
9570 }; // class pass_expand_omp
9572 } // anon namespace
9574 gimple_opt_pass *
9575 make_pass_expand_omp (gcc::context *ctxt)
9577 return new pass_expand_omp (ctxt);
9580 namespace {
9582 const pass_data pass_data_expand_omp_ssa =
9584 GIMPLE_PASS, /* type */
9585 "ompexpssa", /* name */
9586 OPTGROUP_NONE, /* optinfo_flags */
9587 TV_NONE, /* tv_id */
9588 PROP_cfg | PROP_ssa, /* properties_required */
9589 PROP_gimple_eomp, /* properties_provided */
9590 0, /* properties_destroyed */
9591 0, /* todo_flags_start */
9592 TODO_cleanup_cfg | TODO_rebuild_alias, /* todo_flags_finish */
9595 class pass_expand_omp_ssa : public gimple_opt_pass
9597 public:
9598 pass_expand_omp_ssa (gcc::context *ctxt)
9599 : gimple_opt_pass (pass_data_expand_omp_ssa, ctxt)
9602 /* opt_pass methods: */
9603 virtual bool gate (function *fun)
9605 return !(fun->curr_properties & PROP_gimple_eomp);
9607 virtual unsigned int execute (function *) { return execute_expand_omp (); }
9609 }; // class pass_expand_omp_ssa
9611 } // anon namespace
9613 gimple_opt_pass *
9614 make_pass_expand_omp_ssa (gcc::context *ctxt)
9616 return new pass_expand_omp_ssa (ctxt);
9619 /* Routines to lower OMP directives into OMP-GIMPLE. */
9621 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9622 convert it to gimple. */
9623 static void
9624 oacc_gimple_assign (tree dest, tree_code op, tree src, gimple_seq *seq)
9626 gimple stmt;
9628 if (TREE_CODE (TREE_TYPE (dest)) != COMPLEX_TYPE)
9630 stmt = gimple_build_assign (dest, op, dest, src);
9631 gimple_seq_add_stmt (seq, stmt);
9632 return;
9635 tree t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9636 tree rdest = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest);
9637 gimplify_assign (t, rdest, seq);
9638 rdest = t;
9640 t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9641 tree idest = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest);
9642 gimplify_assign (t, idest, seq);
9643 idest = t;
9645 t = create_tmp_var (TREE_TYPE (TREE_TYPE (src)));
9646 tree rsrc = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src);
9647 gimplify_assign (t, rsrc, seq);
9648 rsrc = t;
9650 t = create_tmp_var (TREE_TYPE (TREE_TYPE (src)));
9651 tree isrc = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src);
9652 gimplify_assign (t, isrc, seq);
9653 isrc = t;
9655 tree r = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9656 tree i = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9657 tree result;
9659 if (op == PLUS_EXPR)
9661 stmt = gimple_build_assign (r, op, rdest, rsrc);
9662 gimple_seq_add_stmt (seq, stmt);
9664 stmt = gimple_build_assign (i, op, idest, isrc);
9665 gimple_seq_add_stmt (seq, stmt);
9667 else if (op == MULT_EXPR)
9669 /* Let x = a + ib = dest, y = c + id = src.
9670 x * y = (ac - bd) + i(ad + bc) */
9671 tree ac = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9672 tree bd = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9673 tree ad = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9674 tree bc = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9676 stmt = gimple_build_assign (ac, MULT_EXPR, rdest, rsrc);
9677 gimple_seq_add_stmt (seq, stmt);
9679 stmt = gimple_build_assign (bd, MULT_EXPR, idest, isrc);
9680 gimple_seq_add_stmt (seq, stmt);
9682 stmt = gimple_build_assign (r, MINUS_EXPR, ac, bd);
9683 gimple_seq_add_stmt (seq, stmt);
9685 stmt = gimple_build_assign (ad, MULT_EXPR, rdest, isrc);
9686 gimple_seq_add_stmt (seq, stmt);
9688 stmt = gimple_build_assign (bd, MULT_EXPR, idest, rsrc);
9689 gimple_seq_add_stmt (seq, stmt);
9691 stmt = gimple_build_assign (i, PLUS_EXPR, ad, bc);
9692 gimple_seq_add_stmt (seq, stmt);
9694 else
9695 gcc_unreachable ();
9697 result = build2 (COMPLEX_EXPR, TREE_TYPE (dest), r, i);
9698 gimplify_assign (dest, result, seq);
9701 /* Helper function to initialize local data for the reduction arrays.
9702 The reduction arrays need to be placed inside the calling function
9703 for accelerators, or else the host won't be able to preform the final
9704 reduction. */
9706 static void
9707 oacc_initialize_reduction_data (tree clauses, tree nthreads,
9708 gimple_seq *stmt_seqp, omp_context *ctx)
9710 tree c, t, oc;
9711 gimple stmt;
9712 omp_context *octx;
9714 /* Find the innermost OpenACC parallel context. */
9715 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
9716 && (gimple_omp_target_kind (ctx->stmt)
9717 == GF_OMP_TARGET_KIND_OACC_PARALLEL))
9718 octx = ctx;
9719 else
9720 octx = ctx->outer;
9721 gcc_checking_assert (gimple_code (octx->stmt) == GIMPLE_OMP_TARGET
9722 && (gimple_omp_target_kind (octx->stmt)
9723 == GF_OMP_TARGET_KIND_OACC_PARALLEL));
9725 /* Extract the clauses. */
9726 oc = gimple_omp_target_clauses (octx->stmt);
9728 /* Find the last outer clause. */
9729 for (; oc && OMP_CLAUSE_CHAIN (oc); oc = OMP_CLAUSE_CHAIN (oc))
9732 /* Allocate arrays for each reduction variable. */
9733 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9735 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
9736 continue;
9738 tree var = OMP_CLAUSE_DECL (c);
9739 tree type = get_base_type (var);
9740 tree array = lookup_oacc_reduction (oacc_get_reduction_array_id (var),
9741 ctx);
9742 tree size, call;
9744 /* Calculate size of the reduction array. */
9745 t = create_tmp_var (TREE_TYPE (nthreads));
9746 stmt = gimple_build_assign (t, MULT_EXPR, nthreads,
9747 fold_convert (TREE_TYPE (nthreads),
9748 TYPE_SIZE_UNIT (type)));
9749 gimple_seq_add_stmt (stmt_seqp, stmt);
9751 size = create_tmp_var (sizetype);
9752 gimplify_assign (size, fold_build1 (NOP_EXPR, sizetype, t), stmt_seqp);
9754 /* Now allocate memory for it. */
9755 call = unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA));
9756 stmt = gimple_build_call (call, 1, size);
9757 gimple_call_set_lhs (stmt, array);
9758 gimple_seq_add_stmt (stmt_seqp, stmt);
9760 /* Map this array into the accelerator. */
9762 /* Add the reduction array to the list of clauses. */
9763 tree x = array;
9764 t = build_omp_clause (gimple_location (ctx->stmt), OMP_CLAUSE_MAP);
9765 OMP_CLAUSE_SET_MAP_KIND (t, GOMP_MAP_FORCE_FROM);
9766 OMP_CLAUSE_DECL (t) = x;
9767 OMP_CLAUSE_CHAIN (t) = NULL;
9768 if (oc)
9769 OMP_CLAUSE_CHAIN (oc) = t;
9770 else
9771 gimple_omp_target_set_clauses (as_a <gomp_target *> (octx->stmt), t);
9772 OMP_CLAUSE_SIZE (t) = size;
9773 oc = t;
9777 /* Helper function to process the array of partial reductions. Nthreads
9778 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9779 cannot be used here, because nthreads on the host may be different than
9780 on the accelerator. */
9782 static void
9783 oacc_finalize_reduction_data (tree clauses, tree nthreads,
9784 gimple_seq *stmt_seqp, omp_context *ctx)
9786 tree c, x, var, array, loop_header, loop_body, loop_exit, type;
9787 gimple stmt;
9789 /* Create for loop.
9791 let var = the original reduction variable
9792 let array = reduction variable array
9794 for (i = 0; i < nthreads; i++)
9795 var op= array[i]
9798 loop_header = create_artificial_label (UNKNOWN_LOCATION);
9799 loop_body = create_artificial_label (UNKNOWN_LOCATION);
9800 loop_exit = create_artificial_label (UNKNOWN_LOCATION);
9802 /* Create and initialize an index variable. */
9803 tree ix = create_tmp_var (sizetype);
9804 gimplify_assign (ix, fold_build1 (NOP_EXPR, sizetype, integer_zero_node),
9805 stmt_seqp);
9807 /* Insert the loop header label here. */
9808 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_header));
9810 /* Exit loop if ix >= nthreads. */
9811 x = create_tmp_var (sizetype);
9812 gimplify_assign (x, fold_build1 (NOP_EXPR, sizetype, nthreads), stmt_seqp);
9813 stmt = gimple_build_cond (GE_EXPR, ix, x, loop_exit, loop_body);
9814 gimple_seq_add_stmt (stmt_seqp, stmt);
9816 /* Insert the loop body label here. */
9817 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_body));
9819 /* Collapse each reduction array, one element at a time. */
9820 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9822 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
9823 continue;
9825 tree_code reduction_code = OMP_CLAUSE_REDUCTION_CODE (c);
9827 /* reduction(-:var) sums up the partial results, so it acts
9828 identically to reduction(+:var). */
9829 if (reduction_code == MINUS_EXPR)
9830 reduction_code = PLUS_EXPR;
9832 /* Set up reduction variable var. */
9833 var = OMP_CLAUSE_DECL (c);
9834 type = get_base_type (var);
9835 array = lookup_oacc_reduction (oacc_get_reduction_array_id
9836 (OMP_CLAUSE_DECL (c)), ctx);
9838 /* Calculate the array offset. */
9839 tree offset = create_tmp_var (sizetype);
9840 gimplify_assign (offset, TYPE_SIZE_UNIT (type), stmt_seqp);
9841 stmt = gimple_build_assign (offset, MULT_EXPR, offset, ix);
9842 gimple_seq_add_stmt (stmt_seqp, stmt);
9844 tree ptr = create_tmp_var (TREE_TYPE (array));
9845 stmt = gimple_build_assign (ptr, POINTER_PLUS_EXPR, array, offset);
9846 gimple_seq_add_stmt (stmt_seqp, stmt);
9848 /* Extract array[ix] into mem. */
9849 tree mem = create_tmp_var (type);
9850 gimplify_assign (mem, build_simple_mem_ref (ptr), stmt_seqp);
9852 /* Find the original reduction variable. */
9853 if (is_reference (var))
9854 var = build_simple_mem_ref (var);
9856 tree t = create_tmp_var (type);
9858 x = lang_hooks.decls.omp_clause_assign_op (c, t, var);
9859 gimplify_and_add (unshare_expr(x), stmt_seqp);
9861 /* var = var op mem */
9862 switch (OMP_CLAUSE_REDUCTION_CODE (c))
9864 case TRUTH_ANDIF_EXPR:
9865 case TRUTH_ORIF_EXPR:
9866 t = fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c), integer_type_node,
9867 t, mem);
9868 gimplify_and_add (t, stmt_seqp);
9869 break;
9870 default:
9871 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9872 oacc_gimple_assign (t, OMP_CLAUSE_REDUCTION_CODE (c), mem,
9873 stmt_seqp);
9876 t = fold_build1 (NOP_EXPR, TREE_TYPE (var), t);
9877 x = lang_hooks.decls.omp_clause_assign_op (c, var, t);
9878 gimplify_and_add (unshare_expr(x), stmt_seqp);
9881 /* Increment the induction variable. */
9882 tree one = fold_build1 (NOP_EXPR, sizetype, integer_one_node);
9883 stmt = gimple_build_assign (ix, PLUS_EXPR, ix, one);
9884 gimple_seq_add_stmt (stmt_seqp, stmt);
9886 /* Go back to the top of the loop. */
9887 gimple_seq_add_stmt (stmt_seqp, gimple_build_goto (loop_header));
9889 /* Place the loop exit label here. */
9890 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_exit));
9893 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9894 scan that for reductions. */
9896 static void
9897 oacc_process_reduction_data (gimple_seq *body, gimple_seq *in_stmt_seqp,
9898 gimple_seq *out_stmt_seqp, omp_context *ctx)
9900 gimple_stmt_iterator gsi;
9901 gimple_seq inner = NULL;
9903 /* A collapse clause may have inserted a new bind block. */
9904 gsi = gsi_start (*body);
9905 while (!gsi_end_p (gsi))
9907 gimple stmt = gsi_stmt (gsi);
9908 if (gbind *bind_stmt = dyn_cast <gbind *> (stmt))
9910 inner = gimple_bind_body (bind_stmt);
9911 body = &inner;
9912 gsi = gsi_start (*body);
9914 else if (dyn_cast <gomp_for *> (stmt))
9915 break;
9916 else
9917 gsi_next (&gsi);
9920 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
9922 tree clauses, nthreads, t, c, acc_device, acc_device_host, call,
9923 enter, exit;
9924 bool reduction_found = false;
9926 gimple stmt = gsi_stmt (gsi);
9928 switch (gimple_code (stmt))
9930 case GIMPLE_OMP_FOR:
9931 clauses = gimple_omp_for_clauses (stmt);
9933 /* Search for a reduction clause. */
9934 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9935 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
9937 reduction_found = true;
9938 break;
9941 if (!reduction_found)
9942 break;
9944 ctx = maybe_lookup_ctx (stmt);
9945 t = NULL_TREE;
9947 /* Extract the number of threads. */
9948 nthreads = create_tmp_var (sizetype);
9949 t = oacc_max_threads (ctx);
9950 gimplify_assign (nthreads, t, in_stmt_seqp);
9952 /* Determine if this is kernel will be executed on the host. */
9953 call = builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE);
9954 acc_device = create_tmp_var (integer_type_node, ".acc_device_type");
9955 stmt = gimple_build_call (call, 0);
9956 gimple_call_set_lhs (stmt, acc_device);
9957 gimple_seq_add_stmt (in_stmt_seqp, stmt);
9959 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9960 acc_device_host = create_tmp_var (integer_type_node,
9961 ".acc_device_host");
9962 gimplify_assign (acc_device_host,
9963 build_int_cst (integer_type_node,
9964 GOMP_DEVICE_HOST),
9965 in_stmt_seqp);
9967 enter = create_artificial_label (UNKNOWN_LOCATION);
9968 exit = create_artificial_label (UNKNOWN_LOCATION);
9970 stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host,
9971 enter, exit);
9972 gimple_seq_add_stmt (in_stmt_seqp, stmt);
9973 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter));
9974 gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype,
9975 integer_one_node),
9976 in_stmt_seqp);
9977 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit));
9979 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9980 gimplify_assign (acc_device_host,
9981 build_int_cst (integer_type_node,
9982 GOMP_DEVICE_HOST_NONSHM),
9983 in_stmt_seqp);
9985 enter = create_artificial_label (UNKNOWN_LOCATION);
9986 exit = create_artificial_label (UNKNOWN_LOCATION);
9988 stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host,
9989 enter, exit);
9990 gimple_seq_add_stmt (in_stmt_seqp, stmt);
9991 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter));
9992 gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype,
9993 integer_one_node),
9994 in_stmt_seqp);
9995 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit));
9997 oacc_initialize_reduction_data (clauses, nthreads, in_stmt_seqp,
9998 ctx);
9999 oacc_finalize_reduction_data (clauses, nthreads, out_stmt_seqp, ctx);
10000 break;
10001 default:
10002 // Scan for other directives which support reduction here.
10003 break;
10008 /* If ctx is a worksharing context inside of a cancellable parallel
10009 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
10010 and conditional branch to parallel's cancel_label to handle
10011 cancellation in the implicit barrier. */
10013 static void
10014 maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body)
10016 gimple omp_return = gimple_seq_last_stmt (*body);
10017 gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
10018 if (gimple_omp_return_nowait_p (omp_return))
10019 return;
10020 if (ctx->outer
10021 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL
10022 && ctx->outer->cancellable)
10024 tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
10025 tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
10026 tree lhs = create_tmp_var (c_bool_type);
10027 gimple_omp_return_set_lhs (omp_return, lhs);
10028 tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
10029 gimple g = gimple_build_cond (NE_EXPR, lhs,
10030 fold_convert (c_bool_type,
10031 boolean_false_node),
10032 ctx->outer->cancel_label, fallthru_label);
10033 gimple_seq_add_stmt (body, g);
10034 gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
10038 /* Lower the OpenMP sections directive in the current statement in GSI_P.
10039 CTX is the enclosing OMP context for the current statement. */
10041 static void
10042 lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10044 tree block, control;
10045 gimple_stmt_iterator tgsi;
10046 gomp_sections *stmt;
10047 gimple t;
10048 gbind *new_stmt, *bind;
10049 gimple_seq ilist, dlist, olist, new_body;
10051 stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p));
10053 push_gimplify_context ();
10055 dlist = NULL;
10056 ilist = NULL;
10057 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
10058 &ilist, &dlist, ctx, NULL);
10060 new_body = gimple_omp_body (stmt);
10061 gimple_omp_set_body (stmt, NULL);
10062 tgsi = gsi_start (new_body);
10063 for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
10065 omp_context *sctx;
10066 gimple sec_start;
10068 sec_start = gsi_stmt (tgsi);
10069 sctx = maybe_lookup_ctx (sec_start);
10070 gcc_assert (sctx);
10072 lower_omp (gimple_omp_body_ptr (sec_start), sctx);
10073 gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
10074 GSI_CONTINUE_LINKING);
10075 gimple_omp_set_body (sec_start, NULL);
10077 if (gsi_one_before_end_p (tgsi))
10079 gimple_seq l = NULL;
10080 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
10081 &l, ctx);
10082 gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
10083 gimple_omp_section_set_last (sec_start);
10086 gsi_insert_after (&tgsi, gimple_build_omp_return (false),
10087 GSI_CONTINUE_LINKING);
10090 block = make_node (BLOCK);
10091 bind = gimple_build_bind (NULL, new_body, block);
10093 olist = NULL;
10094 lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx);
10096 block = make_node (BLOCK);
10097 new_stmt = gimple_build_bind (NULL, NULL, block);
10098 gsi_replace (gsi_p, new_stmt, true);
10100 pop_gimplify_context (new_stmt);
10101 gimple_bind_append_vars (new_stmt, ctx->block_vars);
10102 BLOCK_VARS (block) = gimple_bind_vars (bind);
10103 if (BLOCK_VARS (block))
10104 TREE_USED (block) = 1;
10106 new_body = NULL;
10107 gimple_seq_add_seq (&new_body, ilist);
10108 gimple_seq_add_stmt (&new_body, stmt);
10109 gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
10110 gimple_seq_add_stmt (&new_body, bind);
10112 control = create_tmp_var (unsigned_type_node, ".section");
10113 t = gimple_build_omp_continue (control, control);
10114 gimple_omp_sections_set_control (stmt, control);
10115 gimple_seq_add_stmt (&new_body, t);
10117 gimple_seq_add_seq (&new_body, olist);
10118 if (ctx->cancellable)
10119 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
10120 gimple_seq_add_seq (&new_body, dlist);
10122 new_body = maybe_catch_exception (new_body);
10124 t = gimple_build_omp_return
10125 (!!find_omp_clause (gimple_omp_sections_clauses (stmt),
10126 OMP_CLAUSE_NOWAIT));
10127 gimple_seq_add_stmt (&new_body, t);
10128 maybe_add_implicit_barrier_cancel (ctx, &new_body);
10130 gimple_bind_set_body (new_stmt, new_body);
10134 /* A subroutine of lower_omp_single. Expand the simple form of
10135 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10137 if (GOMP_single_start ())
10138 BODY;
10139 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10141 FIXME. It may be better to delay expanding the logic of this until
10142 pass_expand_omp. The expanded logic may make the job more difficult
10143 to a synchronization analysis pass. */
10145 static void
10146 lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p)
10148 location_t loc = gimple_location (single_stmt);
10149 tree tlabel = create_artificial_label (loc);
10150 tree flabel = create_artificial_label (loc);
10151 gimple call, cond;
10152 tree lhs, decl;
10154 decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START);
10155 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
10156 call = gimple_build_call (decl, 0);
10157 gimple_call_set_lhs (call, lhs);
10158 gimple_seq_add_stmt (pre_p, call);
10160 cond = gimple_build_cond (EQ_EXPR, lhs,
10161 fold_convert_loc (loc, TREE_TYPE (lhs),
10162 boolean_true_node),
10163 tlabel, flabel);
10164 gimple_seq_add_stmt (pre_p, cond);
10165 gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
10166 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
10167 gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
10171 /* A subroutine of lower_omp_single. Expand the simple form of
10172 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10174 #pragma omp single copyprivate (a, b, c)
10176 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10179 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10181 BODY;
10182 copyout.a = a;
10183 copyout.b = b;
10184 copyout.c = c;
10185 GOMP_single_copy_end (&copyout);
10187 else
10189 a = copyout_p->a;
10190 b = copyout_p->b;
10191 c = copyout_p->c;
10193 GOMP_barrier ();
10196 FIXME. It may be better to delay expanding the logic of this until
10197 pass_expand_omp. The expanded logic may make the job more difficult
10198 to a synchronization analysis pass. */
10200 static void
10201 lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p,
10202 omp_context *ctx)
10204 tree ptr_type, t, l0, l1, l2, bfn_decl;
10205 gimple_seq copyin_seq;
10206 location_t loc = gimple_location (single_stmt);
10208 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
10210 ptr_type = build_pointer_type (ctx->record_type);
10211 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
10213 l0 = create_artificial_label (loc);
10214 l1 = create_artificial_label (loc);
10215 l2 = create_artificial_label (loc);
10217 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
10218 t = build_call_expr_loc (loc, bfn_decl, 0);
10219 t = fold_convert_loc (loc, ptr_type, t);
10220 gimplify_assign (ctx->receiver_decl, t, pre_p);
10222 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
10223 build_int_cst (ptr_type, 0));
10224 t = build3 (COND_EXPR, void_type_node, t,
10225 build_and_jump (&l0), build_and_jump (&l1));
10226 gimplify_and_add (t, pre_p);
10228 gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
10230 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
10232 copyin_seq = NULL;
10233 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
10234 &copyin_seq, ctx);
10236 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
10237 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
10238 t = build_call_expr_loc (loc, bfn_decl, 1, t);
10239 gimplify_and_add (t, pre_p);
10241 t = build_and_jump (&l2);
10242 gimplify_and_add (t, pre_p);
10244 gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
10246 gimple_seq_add_seq (pre_p, copyin_seq);
10248 gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
10252 /* Expand code for an OpenMP single directive. */
10254 static void
10255 lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10257 tree block;
10258 gimple t;
10259 gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p));
10260 gbind *bind;
10261 gimple_seq bind_body, bind_body_tail = NULL, dlist;
10263 push_gimplify_context ();
10265 block = make_node (BLOCK);
10266 bind = gimple_build_bind (NULL, NULL, block);
10267 gsi_replace (gsi_p, bind, true);
10268 bind_body = NULL;
10269 dlist = NULL;
10270 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
10271 &bind_body, &dlist, ctx, NULL);
10272 lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
10274 gimple_seq_add_stmt (&bind_body, single_stmt);
10276 if (ctx->record_type)
10277 lower_omp_single_copy (single_stmt, &bind_body, ctx);
10278 else
10279 lower_omp_single_simple (single_stmt, &bind_body);
10281 gimple_omp_set_body (single_stmt, NULL);
10283 gimple_seq_add_seq (&bind_body, dlist);
10285 bind_body = maybe_catch_exception (bind_body);
10287 t = gimple_build_omp_return
10288 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt),
10289 OMP_CLAUSE_NOWAIT));
10290 gimple_seq_add_stmt (&bind_body_tail, t);
10291 maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail);
10292 if (ctx->record_type)
10294 gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
10295 tree clobber = build_constructor (ctx->record_type, NULL);
10296 TREE_THIS_VOLATILE (clobber) = 1;
10297 gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
10298 clobber), GSI_SAME_STMT);
10300 gimple_seq_add_seq (&bind_body, bind_body_tail);
10301 gimple_bind_set_body (bind, bind_body);
10303 pop_gimplify_context (bind);
10305 gimple_bind_append_vars (bind, ctx->block_vars);
10306 BLOCK_VARS (block) = ctx->block_vars;
10307 if (BLOCK_VARS (block))
10308 TREE_USED (block) = 1;
10312 /* Expand code for an OpenMP master directive. */
10314 static void
10315 lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10317 tree block, lab = NULL, x, bfn_decl;
10318 gimple stmt = gsi_stmt (*gsi_p);
10319 gbind *bind;
10320 location_t loc = gimple_location (stmt);
10321 gimple_seq tseq;
10323 push_gimplify_context ();
10325 block = make_node (BLOCK);
10326 bind = gimple_build_bind (NULL, NULL, block);
10327 gsi_replace (gsi_p, bind, true);
10328 gimple_bind_add_stmt (bind, stmt);
10330 bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
10331 x = build_call_expr_loc (loc, bfn_decl, 0);
10332 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node);
10333 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
10334 tseq = NULL;
10335 gimplify_and_add (x, &tseq);
10336 gimple_bind_add_seq (bind, tseq);
10338 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10339 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10340 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10341 gimple_omp_set_body (stmt, NULL);
10343 gimple_bind_add_stmt (bind, gimple_build_label (lab));
10345 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10347 pop_gimplify_context (bind);
10349 gimple_bind_append_vars (bind, ctx->block_vars);
10350 BLOCK_VARS (block) = ctx->block_vars;
10354 /* Expand code for an OpenMP taskgroup directive. */
10356 static void
10357 lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10359 gimple stmt = gsi_stmt (*gsi_p);
10360 gcall *x;
10361 gbind *bind;
10362 tree block = make_node (BLOCK);
10364 bind = gimple_build_bind (NULL, NULL, block);
10365 gsi_replace (gsi_p, bind, true);
10366 gimple_bind_add_stmt (bind, stmt);
10368 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
10370 gimple_bind_add_stmt (bind, x);
10372 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10373 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10374 gimple_omp_set_body (stmt, NULL);
10376 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10378 gimple_bind_append_vars (bind, ctx->block_vars);
10379 BLOCK_VARS (block) = ctx->block_vars;
10383 /* Expand code for an OpenMP ordered directive. */
10385 static void
10386 lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10388 tree block;
10389 gimple stmt = gsi_stmt (*gsi_p);
10390 gcall *x;
10391 gbind *bind;
10393 push_gimplify_context ();
10395 block = make_node (BLOCK);
10396 bind = gimple_build_bind (NULL, NULL, block);
10397 gsi_replace (gsi_p, bind, true);
10398 gimple_bind_add_stmt (bind, stmt);
10400 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
10402 gimple_bind_add_stmt (bind, x);
10404 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10405 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10406 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10407 gimple_omp_set_body (stmt, NULL);
10409 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END), 0);
10410 gimple_bind_add_stmt (bind, x);
10412 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10414 pop_gimplify_context (bind);
10416 gimple_bind_append_vars (bind, ctx->block_vars);
10417 BLOCK_VARS (block) = gimple_bind_vars (bind);
10421 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10422 substitution of a couple of function calls. But in the NAMED case,
10423 requires that languages coordinate a symbol name. It is therefore
10424 best put here in common code. */
10426 static GTY(()) hash_map<tree, tree> *critical_name_mutexes;
10428 static void
10429 lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10431 tree block;
10432 tree name, lock, unlock;
10433 gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p));
10434 gbind *bind;
10435 location_t loc = gimple_location (stmt);
10436 gimple_seq tbody;
10438 name = gimple_omp_critical_name (stmt);
10439 if (name)
10441 tree decl;
10443 if (!critical_name_mutexes)
10444 critical_name_mutexes = hash_map<tree, tree>::create_ggc (10);
10446 tree *n = critical_name_mutexes->get (name);
10447 if (n == NULL)
10449 char *new_str;
10451 decl = create_tmp_var_raw (ptr_type_node);
10453 new_str = ACONCAT ((".gomp_critical_user_",
10454 IDENTIFIER_POINTER (name), NULL));
10455 DECL_NAME (decl) = get_identifier (new_str);
10456 TREE_PUBLIC (decl) = 1;
10457 TREE_STATIC (decl) = 1;
10458 DECL_COMMON (decl) = 1;
10459 DECL_ARTIFICIAL (decl) = 1;
10460 DECL_IGNORED_P (decl) = 1;
10462 varpool_node::finalize_decl (decl);
10464 critical_name_mutexes->put (name, decl);
10466 else
10467 decl = *n;
10469 /* If '#pragma omp critical' is inside offloaded region or
10470 inside function marked as offloadable, the symbol must be
10471 marked as offloadable too. */
10472 omp_context *octx;
10473 if (cgraph_node::get (current_function_decl)->offloadable)
10474 varpool_node::get_create (decl)->offloadable = 1;
10475 else
10476 for (octx = ctx->outer; octx; octx = octx->outer)
10477 if (is_gimple_omp_offloaded (octx->stmt))
10479 varpool_node::get_create (decl)->offloadable = 1;
10480 break;
10483 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
10484 lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl));
10486 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
10487 unlock = build_call_expr_loc (loc, unlock, 1,
10488 build_fold_addr_expr_loc (loc, decl));
10490 else
10492 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
10493 lock = build_call_expr_loc (loc, lock, 0);
10495 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
10496 unlock = build_call_expr_loc (loc, unlock, 0);
10499 push_gimplify_context ();
10501 block = make_node (BLOCK);
10502 bind = gimple_build_bind (NULL, NULL, block);
10503 gsi_replace (gsi_p, bind, true);
10504 gimple_bind_add_stmt (bind, stmt);
10506 tbody = gimple_bind_body (bind);
10507 gimplify_and_add (lock, &tbody);
10508 gimple_bind_set_body (bind, tbody);
10510 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10511 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10512 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10513 gimple_omp_set_body (stmt, NULL);
10515 tbody = gimple_bind_body (bind);
10516 gimplify_and_add (unlock, &tbody);
10517 gimple_bind_set_body (bind, tbody);
10519 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10521 pop_gimplify_context (bind);
10522 gimple_bind_append_vars (bind, ctx->block_vars);
10523 BLOCK_VARS (block) = gimple_bind_vars (bind);
10527 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10528 for a lastprivate clause. Given a loop control predicate of (V
10529 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10530 is appended to *DLIST, iterator initialization is appended to
10531 *BODY_P. */
10533 static void
10534 lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
10535 gimple_seq *dlist, struct omp_context *ctx)
10537 tree clauses, cond, vinit;
10538 enum tree_code cond_code;
10539 gimple_seq stmts;
10541 cond_code = fd->loop.cond_code;
10542 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
10544 /* When possible, use a strict equality expression. This can let VRP
10545 type optimizations deduce the value and remove a copy. */
10546 if (tree_fits_shwi_p (fd->loop.step))
10548 HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
10549 if (step == 1 || step == -1)
10550 cond_code = EQ_EXPR;
10553 tree n2 = fd->loop.n2;
10554 if (fd->collapse > 1
10555 && TREE_CODE (n2) != INTEGER_CST
10556 && gimple_omp_for_combined_into_p (fd->for_stmt)
10557 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
10559 gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt);
10560 if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR)
10562 struct omp_for_data outer_fd;
10563 extract_omp_for_data (gfor, &outer_fd, NULL);
10564 n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2);
10567 cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2);
10569 clauses = gimple_omp_for_clauses (fd->for_stmt);
10570 stmts = NULL;
10571 lower_lastprivate_clauses (clauses, cond, &stmts, ctx);
10572 if (!gimple_seq_empty_p (stmts))
10574 gimple_seq_add_seq (&stmts, *dlist);
10575 *dlist = stmts;
10577 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10578 vinit = fd->loop.n1;
10579 if (cond_code == EQ_EXPR
10580 && tree_fits_shwi_p (fd->loop.n2)
10581 && ! integer_zerop (fd->loop.n2))
10582 vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
10583 else
10584 vinit = unshare_expr (vinit);
10586 /* Initialize the iterator variable, so that threads that don't execute
10587 any iterations don't execute the lastprivate clauses by accident. */
10588 gimplify_assign (fd->loop.v, vinit, body_p);
10593 /* Lower code for an OMP loop directive. */
10595 static void
10596 lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10598 tree *rhs_p, block;
10599 struct omp_for_data fd, *fdp = NULL;
10600 gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
10601 gbind *new_stmt;
10602 gimple_seq omp_for_body, body, dlist;
10603 size_t i;
10605 push_gimplify_context ();
10607 lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
10609 block = make_node (BLOCK);
10610 new_stmt = gimple_build_bind (NULL, NULL, block);
10611 /* Replace at gsi right away, so that 'stmt' is no member
10612 of a sequence anymore as we're going to add to to a different
10613 one below. */
10614 gsi_replace (gsi_p, new_stmt, true);
10616 /* Move declaration of temporaries in the loop body before we make
10617 it go away. */
10618 omp_for_body = gimple_omp_body (stmt);
10619 if (!gimple_seq_empty_p (omp_for_body)
10620 && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
10622 gbind *inner_bind
10623 = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body));
10624 tree vars = gimple_bind_vars (inner_bind);
10625 gimple_bind_append_vars (new_stmt, vars);
10626 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10627 keep them on the inner_bind and it's block. */
10628 gimple_bind_set_vars (inner_bind, NULL_TREE);
10629 if (gimple_bind_block (inner_bind))
10630 BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
10633 if (gimple_omp_for_combined_into_p (stmt))
10635 extract_omp_for_data (stmt, &fd, NULL);
10636 fdp = &fd;
10638 /* We need two temporaries with fd.loop.v type (istart/iend)
10639 and then (fd.collapse - 1) temporaries with the same
10640 type for count2 ... countN-1 vars if not constant. */
10641 size_t count = 2;
10642 tree type = fd.iter_type;
10643 if (fd.collapse > 1
10644 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
10645 count += fd.collapse - 1;
10646 bool parallel_for = gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR;
10647 tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
10648 tree clauses = *pc;
10649 if (parallel_for)
10650 outerc
10651 = find_omp_clause (gimple_omp_parallel_clauses (ctx->outer->stmt),
10652 OMP_CLAUSE__LOOPTEMP_);
10653 for (i = 0; i < count; i++)
10655 tree temp;
10656 if (parallel_for)
10658 gcc_assert (outerc);
10659 temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
10660 outerc = find_omp_clause (OMP_CLAUSE_CHAIN (outerc),
10661 OMP_CLAUSE__LOOPTEMP_);
10663 else
10665 temp = create_tmp_var (type);
10666 insert_decl_map (&ctx->outer->cb, temp, temp);
10668 *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
10669 OMP_CLAUSE_DECL (*pc) = temp;
10670 pc = &OMP_CLAUSE_CHAIN (*pc);
10672 *pc = clauses;
10675 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10676 dlist = NULL;
10677 body = NULL;
10678 lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
10679 fdp);
10680 gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt));
10682 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10684 /* Lower the header expressions. At this point, we can assume that
10685 the header is of the form:
10687 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10689 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10690 using the .omp_data_s mapping, if needed. */
10691 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
10693 rhs_p = gimple_omp_for_initial_ptr (stmt, i);
10694 if (!is_gimple_min_invariant (*rhs_p))
10695 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
10697 rhs_p = gimple_omp_for_final_ptr (stmt, i);
10698 if (!is_gimple_min_invariant (*rhs_p))
10699 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
10701 rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
10702 if (!is_gimple_min_invariant (*rhs_p))
10703 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
10706 /* Once lowered, extract the bounds and clauses. */
10707 extract_omp_for_data (stmt, &fd, NULL);
10709 lower_omp_for_lastprivate (&fd, &body, &dlist, ctx);
10711 gimple_seq_add_stmt (&body, stmt);
10712 gimple_seq_add_seq (&body, gimple_omp_body (stmt));
10714 gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
10715 fd.loop.v));
10717 /* After the loop, add exit clauses. */
10718 lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx);
10720 if (ctx->cancellable)
10721 gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
10723 gimple_seq_add_seq (&body, dlist);
10725 body = maybe_catch_exception (body);
10727 /* Region exit marker goes at the end of the loop body. */
10728 gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait));
10729 maybe_add_implicit_barrier_cancel (ctx, &body);
10730 pop_gimplify_context (new_stmt);
10732 gimple_bind_append_vars (new_stmt, ctx->block_vars);
10733 BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
10734 if (BLOCK_VARS (block))
10735 TREE_USED (block) = 1;
10737 gimple_bind_set_body (new_stmt, body);
10738 gimple_omp_set_body (stmt, NULL);
10739 gimple_omp_for_set_pre_body (stmt, NULL);
10742 /* Callback for walk_stmts. Check if the current statement only contains
10743 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10745 static tree
10746 check_combined_parallel (gimple_stmt_iterator *gsi_p,
10747 bool *handled_ops_p,
10748 struct walk_stmt_info *wi)
10750 int *info = (int *) wi->info;
10751 gimple stmt = gsi_stmt (*gsi_p);
10753 *handled_ops_p = true;
10754 switch (gimple_code (stmt))
10756 WALK_SUBSTMTS;
10758 case GIMPLE_OMP_FOR:
10759 case GIMPLE_OMP_SECTIONS:
10760 *info = *info == 0 ? 1 : -1;
10761 break;
10762 default:
10763 *info = -1;
10764 break;
10766 return NULL;
10769 struct omp_taskcopy_context
10771 /* This field must be at the beginning, as we do "inheritance": Some
10772 callback functions for tree-inline.c (e.g., omp_copy_decl)
10773 receive a copy_body_data pointer that is up-casted to an
10774 omp_context pointer. */
10775 copy_body_data cb;
10776 omp_context *ctx;
10779 static tree
10780 task_copyfn_copy_decl (tree var, copy_body_data *cb)
10782 struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
10784 if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
10785 return create_tmp_var (TREE_TYPE (var));
10787 return var;
10790 static tree
10791 task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
10793 tree name, new_fields = NULL, type, f;
10795 type = lang_hooks.types.make_type (RECORD_TYPE);
10796 name = DECL_NAME (TYPE_NAME (orig_type));
10797 name = build_decl (gimple_location (tcctx->ctx->stmt),
10798 TYPE_DECL, name, type);
10799 TYPE_NAME (type) = name;
10801 for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
10803 tree new_f = copy_node (f);
10804 DECL_CONTEXT (new_f) = type;
10805 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
10806 TREE_CHAIN (new_f) = new_fields;
10807 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
10808 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
10809 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
10810 &tcctx->cb, NULL);
10811 new_fields = new_f;
10812 tcctx->cb.decl_map->put (f, new_f);
10814 TYPE_FIELDS (type) = nreverse (new_fields);
10815 layout_type (type);
10816 return type;
10819 /* Create task copyfn. */
10821 static void
10822 create_task_copyfn (gomp_task *task_stmt, omp_context *ctx)
10824 struct function *child_cfun;
10825 tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
10826 tree record_type, srecord_type, bind, list;
10827 bool record_needs_remap = false, srecord_needs_remap = false;
10828 splay_tree_node n;
10829 struct omp_taskcopy_context tcctx;
10830 location_t loc = gimple_location (task_stmt);
10832 child_fn = gimple_omp_task_copy_fn (task_stmt);
10833 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
10834 gcc_assert (child_cfun->cfg == NULL);
10835 DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
10837 /* Reset DECL_CONTEXT on function arguments. */
10838 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
10839 DECL_CONTEXT (t) = child_fn;
10841 /* Populate the function. */
10842 push_gimplify_context ();
10843 push_cfun (child_cfun);
10845 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
10846 TREE_SIDE_EFFECTS (bind) = 1;
10847 list = NULL;
10848 DECL_SAVED_TREE (child_fn) = bind;
10849 DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
10851 /* Remap src and dst argument types if needed. */
10852 record_type = ctx->record_type;
10853 srecord_type = ctx->srecord_type;
10854 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
10855 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
10857 record_needs_remap = true;
10858 break;
10860 for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
10861 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
10863 srecord_needs_remap = true;
10864 break;
10867 if (record_needs_remap || srecord_needs_remap)
10869 memset (&tcctx, '\0', sizeof (tcctx));
10870 tcctx.cb.src_fn = ctx->cb.src_fn;
10871 tcctx.cb.dst_fn = child_fn;
10872 tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn);
10873 gcc_checking_assert (tcctx.cb.src_node);
10874 tcctx.cb.dst_node = tcctx.cb.src_node;
10875 tcctx.cb.src_cfun = ctx->cb.src_cfun;
10876 tcctx.cb.copy_decl = task_copyfn_copy_decl;
10877 tcctx.cb.eh_lp_nr = 0;
10878 tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
10879 tcctx.cb.decl_map = new hash_map<tree, tree>;
10880 tcctx.ctx = ctx;
10882 if (record_needs_remap)
10883 record_type = task_copyfn_remap_type (&tcctx, record_type);
10884 if (srecord_needs_remap)
10885 srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
10887 else
10888 tcctx.cb.decl_map = NULL;
10890 arg = DECL_ARGUMENTS (child_fn);
10891 TREE_TYPE (arg) = build_pointer_type (record_type);
10892 sarg = DECL_CHAIN (arg);
10893 TREE_TYPE (sarg) = build_pointer_type (srecord_type);
10895 /* First pass: initialize temporaries used in record_type and srecord_type
10896 sizes and field offsets. */
10897 if (tcctx.cb.decl_map)
10898 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
10899 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
10901 tree *p;
10903 decl = OMP_CLAUSE_DECL (c);
10904 p = tcctx.cb.decl_map->get (decl);
10905 if (p == NULL)
10906 continue;
10907 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
10908 sf = (tree) n->value;
10909 sf = *tcctx.cb.decl_map->get (sf);
10910 src = build_simple_mem_ref_loc (loc, sarg);
10911 src = omp_build_component_ref (src, sf);
10912 t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
10913 append_to_statement_list (t, &list);
10916 /* Second pass: copy shared var pointers and copy construct non-VLA
10917 firstprivate vars. */
10918 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
10919 switch (OMP_CLAUSE_CODE (c))
10921 case OMP_CLAUSE_SHARED:
10922 decl = OMP_CLAUSE_DECL (c);
10923 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
10924 if (n == NULL)
10925 break;
10926 f = (tree) n->value;
10927 if (tcctx.cb.decl_map)
10928 f = *tcctx.cb.decl_map->get (f);
10929 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
10930 sf = (tree) n->value;
10931 if (tcctx.cb.decl_map)
10932 sf = *tcctx.cb.decl_map->get (sf);
10933 src = build_simple_mem_ref_loc (loc, sarg);
10934 src = omp_build_component_ref (src, sf);
10935 dst = build_simple_mem_ref_loc (loc, arg);
10936 dst = omp_build_component_ref (dst, f);
10937 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
10938 append_to_statement_list (t, &list);
10939 break;
10940 case OMP_CLAUSE_FIRSTPRIVATE:
10941 decl = OMP_CLAUSE_DECL (c);
10942 if (is_variable_sized (decl))
10943 break;
10944 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
10945 if (n == NULL)
10946 break;
10947 f = (tree) n->value;
10948 if (tcctx.cb.decl_map)
10949 f = *tcctx.cb.decl_map->get (f);
10950 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
10951 if (n != NULL)
10953 sf = (tree) n->value;
10954 if (tcctx.cb.decl_map)
10955 sf = *tcctx.cb.decl_map->get (sf);
10956 src = build_simple_mem_ref_loc (loc, sarg);
10957 src = omp_build_component_ref (src, sf);
10958 if (use_pointer_for_field (decl, NULL) || is_reference (decl))
10959 src = build_simple_mem_ref_loc (loc, src);
10961 else
10962 src = decl;
10963 dst = build_simple_mem_ref_loc (loc, arg);
10964 dst = omp_build_component_ref (dst, f);
10965 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
10966 append_to_statement_list (t, &list);
10967 break;
10968 case OMP_CLAUSE_PRIVATE:
10969 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
10970 break;
10971 decl = OMP_CLAUSE_DECL (c);
10972 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
10973 f = (tree) n->value;
10974 if (tcctx.cb.decl_map)
10975 f = *tcctx.cb.decl_map->get (f);
10976 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
10977 if (n != NULL)
10979 sf = (tree) n->value;
10980 if (tcctx.cb.decl_map)
10981 sf = *tcctx.cb.decl_map->get (sf);
10982 src = build_simple_mem_ref_loc (loc, sarg);
10983 src = omp_build_component_ref (src, sf);
10984 if (use_pointer_for_field (decl, NULL))
10985 src = build_simple_mem_ref_loc (loc, src);
10987 else
10988 src = decl;
10989 dst = build_simple_mem_ref_loc (loc, arg);
10990 dst = omp_build_component_ref (dst, f);
10991 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
10992 append_to_statement_list (t, &list);
10993 break;
10994 default:
10995 break;
10998 /* Last pass: handle VLA firstprivates. */
10999 if (tcctx.cb.decl_map)
11000 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
11001 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
11003 tree ind, ptr, df;
11005 decl = OMP_CLAUSE_DECL (c);
11006 if (!is_variable_sized (decl))
11007 continue;
11008 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
11009 if (n == NULL)
11010 continue;
11011 f = (tree) n->value;
11012 f = *tcctx.cb.decl_map->get (f);
11013 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
11014 ind = DECL_VALUE_EXPR (decl);
11015 gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
11016 gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
11017 n = splay_tree_lookup (ctx->sfield_map,
11018 (splay_tree_key) TREE_OPERAND (ind, 0));
11019 sf = (tree) n->value;
11020 sf = *tcctx.cb.decl_map->get (sf);
11021 src = build_simple_mem_ref_loc (loc, sarg);
11022 src = omp_build_component_ref (src, sf);
11023 src = build_simple_mem_ref_loc (loc, src);
11024 dst = build_simple_mem_ref_loc (loc, arg);
11025 dst = omp_build_component_ref (dst, f);
11026 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
11027 append_to_statement_list (t, &list);
11028 n = splay_tree_lookup (ctx->field_map,
11029 (splay_tree_key) TREE_OPERAND (ind, 0));
11030 df = (tree) n->value;
11031 df = *tcctx.cb.decl_map->get (df);
11032 ptr = build_simple_mem_ref_loc (loc, arg);
11033 ptr = omp_build_component_ref (ptr, df);
11034 t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
11035 build_fold_addr_expr_loc (loc, dst));
11036 append_to_statement_list (t, &list);
11039 t = build1 (RETURN_EXPR, void_type_node, NULL);
11040 append_to_statement_list (t, &list);
11042 if (tcctx.cb.decl_map)
11043 delete tcctx.cb.decl_map;
11044 pop_gimplify_context (NULL);
11045 BIND_EXPR_BODY (bind) = list;
11046 pop_cfun ();
11049 static void
11050 lower_depend_clauses (gimple stmt, gimple_seq *iseq, gimple_seq *oseq)
11052 tree c, clauses;
11053 gimple g;
11054 size_t n_in = 0, n_out = 0, idx = 2, i;
11056 clauses = find_omp_clause (gimple_omp_task_clauses (stmt),
11057 OMP_CLAUSE_DEPEND);
11058 gcc_assert (clauses);
11059 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
11060 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
11061 switch (OMP_CLAUSE_DEPEND_KIND (c))
11063 case OMP_CLAUSE_DEPEND_IN:
11064 n_in++;
11065 break;
11066 case OMP_CLAUSE_DEPEND_OUT:
11067 case OMP_CLAUSE_DEPEND_INOUT:
11068 n_out++;
11069 break;
11070 default:
11071 gcc_unreachable ();
11073 tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2);
11074 tree array = create_tmp_var (type);
11075 tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
11076 NULL_TREE);
11077 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out));
11078 gimple_seq_add_stmt (iseq, g);
11079 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
11080 NULL_TREE);
11081 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out));
11082 gimple_seq_add_stmt (iseq, g);
11083 for (i = 0; i < 2; i++)
11085 if ((i ? n_in : n_out) == 0)
11086 continue;
11087 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
11088 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
11089 && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i))
11091 tree t = OMP_CLAUSE_DECL (c);
11092 t = fold_convert (ptr_type_node, t);
11093 gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
11094 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
11095 NULL_TREE, NULL_TREE);
11096 g = gimple_build_assign (r, t);
11097 gimple_seq_add_stmt (iseq, g);
11100 tree *p = gimple_omp_task_clauses_ptr (stmt);
11101 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
11102 OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
11103 OMP_CLAUSE_CHAIN (c) = *p;
11104 *p = c;
11105 tree clobber = build_constructor (type, NULL);
11106 TREE_THIS_VOLATILE (clobber) = 1;
11107 g = gimple_build_assign (array, clobber);
11108 gimple_seq_add_stmt (oseq, g);
11111 /* Lower the OpenMP parallel or task directive in the current statement
11112 in GSI_P. CTX holds context information for the directive. */
11114 static void
11115 lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11117 tree clauses;
11118 tree child_fn, t;
11119 gimple stmt = gsi_stmt (*gsi_p);
11120 gbind *par_bind, *bind, *dep_bind = NULL;
11121 gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body;
11122 location_t loc = gimple_location (stmt);
11124 clauses = gimple_omp_taskreg_clauses (stmt);
11125 par_bind
11126 = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
11127 par_body = gimple_bind_body (par_bind);
11128 child_fn = ctx->cb.dst_fn;
11129 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
11130 && !gimple_omp_parallel_combined_p (stmt))
11132 struct walk_stmt_info wi;
11133 int ws_num = 0;
11135 memset (&wi, 0, sizeof (wi));
11136 wi.info = &ws_num;
11137 wi.val_only = true;
11138 walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
11139 if (ws_num == 1)
11140 gimple_omp_parallel_set_combined_p (stmt, true);
11142 gimple_seq dep_ilist = NULL;
11143 gimple_seq dep_olist = NULL;
11144 if (gimple_code (stmt) == GIMPLE_OMP_TASK
11145 && find_omp_clause (clauses, OMP_CLAUSE_DEPEND))
11147 push_gimplify_context ();
11148 dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
11149 lower_depend_clauses (stmt, &dep_ilist, &dep_olist);
11152 if (ctx->srecord_type)
11153 create_task_copyfn (as_a <gomp_task *> (stmt), ctx);
11155 push_gimplify_context ();
11157 par_olist = NULL;
11158 par_ilist = NULL;
11159 par_rlist = NULL;
11160 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
11161 lower_omp (&par_body, ctx);
11162 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL)
11163 lower_reduction_clauses (clauses, &par_rlist, ctx);
11165 /* Declare all the variables created by mapping and the variables
11166 declared in the scope of the parallel body. */
11167 record_vars_into (ctx->block_vars, child_fn);
11168 record_vars_into (gimple_bind_vars (par_bind), child_fn);
11170 if (ctx->record_type)
11172 ctx->sender_decl
11173 = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
11174 : ctx->record_type, ".omp_data_o");
11175 DECL_NAMELESS (ctx->sender_decl) = 1;
11176 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
11177 gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
11180 olist = NULL;
11181 ilist = NULL;
11182 lower_send_clauses (clauses, &ilist, &olist, ctx);
11183 lower_send_shared_vars (&ilist, &olist, ctx);
11185 if (ctx->record_type)
11187 tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL);
11188 TREE_THIS_VOLATILE (clobber) = 1;
11189 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
11190 clobber));
11193 /* Once all the expansions are done, sequence all the different
11194 fragments inside gimple_omp_body. */
11196 new_body = NULL;
11198 if (ctx->record_type)
11200 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
11201 /* fixup_child_record_type might have changed receiver_decl's type. */
11202 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
11203 gimple_seq_add_stmt (&new_body,
11204 gimple_build_assign (ctx->receiver_decl, t));
11207 gimple_seq_add_seq (&new_body, par_ilist);
11208 gimple_seq_add_seq (&new_body, par_body);
11209 gimple_seq_add_seq (&new_body, par_rlist);
11210 if (ctx->cancellable)
11211 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
11212 gimple_seq_add_seq (&new_body, par_olist);
11213 new_body = maybe_catch_exception (new_body);
11214 if (gimple_code (stmt) == GIMPLE_OMP_TASK)
11215 gimple_seq_add_stmt (&new_body,
11216 gimple_build_omp_continue (integer_zero_node,
11217 integer_zero_node));
11218 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
11219 gimple_omp_set_body (stmt, new_body);
11221 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
11222 gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
11223 gimple_bind_add_seq (bind, ilist);
11224 gimple_bind_add_stmt (bind, stmt);
11225 gimple_bind_add_seq (bind, olist);
11227 pop_gimplify_context (NULL);
11229 if (dep_bind)
11231 gimple_bind_add_seq (dep_bind, dep_ilist);
11232 gimple_bind_add_stmt (dep_bind, bind);
11233 gimple_bind_add_seq (dep_bind, dep_olist);
11234 pop_gimplify_context (dep_bind);
11238 /* Lower the GIMPLE_OMP_TARGET in the current statement
11239 in GSI_P. CTX holds context information for the directive. */
11241 static void
11242 lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11244 tree clauses;
11245 tree child_fn, t, c;
11246 gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p));
11247 gbind *tgt_bind, *bind;
11248 gimple_seq tgt_body, olist, ilist, orlist, irlist, new_body;
11249 location_t loc = gimple_location (stmt);
11250 bool offloaded, data_region;
11251 unsigned int map_cnt = 0;
11253 offloaded = is_gimple_omp_offloaded (stmt);
11254 switch (gimple_omp_target_kind (stmt))
11256 case GF_OMP_TARGET_KIND_REGION:
11257 case GF_OMP_TARGET_KIND_UPDATE:
11258 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
11259 case GF_OMP_TARGET_KIND_OACC_KERNELS:
11260 case GF_OMP_TARGET_KIND_OACC_UPDATE:
11261 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
11262 data_region = false;
11263 break;
11264 case GF_OMP_TARGET_KIND_DATA:
11265 case GF_OMP_TARGET_KIND_OACC_DATA:
11266 data_region = true;
11267 break;
11268 default:
11269 gcc_unreachable ();
11272 clauses = gimple_omp_target_clauses (stmt);
11274 tgt_bind = NULL;
11275 tgt_body = NULL;
11276 if (offloaded)
11278 tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt));
11279 tgt_body = gimple_bind_body (tgt_bind);
11281 else if (data_region)
11282 tgt_body = gimple_omp_body (stmt);
11283 child_fn = ctx->cb.dst_fn;
11285 push_gimplify_context ();
11287 irlist = NULL;
11288 orlist = NULL;
11289 if (offloaded
11290 && is_gimple_omp_oacc (stmt))
11291 oacc_process_reduction_data (&tgt_body, &irlist, &orlist, ctx);
11293 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
11294 switch (OMP_CLAUSE_CODE (c))
11296 tree var, x;
11298 default:
11299 break;
11300 case OMP_CLAUSE_MAP:
11301 #ifdef ENABLE_CHECKING
11302 /* First check what we're prepared to handle in the following. */
11303 switch (OMP_CLAUSE_MAP_KIND (c))
11305 case GOMP_MAP_ALLOC:
11306 case GOMP_MAP_TO:
11307 case GOMP_MAP_FROM:
11308 case GOMP_MAP_TOFROM:
11309 case GOMP_MAP_POINTER:
11310 case GOMP_MAP_TO_PSET:
11311 break;
11312 case GOMP_MAP_FORCE_ALLOC:
11313 case GOMP_MAP_FORCE_TO:
11314 case GOMP_MAP_FORCE_FROM:
11315 case GOMP_MAP_FORCE_TOFROM:
11316 case GOMP_MAP_FORCE_PRESENT:
11317 case GOMP_MAP_FORCE_DEALLOC:
11318 case GOMP_MAP_FORCE_DEVICEPTR:
11319 gcc_assert (is_gimple_omp_oacc (stmt));
11320 break;
11321 default:
11322 gcc_unreachable ();
11324 #endif
11325 /* FALLTHRU */
11326 case OMP_CLAUSE_TO:
11327 case OMP_CLAUSE_FROM:
11328 var = OMP_CLAUSE_DECL (c);
11329 if (!DECL_P (var))
11331 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
11332 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
11333 map_cnt++;
11334 continue;
11337 if (DECL_SIZE (var)
11338 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
11340 tree var2 = DECL_VALUE_EXPR (var);
11341 gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
11342 var2 = TREE_OPERAND (var2, 0);
11343 gcc_assert (DECL_P (var2));
11344 var = var2;
11347 if (!maybe_lookup_field (var, ctx))
11348 continue;
11350 if (offloaded)
11352 x = build_receiver_ref (var, true, ctx);
11353 tree new_var = lookup_decl (var, ctx);
11354 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
11355 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
11356 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
11357 x = build_simple_mem_ref (x);
11358 SET_DECL_VALUE_EXPR (new_var, x);
11359 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
11361 map_cnt++;
11364 if (offloaded)
11366 target_nesting_level++;
11367 lower_omp (&tgt_body, ctx);
11368 target_nesting_level--;
11370 else if (data_region)
11371 lower_omp (&tgt_body, ctx);
11373 if (offloaded)
11375 /* Declare all the variables created by mapping and the variables
11376 declared in the scope of the target body. */
11377 record_vars_into (ctx->block_vars, child_fn);
11378 record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
11381 olist = NULL;
11382 ilist = NULL;
11383 if (ctx->record_type)
11385 ctx->sender_decl
11386 = create_tmp_var (ctx->record_type, ".omp_data_arr");
11387 DECL_NAMELESS (ctx->sender_decl) = 1;
11388 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
11389 t = make_tree_vec (3);
11390 TREE_VEC_ELT (t, 0) = ctx->sender_decl;
11391 TREE_VEC_ELT (t, 1)
11392 = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt),
11393 ".omp_data_sizes");
11394 DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
11395 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
11396 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
11397 tree tkind_type;
11398 int talign_shift;
11399 if (is_gimple_omp_oacc (stmt))
11401 tkind_type = short_unsigned_type_node;
11402 talign_shift = 8;
11404 else
11406 tkind_type = unsigned_char_type_node;
11407 talign_shift = 3;
11409 TREE_VEC_ELT (t, 2)
11410 = create_tmp_var (build_array_type_nelts (tkind_type, map_cnt),
11411 ".omp_data_kinds");
11412 DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
11413 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
11414 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
11415 gimple_omp_target_set_data_arg (stmt, t);
11417 vec<constructor_elt, va_gc> *vsize;
11418 vec<constructor_elt, va_gc> *vkind;
11419 vec_alloc (vsize, map_cnt);
11420 vec_alloc (vkind, map_cnt);
11421 unsigned int map_idx = 0;
11423 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
11424 switch (OMP_CLAUSE_CODE (c))
11426 tree ovar, nc;
11428 default:
11429 break;
11430 case OMP_CLAUSE_MAP:
11431 case OMP_CLAUSE_TO:
11432 case OMP_CLAUSE_FROM:
11433 nc = c;
11434 ovar = OMP_CLAUSE_DECL (c);
11435 if (!DECL_P (ovar))
11437 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
11438 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
11440 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c))
11441 == get_base_address (ovar));
11442 nc = OMP_CLAUSE_CHAIN (c);
11443 ovar = OMP_CLAUSE_DECL (nc);
11445 else
11447 tree x = build_sender_ref (ovar, ctx);
11448 tree v
11449 = build_fold_addr_expr_with_type (ovar, ptr_type_node);
11450 gimplify_assign (x, v, &ilist);
11451 nc = NULL_TREE;
11454 else
11456 if (DECL_SIZE (ovar)
11457 && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST)
11459 tree ovar2 = DECL_VALUE_EXPR (ovar);
11460 gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
11461 ovar2 = TREE_OPERAND (ovar2, 0);
11462 gcc_assert (DECL_P (ovar2));
11463 ovar = ovar2;
11465 if (!maybe_lookup_field (ovar, ctx))
11466 continue;
11469 unsigned int talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
11470 if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
11471 talign = DECL_ALIGN_UNIT (ovar);
11472 if (nc)
11474 tree var = lookup_decl_in_outer_ctx (ovar, ctx);
11475 tree x = build_sender_ref (ovar, ctx);
11476 if (maybe_lookup_oacc_reduction (var, ctx))
11478 gcc_checking_assert (offloaded
11479 && is_gimple_omp_oacc (stmt));
11480 gimplify_assign (x, var, &ilist);
11482 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
11483 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
11484 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
11485 && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
11487 gcc_assert (offloaded);
11488 tree avar
11489 = create_tmp_var (TREE_TYPE (TREE_TYPE (x)));
11490 mark_addressable (avar);
11491 gimplify_assign (avar, build_fold_addr_expr (var), &ilist);
11492 talign = DECL_ALIGN_UNIT (avar);
11493 avar = build_fold_addr_expr (avar);
11494 gimplify_assign (x, avar, &ilist);
11496 else if (is_gimple_reg (var))
11498 gcc_assert (offloaded);
11499 tree avar = create_tmp_var (TREE_TYPE (var));
11500 mark_addressable (avar);
11501 enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c);
11502 if (GOMP_MAP_COPY_TO_P (map_kind)
11503 || map_kind == GOMP_MAP_POINTER
11504 || map_kind == GOMP_MAP_TO_PSET
11505 || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
11506 gimplify_assign (avar, var, &ilist);
11507 avar = build_fold_addr_expr (avar);
11508 gimplify_assign (x, avar, &ilist);
11509 if ((GOMP_MAP_COPY_FROM_P (map_kind)
11510 || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
11511 && !TYPE_READONLY (TREE_TYPE (var)))
11513 x = build_sender_ref (ovar, ctx);
11514 x = build_simple_mem_ref (x);
11515 gimplify_assign (var, x, &olist);
11518 else
11520 var = build_fold_addr_expr (var);
11521 gimplify_assign (x, var, &ilist);
11524 tree s = OMP_CLAUSE_SIZE (c);
11525 if (s == NULL_TREE)
11526 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
11527 s = fold_convert (size_type_node, s);
11528 tree purpose = size_int (map_idx++);
11529 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
11530 if (TREE_CODE (s) != INTEGER_CST)
11531 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
11533 unsigned HOST_WIDE_INT tkind;
11534 switch (OMP_CLAUSE_CODE (c))
11536 case OMP_CLAUSE_MAP:
11537 tkind = OMP_CLAUSE_MAP_KIND (c);
11538 break;
11539 case OMP_CLAUSE_TO:
11540 tkind = GOMP_MAP_TO;
11541 break;
11542 case OMP_CLAUSE_FROM:
11543 tkind = GOMP_MAP_FROM;
11544 break;
11545 default:
11546 gcc_unreachable ();
11548 gcc_checking_assert (tkind
11549 < (HOST_WIDE_INT_C (1U) << talign_shift));
11550 talign = ceil_log2 (talign);
11551 tkind |= talign << talign_shift;
11552 gcc_checking_assert (tkind
11553 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
11554 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
11555 build_int_cstu (tkind_type, tkind));
11556 if (nc && nc != c)
11557 c = nc;
11560 gcc_assert (map_idx == map_cnt);
11562 DECL_INITIAL (TREE_VEC_ELT (t, 1))
11563 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
11564 DECL_INITIAL (TREE_VEC_ELT (t, 2))
11565 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
11566 if (!TREE_STATIC (TREE_VEC_ELT (t, 1)))
11568 gimple_seq initlist = NULL;
11569 force_gimple_operand (build1 (DECL_EXPR, void_type_node,
11570 TREE_VEC_ELT (t, 1)),
11571 &initlist, true, NULL_TREE);
11572 gimple_seq_add_seq (&ilist, initlist);
11574 tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)),
11575 NULL);
11576 TREE_THIS_VOLATILE (clobber) = 1;
11577 gimple_seq_add_stmt (&olist,
11578 gimple_build_assign (TREE_VEC_ELT (t, 1),
11579 clobber));
11582 tree clobber = build_constructor (ctx->record_type, NULL);
11583 TREE_THIS_VOLATILE (clobber) = 1;
11584 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
11585 clobber));
11588 /* Once all the expansions are done, sequence all the different
11589 fragments inside gimple_omp_body. */
11591 new_body = NULL;
11593 if (offloaded
11594 && ctx->record_type)
11596 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
11597 /* fixup_child_record_type might have changed receiver_decl's type. */
11598 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
11599 gimple_seq_add_stmt (&new_body,
11600 gimple_build_assign (ctx->receiver_decl, t));
11603 if (offloaded)
11605 gimple_seq_add_seq (&new_body, tgt_body);
11606 new_body = maybe_catch_exception (new_body);
11608 else if (data_region)
11609 new_body = tgt_body;
11610 if (offloaded || data_region)
11612 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
11613 gimple_omp_set_body (stmt, new_body);
11616 bind = gimple_build_bind (NULL, NULL,
11617 tgt_bind ? gimple_bind_block (tgt_bind)
11618 : NULL_TREE);
11619 gsi_replace (gsi_p, bind, true);
11620 gimple_bind_add_seq (bind, irlist);
11621 gimple_bind_add_seq (bind, ilist);
11622 gimple_bind_add_stmt (bind, stmt);
11623 gimple_bind_add_seq (bind, olist);
11624 gimple_bind_add_seq (bind, orlist);
11626 pop_gimplify_context (NULL);
11629 /* Expand code for an OpenMP teams directive. */
11631 static void
11632 lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11634 gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p));
11635 push_gimplify_context ();
11637 tree block = make_node (BLOCK);
11638 gbind *bind = gimple_build_bind (NULL, NULL, block);
11639 gsi_replace (gsi_p, bind, true);
11640 gimple_seq bind_body = NULL;
11641 gimple_seq dlist = NULL;
11642 gimple_seq olist = NULL;
11644 tree num_teams = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
11645 OMP_CLAUSE_NUM_TEAMS);
11646 if (num_teams == NULL_TREE)
11647 num_teams = build_int_cst (unsigned_type_node, 0);
11648 else
11650 num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams);
11651 num_teams = fold_convert (unsigned_type_node, num_teams);
11652 gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
11654 tree thread_limit = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
11655 OMP_CLAUSE_THREAD_LIMIT);
11656 if (thread_limit == NULL_TREE)
11657 thread_limit = build_int_cst (unsigned_type_node, 0);
11658 else
11660 thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
11661 thread_limit = fold_convert (unsigned_type_node, thread_limit);
11662 gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
11663 fb_rvalue);
11666 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
11667 &bind_body, &dlist, ctx, NULL);
11668 lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
11669 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx);
11670 gimple_seq_add_stmt (&bind_body, teams_stmt);
11672 location_t loc = gimple_location (teams_stmt);
11673 tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS);
11674 gimple call = gimple_build_call (decl, 2, num_teams, thread_limit);
11675 gimple_set_location (call, loc);
11676 gimple_seq_add_stmt (&bind_body, call);
11678 gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
11679 gimple_omp_set_body (teams_stmt, NULL);
11680 gimple_seq_add_seq (&bind_body, olist);
11681 gimple_seq_add_seq (&bind_body, dlist);
11682 gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
11683 gimple_bind_set_body (bind, bind_body);
11685 pop_gimplify_context (bind);
11687 gimple_bind_append_vars (bind, ctx->block_vars);
11688 BLOCK_VARS (block) = ctx->block_vars;
11689 if (BLOCK_VARS (block))
11690 TREE_USED (block) = 1;
11694 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11695 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11696 of OMP context, but with task_shared_vars set. */
11698 static tree
11699 lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
11700 void *data)
11702 tree t = *tp;
11704 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11705 if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t))
11706 return t;
11708 if (task_shared_vars
11709 && DECL_P (t)
11710 && bitmap_bit_p (task_shared_vars, DECL_UID (t)))
11711 return t;
11713 /* If a global variable has been privatized, TREE_CONSTANT on
11714 ADDR_EXPR might be wrong. */
11715 if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
11716 recompute_tree_invariant_for_addr_expr (t);
11718 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
11719 return NULL_TREE;
11722 static void
11723 lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11725 gimple stmt = gsi_stmt (*gsi_p);
11726 struct walk_stmt_info wi;
11727 gcall *call_stmt;
11729 if (gimple_has_location (stmt))
11730 input_location = gimple_location (stmt);
11732 if (task_shared_vars)
11733 memset (&wi, '\0', sizeof (wi));
11735 /* If we have issued syntax errors, avoid doing any heavy lifting.
11736 Just replace the OMP directives with a NOP to avoid
11737 confusing RTL expansion. */
11738 if (seen_error () && is_gimple_omp (stmt))
11740 gsi_replace (gsi_p, gimple_build_nop (), true);
11741 return;
11744 switch (gimple_code (stmt))
11746 case GIMPLE_COND:
11748 gcond *cond_stmt = as_a <gcond *> (stmt);
11749 if ((ctx || task_shared_vars)
11750 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
11751 lower_omp_regimplify_p,
11752 ctx ? NULL : &wi, NULL)
11753 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
11754 lower_omp_regimplify_p,
11755 ctx ? NULL : &wi, NULL)))
11756 gimple_regimplify_operands (cond_stmt, gsi_p);
11758 break;
11759 case GIMPLE_CATCH:
11760 lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx);
11761 break;
11762 case GIMPLE_EH_FILTER:
11763 lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
11764 break;
11765 case GIMPLE_TRY:
11766 lower_omp (gimple_try_eval_ptr (stmt), ctx);
11767 lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
11768 break;
11769 case GIMPLE_TRANSACTION:
11770 lower_omp (gimple_transaction_body_ptr (
11771 as_a <gtransaction *> (stmt)),
11772 ctx);
11773 break;
11774 case GIMPLE_BIND:
11775 lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx);
11776 break;
11777 case GIMPLE_OMP_PARALLEL:
11778 case GIMPLE_OMP_TASK:
11779 ctx = maybe_lookup_ctx (stmt);
11780 gcc_assert (ctx);
11781 if (ctx->cancellable)
11782 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
11783 lower_omp_taskreg (gsi_p, ctx);
11784 break;
11785 case GIMPLE_OMP_FOR:
11786 ctx = maybe_lookup_ctx (stmt);
11787 gcc_assert (ctx);
11788 if (ctx->cancellable)
11789 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
11790 lower_omp_for (gsi_p, ctx);
11791 break;
11792 case GIMPLE_OMP_SECTIONS:
11793 ctx = maybe_lookup_ctx (stmt);
11794 gcc_assert (ctx);
11795 if (ctx->cancellable)
11796 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
11797 lower_omp_sections (gsi_p, ctx);
11798 break;
11799 case GIMPLE_OMP_SINGLE:
11800 ctx = maybe_lookup_ctx (stmt);
11801 gcc_assert (ctx);
11802 lower_omp_single (gsi_p, ctx);
11803 break;
11804 case GIMPLE_OMP_MASTER:
11805 ctx = maybe_lookup_ctx (stmt);
11806 gcc_assert (ctx);
11807 lower_omp_master (gsi_p, ctx);
11808 break;
11809 case GIMPLE_OMP_TASKGROUP:
11810 ctx = maybe_lookup_ctx (stmt);
11811 gcc_assert (ctx);
11812 lower_omp_taskgroup (gsi_p, ctx);
11813 break;
11814 case GIMPLE_OMP_ORDERED:
11815 ctx = maybe_lookup_ctx (stmt);
11816 gcc_assert (ctx);
11817 lower_omp_ordered (gsi_p, ctx);
11818 break;
11819 case GIMPLE_OMP_CRITICAL:
11820 ctx = maybe_lookup_ctx (stmt);
11821 gcc_assert (ctx);
11822 lower_omp_critical (gsi_p, ctx);
11823 break;
11824 case GIMPLE_OMP_ATOMIC_LOAD:
11825 if ((ctx || task_shared_vars)
11826 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
11827 as_a <gomp_atomic_load *> (stmt)),
11828 lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
11829 gimple_regimplify_operands (stmt, gsi_p);
11830 break;
11831 case GIMPLE_OMP_TARGET:
11832 ctx = maybe_lookup_ctx (stmt);
11833 gcc_assert (ctx);
11834 lower_omp_target (gsi_p, ctx);
11835 break;
11836 case GIMPLE_OMP_TEAMS:
11837 ctx = maybe_lookup_ctx (stmt);
11838 gcc_assert (ctx);
11839 lower_omp_teams (gsi_p, ctx);
11840 break;
11841 case GIMPLE_CALL:
11842 tree fndecl;
11843 call_stmt = as_a <gcall *> (stmt);
11844 fndecl = gimple_call_fndecl (call_stmt);
11845 if (fndecl
11846 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
11847 switch (DECL_FUNCTION_CODE (fndecl))
11849 case BUILT_IN_GOMP_BARRIER:
11850 if (ctx == NULL)
11851 break;
11852 /* FALLTHRU */
11853 case BUILT_IN_GOMP_CANCEL:
11854 case BUILT_IN_GOMP_CANCELLATION_POINT:
11855 omp_context *cctx;
11856 cctx = ctx;
11857 if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
11858 cctx = cctx->outer;
11859 gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE);
11860 if (!cctx->cancellable)
11862 if (DECL_FUNCTION_CODE (fndecl)
11863 == BUILT_IN_GOMP_CANCELLATION_POINT)
11865 stmt = gimple_build_nop ();
11866 gsi_replace (gsi_p, stmt, false);
11868 break;
11870 if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
11872 fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
11873 gimple_call_set_fndecl (call_stmt, fndecl);
11874 gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl));
11876 tree lhs;
11877 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)));
11878 gimple_call_set_lhs (call_stmt, lhs);
11879 tree fallthru_label;
11880 fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
11881 gimple g;
11882 g = gimple_build_label (fallthru_label);
11883 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
11884 g = gimple_build_cond (NE_EXPR, lhs,
11885 fold_convert (TREE_TYPE (lhs),
11886 boolean_false_node),
11887 cctx->cancel_label, fallthru_label);
11888 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
11889 break;
11890 default:
11891 break;
11893 /* FALLTHRU */
11894 default:
11895 if ((ctx || task_shared_vars)
11896 && walk_gimple_op (stmt, lower_omp_regimplify_p,
11897 ctx ? NULL : &wi))
11899 /* Just remove clobbers, this should happen only if we have
11900 "privatized" local addressable variables in SIMD regions,
11901 the clobber isn't needed in that case and gimplifying address
11902 of the ARRAY_REF into a pointer and creating MEM_REF based
11903 clobber would create worse code than we get with the clobber
11904 dropped. */
11905 if (gimple_clobber_p (stmt))
11907 gsi_replace (gsi_p, gimple_build_nop (), true);
11908 break;
11910 gimple_regimplify_operands (stmt, gsi_p);
11912 break;
11916 static void
11917 lower_omp (gimple_seq *body, omp_context *ctx)
11919 location_t saved_location = input_location;
11920 gimple_stmt_iterator gsi;
11921 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
11922 lower_omp_1 (&gsi, ctx);
11923 /* During gimplification, we haven't folded statments inside offloading
11924 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
11925 if (target_nesting_level || taskreg_nesting_level)
11926 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
11927 fold_stmt (&gsi);
11928 input_location = saved_location;
11931 /* Main entry point. */
11933 static unsigned int
11934 execute_lower_omp (void)
11936 gimple_seq body;
11937 int i;
11938 omp_context *ctx;
11940 /* This pass always runs, to provide PROP_gimple_lomp.
11941 But often, there is nothing to do. */
11942 if (flag_cilkplus == 0 && flag_openacc == 0 && flag_openmp == 0
11943 && flag_openmp_simd == 0)
11944 return 0;
11946 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
11947 delete_omp_context);
11949 body = gimple_body (current_function_decl);
11950 scan_omp (&body, NULL);
11951 gcc_assert (taskreg_nesting_level == 0);
11952 FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
11953 finish_taskreg_scan (ctx);
11954 taskreg_contexts.release ();
11956 if (all_contexts->root)
11958 if (task_shared_vars)
11959 push_gimplify_context ();
11960 lower_omp (&body, NULL);
11961 if (task_shared_vars)
11962 pop_gimplify_context (NULL);
11965 if (all_contexts)
11967 splay_tree_delete (all_contexts);
11968 all_contexts = NULL;
11970 BITMAP_FREE (task_shared_vars);
11971 return 0;
11974 namespace {
11976 const pass_data pass_data_lower_omp =
11978 GIMPLE_PASS, /* type */
11979 "omplower", /* name */
11980 OPTGROUP_NONE, /* optinfo_flags */
11981 TV_NONE, /* tv_id */
11982 PROP_gimple_any, /* properties_required */
11983 PROP_gimple_lomp, /* properties_provided */
11984 0, /* properties_destroyed */
11985 0, /* todo_flags_start */
11986 0, /* todo_flags_finish */
11989 class pass_lower_omp : public gimple_opt_pass
11991 public:
11992 pass_lower_omp (gcc::context *ctxt)
11993 : gimple_opt_pass (pass_data_lower_omp, ctxt)
11996 /* opt_pass methods: */
11997 virtual unsigned int execute (function *) { return execute_lower_omp (); }
11999 }; // class pass_lower_omp
12001 } // anon namespace
12003 gimple_opt_pass *
12004 make_pass_lower_omp (gcc::context *ctxt)
12006 return new pass_lower_omp (ctxt);
12009 /* The following is a utility to diagnose structured block violations.
12010 It is not part of the "omplower" pass, as that's invoked too late. It
12011 should be invoked by the respective front ends after gimplification. */
12013 static splay_tree all_labels;
12015 /* Check for mismatched contexts and generate an error if needed. Return
12016 true if an error is detected. */
12018 static bool
12019 diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
12020 gimple branch_ctx, gimple label_ctx)
12022 gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx));
12023 gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx));
12025 if (label_ctx == branch_ctx)
12026 return false;
12028 const char* kind = NULL;
12030 if (flag_cilkplus)
12032 if ((branch_ctx
12033 && gimple_code (branch_ctx) == GIMPLE_OMP_FOR
12034 && gimple_omp_for_kind (branch_ctx) == GF_OMP_FOR_KIND_CILKSIMD)
12035 || (label_ctx
12036 && gimple_code (label_ctx) == GIMPLE_OMP_FOR
12037 && gimple_omp_for_kind (label_ctx) == GF_OMP_FOR_KIND_CILKSIMD))
12038 kind = "Cilk Plus";
12040 if (flag_openacc)
12042 if ((branch_ctx && is_gimple_omp_oacc (branch_ctx))
12043 || (label_ctx && is_gimple_omp_oacc (label_ctx)))
12045 gcc_checking_assert (kind == NULL);
12046 kind = "OpenACC";
12049 if (kind == NULL)
12051 gcc_checking_assert (flag_openmp);
12052 kind = "OpenMP";
12056 Previously we kept track of the label's entire context in diagnose_sb_[12]
12057 so we could traverse it and issue a correct "exit" or "enter" error
12058 message upon a structured block violation.
12060 We built the context by building a list with tree_cons'ing, but there is
12061 no easy counterpart in gimple tuples. It seems like far too much work
12062 for issuing exit/enter error messages. If someone really misses the
12063 distinct error message... patches welcome.
12066 #if 0
12067 /* Try to avoid confusing the user by producing and error message
12068 with correct "exit" or "enter" verbiage. We prefer "exit"
12069 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12070 if (branch_ctx == NULL)
12071 exit_p = false;
12072 else
12074 while (label_ctx)
12076 if (TREE_VALUE (label_ctx) == branch_ctx)
12078 exit_p = false;
12079 break;
12081 label_ctx = TREE_CHAIN (label_ctx);
12085 if (exit_p)
12086 error ("invalid exit from %s structured block", kind);
12087 else
12088 error ("invalid entry to %s structured block", kind);
12089 #endif
12091 /* If it's obvious we have an invalid entry, be specific about the error. */
12092 if (branch_ctx == NULL)
12093 error ("invalid entry to %s structured block", kind);
12094 else
12096 /* Otherwise, be vague and lazy, but efficient. */
12097 error ("invalid branch to/from %s structured block", kind);
12100 gsi_replace (gsi_p, gimple_build_nop (), false);
12101 return true;
12104 /* Pass 1: Create a minimal tree of structured blocks, and record
12105 where each label is found. */
12107 static tree
12108 diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
12109 struct walk_stmt_info *wi)
12111 gimple context = (gimple) wi->info;
12112 gimple inner_context;
12113 gimple stmt = gsi_stmt (*gsi_p);
12115 *handled_ops_p = true;
12117 switch (gimple_code (stmt))
12119 WALK_SUBSTMTS;
12121 case GIMPLE_OMP_PARALLEL:
12122 case GIMPLE_OMP_TASK:
12123 case GIMPLE_OMP_SECTIONS:
12124 case GIMPLE_OMP_SINGLE:
12125 case GIMPLE_OMP_SECTION:
12126 case GIMPLE_OMP_MASTER:
12127 case GIMPLE_OMP_ORDERED:
12128 case GIMPLE_OMP_CRITICAL:
12129 case GIMPLE_OMP_TARGET:
12130 case GIMPLE_OMP_TEAMS:
12131 case GIMPLE_OMP_TASKGROUP:
12132 /* The minimal context here is just the current OMP construct. */
12133 inner_context = stmt;
12134 wi->info = inner_context;
12135 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
12136 wi->info = context;
12137 break;
12139 case GIMPLE_OMP_FOR:
12140 inner_context = stmt;
12141 wi->info = inner_context;
12142 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12143 walk them. */
12144 walk_gimple_seq (gimple_omp_for_pre_body (stmt),
12145 diagnose_sb_1, NULL, wi);
12146 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
12147 wi->info = context;
12148 break;
12150 case GIMPLE_LABEL:
12151 splay_tree_insert (all_labels,
12152 (splay_tree_key) gimple_label_label (
12153 as_a <glabel *> (stmt)),
12154 (splay_tree_value) context);
12155 break;
12157 default:
12158 break;
12161 return NULL_TREE;
12164 /* Pass 2: Check each branch and see if its context differs from that of
12165 the destination label's context. */
12167 static tree
12168 diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
12169 struct walk_stmt_info *wi)
12171 gimple context = (gimple) wi->info;
12172 splay_tree_node n;
12173 gimple stmt = gsi_stmt (*gsi_p);
12175 *handled_ops_p = true;
12177 switch (gimple_code (stmt))
12179 WALK_SUBSTMTS;
12181 case GIMPLE_OMP_PARALLEL:
12182 case GIMPLE_OMP_TASK:
12183 case GIMPLE_OMP_SECTIONS:
12184 case GIMPLE_OMP_SINGLE:
12185 case GIMPLE_OMP_SECTION:
12186 case GIMPLE_OMP_MASTER:
12187 case GIMPLE_OMP_ORDERED:
12188 case GIMPLE_OMP_CRITICAL:
12189 case GIMPLE_OMP_TARGET:
12190 case GIMPLE_OMP_TEAMS:
12191 case GIMPLE_OMP_TASKGROUP:
12192 wi->info = stmt;
12193 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
12194 wi->info = context;
12195 break;
12197 case GIMPLE_OMP_FOR:
12198 wi->info = stmt;
12199 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12200 walk them. */
12201 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
12202 diagnose_sb_2, NULL, wi);
12203 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
12204 wi->info = context;
12205 break;
12207 case GIMPLE_COND:
12209 gcond *cond_stmt = as_a <gcond *> (stmt);
12210 tree lab = gimple_cond_true_label (cond_stmt);
12211 if (lab)
12213 n = splay_tree_lookup (all_labels,
12214 (splay_tree_key) lab);
12215 diagnose_sb_0 (gsi_p, context,
12216 n ? (gimple) n->value : NULL);
12218 lab = gimple_cond_false_label (cond_stmt);
12219 if (lab)
12221 n = splay_tree_lookup (all_labels,
12222 (splay_tree_key) lab);
12223 diagnose_sb_0 (gsi_p, context,
12224 n ? (gimple) n->value : NULL);
12227 break;
12229 case GIMPLE_GOTO:
12231 tree lab = gimple_goto_dest (stmt);
12232 if (TREE_CODE (lab) != LABEL_DECL)
12233 break;
12235 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
12236 diagnose_sb_0 (gsi_p, context, n ? (gimple) n->value : NULL);
12238 break;
12240 case GIMPLE_SWITCH:
12242 gswitch *switch_stmt = as_a <gswitch *> (stmt);
12243 unsigned int i;
12244 for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i)
12246 tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i));
12247 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
12248 if (n && diagnose_sb_0 (gsi_p, context, (gimple) n->value))
12249 break;
12252 break;
12254 case GIMPLE_RETURN:
12255 diagnose_sb_0 (gsi_p, context, NULL);
12256 break;
12258 default:
12259 break;
12262 return NULL_TREE;
12265 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12266 GIMPLE_* codes. */
12267 bool
12268 make_gimple_omp_edges (basic_block bb, struct omp_region **region,
12269 int *region_idx)
12271 gimple last = last_stmt (bb);
12272 enum gimple_code code = gimple_code (last);
12273 struct omp_region *cur_region = *region;
12274 bool fallthru = false;
12276 switch (code)
12278 case GIMPLE_OMP_PARALLEL:
12279 case GIMPLE_OMP_TASK:
12280 case GIMPLE_OMP_FOR:
12281 case GIMPLE_OMP_SINGLE:
12282 case GIMPLE_OMP_TEAMS:
12283 case GIMPLE_OMP_MASTER:
12284 case GIMPLE_OMP_TASKGROUP:
12285 case GIMPLE_OMP_ORDERED:
12286 case GIMPLE_OMP_CRITICAL:
12287 case GIMPLE_OMP_SECTION:
12288 cur_region = new_omp_region (bb, code, cur_region);
12289 fallthru = true;
12290 break;
12292 case GIMPLE_OMP_TARGET:
12293 cur_region = new_omp_region (bb, code, cur_region);
12294 fallthru = true;
12295 switch (gimple_omp_target_kind (last))
12297 case GF_OMP_TARGET_KIND_REGION:
12298 case GF_OMP_TARGET_KIND_DATA:
12299 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
12300 case GF_OMP_TARGET_KIND_OACC_KERNELS:
12301 case GF_OMP_TARGET_KIND_OACC_DATA:
12302 break;
12303 case GF_OMP_TARGET_KIND_UPDATE:
12304 case GF_OMP_TARGET_KIND_OACC_UPDATE:
12305 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
12306 cur_region = cur_region->outer;
12307 break;
12308 default:
12309 gcc_unreachable ();
12311 break;
12313 case GIMPLE_OMP_SECTIONS:
12314 cur_region = new_omp_region (bb, code, cur_region);
12315 fallthru = true;
12316 break;
12318 case GIMPLE_OMP_SECTIONS_SWITCH:
12319 fallthru = false;
12320 break;
12322 case GIMPLE_OMP_ATOMIC_LOAD:
12323 case GIMPLE_OMP_ATOMIC_STORE:
12324 fallthru = true;
12325 break;
12327 case GIMPLE_OMP_RETURN:
12328 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12329 somewhere other than the next block. This will be
12330 created later. */
12331 cur_region->exit = bb;
12332 if (cur_region->type == GIMPLE_OMP_TASK)
12333 /* Add an edge corresponding to not scheduling the task
12334 immediately. */
12335 make_edge (cur_region->entry, bb, EDGE_ABNORMAL);
12336 fallthru = cur_region->type != GIMPLE_OMP_SECTION;
12337 cur_region = cur_region->outer;
12338 break;
12340 case GIMPLE_OMP_CONTINUE:
12341 cur_region->cont = bb;
12342 switch (cur_region->type)
12344 case GIMPLE_OMP_FOR:
12345 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12346 succs edges as abnormal to prevent splitting
12347 them. */
12348 single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL;
12349 /* Make the loopback edge. */
12350 make_edge (bb, single_succ (cur_region->entry),
12351 EDGE_ABNORMAL);
12353 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12354 corresponds to the case that the body of the loop
12355 is not executed at all. */
12356 make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL);
12357 make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL);
12358 fallthru = false;
12359 break;
12361 case GIMPLE_OMP_SECTIONS:
12362 /* Wire up the edges into and out of the nested sections. */
12364 basic_block switch_bb = single_succ (cur_region->entry);
12366 struct omp_region *i;
12367 for (i = cur_region->inner; i ; i = i->next)
12369 gcc_assert (i->type == GIMPLE_OMP_SECTION);
12370 make_edge (switch_bb, i->entry, 0);
12371 make_edge (i->exit, bb, EDGE_FALLTHRU);
12374 /* Make the loopback edge to the block with
12375 GIMPLE_OMP_SECTIONS_SWITCH. */
12376 make_edge (bb, switch_bb, 0);
12378 /* Make the edge from the switch to exit. */
12379 make_edge (switch_bb, bb->next_bb, 0);
12380 fallthru = false;
12382 break;
12384 case GIMPLE_OMP_TASK:
12385 fallthru = true;
12386 break;
12388 default:
12389 gcc_unreachable ();
12391 break;
12393 default:
12394 gcc_unreachable ();
12397 if (*region != cur_region)
12399 *region = cur_region;
12400 if (cur_region)
12401 *region_idx = cur_region->entry->index;
12402 else
12403 *region_idx = 0;
12406 return fallthru;
12409 static unsigned int
12410 diagnose_omp_structured_block_errors (void)
12412 struct walk_stmt_info wi;
12413 gimple_seq body = gimple_body (current_function_decl);
12415 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
12417 memset (&wi, 0, sizeof (wi));
12418 walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
12420 memset (&wi, 0, sizeof (wi));
12421 wi.want_locations = true;
12422 walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
12424 gimple_set_body (current_function_decl, body);
12426 splay_tree_delete (all_labels);
12427 all_labels = NULL;
12429 return 0;
12432 namespace {
12434 const pass_data pass_data_diagnose_omp_blocks =
12436 GIMPLE_PASS, /* type */
12437 "*diagnose_omp_blocks", /* name */
12438 OPTGROUP_NONE, /* optinfo_flags */
12439 TV_NONE, /* tv_id */
12440 PROP_gimple_any, /* properties_required */
12441 0, /* properties_provided */
12442 0, /* properties_destroyed */
12443 0, /* todo_flags_start */
12444 0, /* todo_flags_finish */
12447 class pass_diagnose_omp_blocks : public gimple_opt_pass
12449 public:
12450 pass_diagnose_omp_blocks (gcc::context *ctxt)
12451 : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
12454 /* opt_pass methods: */
12455 virtual bool gate (function *)
12457 return flag_cilkplus || flag_openacc || flag_openmp;
12459 virtual unsigned int execute (function *)
12461 return diagnose_omp_structured_block_errors ();
12464 }; // class pass_diagnose_omp_blocks
12466 } // anon namespace
12468 gimple_opt_pass *
12469 make_pass_diagnose_omp_blocks (gcc::context *ctxt)
12471 return new pass_diagnose_omp_blocks (ctxt);
12474 /* SIMD clone supporting code. */
12476 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12477 of arguments to reserve space for. */
12479 static struct cgraph_simd_clone *
12480 simd_clone_struct_alloc (int nargs)
12482 struct cgraph_simd_clone *clone_info;
12483 size_t len = (sizeof (struct cgraph_simd_clone)
12484 + nargs * sizeof (struct cgraph_simd_clone_arg));
12485 clone_info = (struct cgraph_simd_clone *)
12486 ggc_internal_cleared_alloc (len);
12487 return clone_info;
12490 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12492 static inline void
12493 simd_clone_struct_copy (struct cgraph_simd_clone *to,
12494 struct cgraph_simd_clone *from)
12496 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
12497 + ((from->nargs - from->inbranch)
12498 * sizeof (struct cgraph_simd_clone_arg))));
12501 /* Return vector of parameter types of function FNDECL. This uses
12502 TYPE_ARG_TYPES if available, otherwise falls back to types of
12503 DECL_ARGUMENTS types. */
12505 vec<tree>
12506 simd_clone_vector_of_formal_parm_types (tree fndecl)
12508 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
12509 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
12510 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
12511 unsigned int i;
12512 tree arg;
12513 FOR_EACH_VEC_ELT (args, i, arg)
12514 args[i] = TREE_TYPE (args[i]);
12515 return args;
12518 /* Given a simd function in NODE, extract the simd specific
12519 information from the OMP clauses passed in CLAUSES, and return
12520 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12521 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12522 otherwise set to FALSE. */
12524 static struct cgraph_simd_clone *
12525 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
12526 bool *inbranch_specified)
12528 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
12529 tree t;
12530 int n;
12531 *inbranch_specified = false;
12533 n = args.length ();
12534 if (n > 0 && args.last () == void_type_node)
12535 n--;
12537 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12538 be cloned have a distinctive artificial label in addition to "omp
12539 declare simd". */
12540 bool cilk_clone
12541 = (flag_cilkplus
12542 && lookup_attribute ("cilk simd function",
12543 DECL_ATTRIBUTES (node->decl)));
12545 /* Allocate one more than needed just in case this is an in-branch
12546 clone which will require a mask argument. */
12547 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
12548 clone_info->nargs = n;
12549 clone_info->cilk_elemental = cilk_clone;
12551 if (!clauses)
12553 args.release ();
12554 return clone_info;
12556 clauses = TREE_VALUE (clauses);
12557 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
12558 return clone_info;
12560 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
12562 switch (OMP_CLAUSE_CODE (t))
12564 case OMP_CLAUSE_INBRANCH:
12565 clone_info->inbranch = 1;
12566 *inbranch_specified = true;
12567 break;
12568 case OMP_CLAUSE_NOTINBRANCH:
12569 clone_info->inbranch = 0;
12570 *inbranch_specified = true;
12571 break;
12572 case OMP_CLAUSE_SIMDLEN:
12573 clone_info->simdlen
12574 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
12575 break;
12576 case OMP_CLAUSE_LINEAR:
12578 tree decl = OMP_CLAUSE_DECL (t);
12579 tree step = OMP_CLAUSE_LINEAR_STEP (t);
12580 int argno = TREE_INT_CST_LOW (decl);
12581 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
12583 clone_info->args[argno].arg_type
12584 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
12585 clone_info->args[argno].linear_step = tree_to_shwi (step);
12586 gcc_assert (clone_info->args[argno].linear_step >= 0
12587 && clone_info->args[argno].linear_step < n);
12589 else
12591 if (POINTER_TYPE_P (args[argno]))
12592 step = fold_convert (ssizetype, step);
12593 if (!tree_fits_shwi_p (step))
12595 warning_at (OMP_CLAUSE_LOCATION (t), 0,
12596 "ignoring large linear step");
12597 args.release ();
12598 return NULL;
12600 else if (integer_zerop (step))
12602 warning_at (OMP_CLAUSE_LOCATION (t), 0,
12603 "ignoring zero linear step");
12604 args.release ();
12605 return NULL;
12607 else
12609 clone_info->args[argno].arg_type
12610 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
12611 clone_info->args[argno].linear_step = tree_to_shwi (step);
12614 break;
12616 case OMP_CLAUSE_UNIFORM:
12618 tree decl = OMP_CLAUSE_DECL (t);
12619 int argno = tree_to_uhwi (decl);
12620 clone_info->args[argno].arg_type
12621 = SIMD_CLONE_ARG_TYPE_UNIFORM;
12622 break;
12624 case OMP_CLAUSE_ALIGNED:
12626 tree decl = OMP_CLAUSE_DECL (t);
12627 int argno = tree_to_uhwi (decl);
12628 clone_info->args[argno].alignment
12629 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
12630 break;
12632 default:
12633 break;
12636 args.release ();
12637 return clone_info;
12640 /* Given a SIMD clone in NODE, calculate the characteristic data
12641 type and return the coresponding type. The characteristic data
12642 type is computed as described in the Intel Vector ABI. */
12644 static tree
12645 simd_clone_compute_base_data_type (struct cgraph_node *node,
12646 struct cgraph_simd_clone *clone_info)
12648 tree type = integer_type_node;
12649 tree fndecl = node->decl;
12651 /* a) For non-void function, the characteristic data type is the
12652 return type. */
12653 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
12654 type = TREE_TYPE (TREE_TYPE (fndecl));
12656 /* b) If the function has any non-uniform, non-linear parameters,
12657 then the characteristic data type is the type of the first
12658 such parameter. */
12659 else
12661 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
12662 for (unsigned int i = 0; i < clone_info->nargs; ++i)
12663 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
12665 type = map[i];
12666 break;
12668 map.release ();
12671 /* c) If the characteristic data type determined by a) or b) above
12672 is struct, union, or class type which is pass-by-value (except
12673 for the type that maps to the built-in complex data type), the
12674 characteristic data type is int. */
12675 if (RECORD_OR_UNION_TYPE_P (type)
12676 && !aggregate_value_p (type, NULL)
12677 && TREE_CODE (type) != COMPLEX_TYPE)
12678 return integer_type_node;
12680 /* d) If none of the above three classes is applicable, the
12681 characteristic data type is int. */
12683 return type;
12685 /* e) For Intel Xeon Phi native and offload compilation, if the
12686 resulting characteristic data type is 8-bit or 16-bit integer
12687 data type, the characteristic data type is int. */
12688 /* Well, we don't handle Xeon Phi yet. */
12691 static tree
12692 simd_clone_mangle (struct cgraph_node *node,
12693 struct cgraph_simd_clone *clone_info)
12695 char vecsize_mangle = clone_info->vecsize_mangle;
12696 char mask = clone_info->inbranch ? 'M' : 'N';
12697 unsigned int simdlen = clone_info->simdlen;
12698 unsigned int n;
12699 pretty_printer pp;
12701 gcc_assert (vecsize_mangle && simdlen);
12703 pp_string (&pp, "_ZGV");
12704 pp_character (&pp, vecsize_mangle);
12705 pp_character (&pp, mask);
12706 pp_decimal_int (&pp, simdlen);
12708 for (n = 0; n < clone_info->nargs; ++n)
12710 struct cgraph_simd_clone_arg arg = clone_info->args[n];
12712 if (arg.arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM)
12713 pp_character (&pp, 'u');
12714 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
12716 gcc_assert (arg.linear_step != 0);
12717 pp_character (&pp, 'l');
12718 if (arg.linear_step > 1)
12719 pp_unsigned_wide_integer (&pp, arg.linear_step);
12720 else if (arg.linear_step < 0)
12722 pp_character (&pp, 'n');
12723 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
12724 arg.linear_step));
12727 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
12729 pp_character (&pp, 's');
12730 pp_unsigned_wide_integer (&pp, arg.linear_step);
12732 else
12733 pp_character (&pp, 'v');
12734 if (arg.alignment)
12736 pp_character (&pp, 'a');
12737 pp_decimal_int (&pp, arg.alignment);
12741 pp_underscore (&pp);
12742 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
12743 if (*str == '*')
12744 ++str;
12745 pp_string (&pp, str);
12746 str = pp_formatted_text (&pp);
12748 /* If there already is a SIMD clone with the same mangled name, don't
12749 add another one. This can happen e.g. for
12750 #pragma omp declare simd
12751 #pragma omp declare simd simdlen(8)
12752 int foo (int, int);
12753 if the simdlen is assumed to be 8 for the first one, etc. */
12754 for (struct cgraph_node *clone = node->simd_clones; clone;
12755 clone = clone->simdclone->next_clone)
12756 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)),
12757 str) == 0)
12758 return NULL_TREE;
12760 return get_identifier (str);
12763 /* Create a simd clone of OLD_NODE and return it. */
12765 static struct cgraph_node *
12766 simd_clone_create (struct cgraph_node *old_node)
12768 struct cgraph_node *new_node;
12769 if (old_node->definition)
12771 if (!old_node->has_gimple_body_p ())
12772 return NULL;
12773 old_node->get_body ();
12774 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
12775 false, NULL, NULL,
12776 "simdclone");
12778 else
12780 tree old_decl = old_node->decl;
12781 tree new_decl = copy_node (old_node->decl);
12782 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
12783 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
12784 SET_DECL_RTL (new_decl, NULL);
12785 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
12786 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
12787 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
12788 if (old_node->in_other_partition)
12789 new_node->in_other_partition = 1;
12791 if (new_node == NULL)
12792 return new_node;
12794 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
12796 /* The function cgraph_function_versioning () will force the new
12797 symbol local. Undo this, and inherit external visability from
12798 the old node. */
12799 new_node->local.local = old_node->local.local;
12800 new_node->externally_visible = old_node->externally_visible;
12802 return new_node;
12805 /* Adjust the return type of the given function to its appropriate
12806 vector counterpart. Returns a simd array to be used throughout the
12807 function as a return value. */
12809 static tree
12810 simd_clone_adjust_return_type (struct cgraph_node *node)
12812 tree fndecl = node->decl;
12813 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
12814 unsigned int veclen;
12815 tree t;
12817 /* Adjust the function return type. */
12818 if (orig_rettype == void_type_node)
12819 return NULL_TREE;
12820 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
12821 t = TREE_TYPE (TREE_TYPE (fndecl));
12822 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
12823 veclen = node->simdclone->vecsize_int;
12824 else
12825 veclen = node->simdclone->vecsize_float;
12826 veclen /= GET_MODE_BITSIZE (TYPE_MODE (t));
12827 if (veclen > node->simdclone->simdlen)
12828 veclen = node->simdclone->simdlen;
12829 if (POINTER_TYPE_P (t))
12830 t = pointer_sized_int_node;
12831 if (veclen == node->simdclone->simdlen)
12832 t = build_vector_type (t, node->simdclone->simdlen);
12833 else
12835 t = build_vector_type (t, veclen);
12836 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
12838 TREE_TYPE (TREE_TYPE (fndecl)) = t;
12839 if (!node->definition)
12840 return NULL_TREE;
12842 t = DECL_RESULT (fndecl);
12843 /* Adjust the DECL_RESULT. */
12844 gcc_assert (TREE_TYPE (t) != void_type_node);
12845 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
12846 relayout_decl (t);
12848 tree atype = build_array_type_nelts (orig_rettype,
12849 node->simdclone->simdlen);
12850 if (veclen != node->simdclone->simdlen)
12851 return build1 (VIEW_CONVERT_EXPR, atype, t);
12853 /* Set up a SIMD array to use as the return value. */
12854 tree retval = create_tmp_var_raw (atype, "retval");
12855 gimple_add_tmp_var (retval);
12856 return retval;
12859 /* Each vector argument has a corresponding array to be used locally
12860 as part of the eventual loop. Create such temporary array and
12861 return it.
12863 PREFIX is the prefix to be used for the temporary.
12865 TYPE is the inner element type.
12867 SIMDLEN is the number of elements. */
12869 static tree
12870 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
12872 tree atype = build_array_type_nelts (type, simdlen);
12873 tree avar = create_tmp_var_raw (atype, prefix);
12874 gimple_add_tmp_var (avar);
12875 return avar;
12878 /* Modify the function argument types to their corresponding vector
12879 counterparts if appropriate. Also, create one array for each simd
12880 argument to be used locally when using the function arguments as
12881 part of the loop.
12883 NODE is the function whose arguments are to be adjusted.
12885 Returns an adjustment vector that will be filled describing how the
12886 argument types will be adjusted. */
12888 static ipa_parm_adjustment_vec
12889 simd_clone_adjust_argument_types (struct cgraph_node *node)
12891 vec<tree> args;
12892 ipa_parm_adjustment_vec adjustments;
12894 if (node->definition)
12895 args = ipa_get_vector_of_formal_parms (node->decl);
12896 else
12897 args = simd_clone_vector_of_formal_parm_types (node->decl);
12898 adjustments.create (args.length ());
12899 unsigned i, j, veclen;
12900 struct ipa_parm_adjustment adj;
12901 for (i = 0; i < node->simdclone->nargs; ++i)
12903 memset (&adj, 0, sizeof (adj));
12904 tree parm = args[i];
12905 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
12906 adj.base_index = i;
12907 adj.base = parm;
12909 node->simdclone->args[i].orig_arg = node->definition ? parm : NULL_TREE;
12910 node->simdclone->args[i].orig_type = parm_type;
12912 if (node->simdclone->args[i].arg_type != SIMD_CLONE_ARG_TYPE_VECTOR)
12914 /* No adjustment necessary for scalar arguments. */
12915 adj.op = IPA_PARM_OP_COPY;
12917 else
12919 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
12920 veclen = node->simdclone->vecsize_int;
12921 else
12922 veclen = node->simdclone->vecsize_float;
12923 veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type));
12924 if (veclen > node->simdclone->simdlen)
12925 veclen = node->simdclone->simdlen;
12926 adj.arg_prefix = "simd";
12927 if (POINTER_TYPE_P (parm_type))
12928 adj.type = build_vector_type (pointer_sized_int_node, veclen);
12929 else
12930 adj.type = build_vector_type (parm_type, veclen);
12931 node->simdclone->args[i].vector_type = adj.type;
12932 for (j = veclen; j < node->simdclone->simdlen; j += veclen)
12934 adjustments.safe_push (adj);
12935 if (j == veclen)
12937 memset (&adj, 0, sizeof (adj));
12938 adj.op = IPA_PARM_OP_NEW;
12939 adj.arg_prefix = "simd";
12940 adj.base_index = i;
12941 adj.type = node->simdclone->args[i].vector_type;
12945 if (node->definition)
12946 node->simdclone->args[i].simd_array
12947 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
12948 parm_type, node->simdclone->simdlen);
12950 adjustments.safe_push (adj);
12953 if (node->simdclone->inbranch)
12955 tree base_type
12956 = simd_clone_compute_base_data_type (node->simdclone->origin,
12957 node->simdclone);
12959 memset (&adj, 0, sizeof (adj));
12960 adj.op = IPA_PARM_OP_NEW;
12961 adj.arg_prefix = "mask";
12963 adj.base_index = i;
12964 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
12965 veclen = node->simdclone->vecsize_int;
12966 else
12967 veclen = node->simdclone->vecsize_float;
12968 veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
12969 if (veclen > node->simdclone->simdlen)
12970 veclen = node->simdclone->simdlen;
12971 if (POINTER_TYPE_P (base_type))
12972 adj.type = build_vector_type (pointer_sized_int_node, veclen);
12973 else
12974 adj.type = build_vector_type (base_type, veclen);
12975 adjustments.safe_push (adj);
12977 for (j = veclen; j < node->simdclone->simdlen; j += veclen)
12978 adjustments.safe_push (adj);
12980 /* We have previously allocated one extra entry for the mask. Use
12981 it and fill it. */
12982 struct cgraph_simd_clone *sc = node->simdclone;
12983 sc->nargs++;
12984 if (node->definition)
12986 sc->args[i].orig_arg
12987 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
12988 sc->args[i].simd_array
12989 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
12991 sc->args[i].orig_type = base_type;
12992 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
12995 if (node->definition)
12996 ipa_modify_formal_parameters (node->decl, adjustments);
12997 else
12999 tree new_arg_types = NULL_TREE, new_reversed;
13000 bool last_parm_void = false;
13001 if (args.length () > 0 && args.last () == void_type_node)
13002 last_parm_void = true;
13004 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
13005 j = adjustments.length ();
13006 for (i = 0; i < j; i++)
13008 struct ipa_parm_adjustment *adj = &adjustments[i];
13009 tree ptype;
13010 if (adj->op == IPA_PARM_OP_COPY)
13011 ptype = args[adj->base_index];
13012 else
13013 ptype = adj->type;
13014 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
13016 new_reversed = nreverse (new_arg_types);
13017 if (last_parm_void)
13019 if (new_reversed)
13020 TREE_CHAIN (new_arg_types) = void_list_node;
13021 else
13022 new_reversed = void_list_node;
13025 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
13026 TYPE_ARG_TYPES (new_type) = new_reversed;
13027 TREE_TYPE (node->decl) = new_type;
13029 adjustments.release ();
13031 args.release ();
13032 return adjustments;
13035 /* Initialize and copy the function arguments in NODE to their
13036 corresponding local simd arrays. Returns a fresh gimple_seq with
13037 the instruction sequence generated. */
13039 static gimple_seq
13040 simd_clone_init_simd_arrays (struct cgraph_node *node,
13041 ipa_parm_adjustment_vec adjustments)
13043 gimple_seq seq = NULL;
13044 unsigned i = 0, j = 0, k;
13046 for (tree arg = DECL_ARGUMENTS (node->decl);
13047 arg;
13048 arg = DECL_CHAIN (arg), i++, j++)
13050 if (adjustments[j].op == IPA_PARM_OP_COPY)
13051 continue;
13053 node->simdclone->args[i].vector_arg = arg;
13055 tree array = node->simdclone->args[i].simd_array;
13056 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
13058 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
13059 tree ptr = build_fold_addr_expr (array);
13060 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
13061 build_int_cst (ptype, 0));
13062 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
13063 gimplify_and_add (t, &seq);
13065 else
13067 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
13068 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
13069 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
13071 tree ptr = build_fold_addr_expr (array);
13072 int elemsize;
13073 if (k)
13075 arg = DECL_CHAIN (arg);
13076 j++;
13078 elemsize
13079 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg))));
13080 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
13081 build_int_cst (ptype, k * elemsize));
13082 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
13083 gimplify_and_add (t, &seq);
13087 return seq;
13090 /* Callback info for ipa_simd_modify_stmt_ops below. */
13092 struct modify_stmt_info {
13093 ipa_parm_adjustment_vec adjustments;
13094 gimple stmt;
13095 /* True if the parent statement was modified by
13096 ipa_simd_modify_stmt_ops. */
13097 bool modified;
13100 /* Callback for walk_gimple_op.
13102 Adjust operands from a given statement as specified in the
13103 adjustments vector in the callback data. */
13105 static tree
13106 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
13108 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
13109 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
13110 tree *orig_tp = tp;
13111 if (TREE_CODE (*tp) == ADDR_EXPR)
13112 tp = &TREE_OPERAND (*tp, 0);
13113 struct ipa_parm_adjustment *cand = NULL;
13114 if (TREE_CODE (*tp) == PARM_DECL)
13115 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
13116 else
13118 if (TYPE_P (*tp))
13119 *walk_subtrees = 0;
13122 tree repl = NULL_TREE;
13123 if (cand)
13124 repl = unshare_expr (cand->new_decl);
13125 else
13127 if (tp != orig_tp)
13129 *walk_subtrees = 0;
13130 bool modified = info->modified;
13131 info->modified = false;
13132 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
13133 if (!info->modified)
13135 info->modified = modified;
13136 return NULL_TREE;
13138 info->modified = modified;
13139 repl = *tp;
13141 else
13142 return NULL_TREE;
13145 if (tp != orig_tp)
13147 repl = build_fold_addr_expr (repl);
13148 gimple stmt;
13149 if (is_gimple_debug (info->stmt))
13151 tree vexpr = make_node (DEBUG_EXPR_DECL);
13152 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
13153 DECL_ARTIFICIAL (vexpr) = 1;
13154 TREE_TYPE (vexpr) = TREE_TYPE (repl);
13155 DECL_MODE (vexpr) = TYPE_MODE (TREE_TYPE (repl));
13156 repl = vexpr;
13158 else
13160 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
13161 repl = gimple_assign_lhs (stmt);
13163 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
13164 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
13165 *orig_tp = repl;
13167 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
13169 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
13170 *tp = vce;
13172 else
13173 *tp = repl;
13175 info->modified = true;
13176 return NULL_TREE;
13179 /* Traverse the function body and perform all modifications as
13180 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13181 modified such that the replacement/reduction value will now be an
13182 offset into the corresponding simd_array.
13184 This function will replace all function argument uses with their
13185 corresponding simd array elements, and ajust the return values
13186 accordingly. */
13188 static void
13189 ipa_simd_modify_function_body (struct cgraph_node *node,
13190 ipa_parm_adjustment_vec adjustments,
13191 tree retval_array, tree iter)
13193 basic_block bb;
13194 unsigned int i, j, l;
13196 /* Re-use the adjustments array, but this time use it to replace
13197 every function argument use to an offset into the corresponding
13198 simd_array. */
13199 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
13201 if (!node->simdclone->args[i].vector_arg)
13202 continue;
13204 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
13205 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
13206 adjustments[j].new_decl
13207 = build4 (ARRAY_REF,
13208 basetype,
13209 node->simdclone->args[i].simd_array,
13210 iter,
13211 NULL_TREE, NULL_TREE);
13212 if (adjustments[j].op == IPA_PARM_OP_NONE
13213 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
13214 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
13217 l = adjustments.length ();
13218 for (i = 1; i < num_ssa_names; i++)
13220 tree name = ssa_name (i);
13221 if (name
13222 && SSA_NAME_VAR (name)
13223 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
13225 for (j = 0; j < l; j++)
13226 if (SSA_NAME_VAR (name) == adjustments[j].base
13227 && adjustments[j].new_decl)
13229 tree base_var;
13230 if (adjustments[j].new_ssa_base == NULL_TREE)
13232 base_var
13233 = copy_var_decl (adjustments[j].base,
13234 DECL_NAME (adjustments[j].base),
13235 TREE_TYPE (adjustments[j].base));
13236 adjustments[j].new_ssa_base = base_var;
13238 else
13239 base_var = adjustments[j].new_ssa_base;
13240 if (SSA_NAME_IS_DEFAULT_DEF (name))
13242 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
13243 gimple_stmt_iterator gsi = gsi_after_labels (bb);
13244 tree new_decl = unshare_expr (adjustments[j].new_decl);
13245 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
13246 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
13247 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
13248 gimple stmt = gimple_build_assign (name, new_decl);
13249 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
13251 else
13252 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
13257 struct modify_stmt_info info;
13258 info.adjustments = adjustments;
13260 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
13262 gimple_stmt_iterator gsi;
13264 gsi = gsi_start_bb (bb);
13265 while (!gsi_end_p (gsi))
13267 gimple stmt = gsi_stmt (gsi);
13268 info.stmt = stmt;
13269 struct walk_stmt_info wi;
13271 memset (&wi, 0, sizeof (wi));
13272 info.modified = false;
13273 wi.info = &info;
13274 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
13276 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
13278 tree retval = gimple_return_retval (return_stmt);
13279 if (!retval)
13281 gsi_remove (&gsi, true);
13282 continue;
13285 /* Replace `return foo' with `retval_array[iter] = foo'. */
13286 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
13287 retval_array, iter, NULL, NULL);
13288 stmt = gimple_build_assign (ref, retval);
13289 gsi_replace (&gsi, stmt, true);
13290 info.modified = true;
13293 if (info.modified)
13295 update_stmt (stmt);
13296 if (maybe_clean_eh_stmt (stmt))
13297 gimple_purge_dead_eh_edges (gimple_bb (stmt));
13299 gsi_next (&gsi);
13304 /* Adjust the argument types in NODE to their appropriate vector
13305 counterparts. */
13307 static void
13308 simd_clone_adjust (struct cgraph_node *node)
13310 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
13312 targetm.simd_clone.adjust (node);
13314 tree retval = simd_clone_adjust_return_type (node);
13315 ipa_parm_adjustment_vec adjustments
13316 = simd_clone_adjust_argument_types (node);
13318 push_gimplify_context ();
13320 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
13322 /* Adjust all uses of vector arguments accordingly. Adjust all
13323 return values accordingly. */
13324 tree iter = create_tmp_var (unsigned_type_node, "iter");
13325 tree iter1 = make_ssa_name (iter);
13326 tree iter2 = make_ssa_name (iter);
13327 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
13329 /* Initialize the iteration variable. */
13330 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
13331 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
13332 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
13333 /* Insert the SIMD array and iv initialization at function
13334 entry. */
13335 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
13337 pop_gimplify_context (NULL);
13339 /* Create a new BB right before the original exit BB, to hold the
13340 iteration increment and the condition/branch. */
13341 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
13342 basic_block incr_bb = create_empty_bb (orig_exit);
13343 add_bb_to_loop (incr_bb, body_bb->loop_father);
13344 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13345 flag. Set it now to be a FALLTHRU_EDGE. */
13346 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
13347 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
13348 for (unsigned i = 0;
13349 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
13351 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
13352 redirect_edge_succ (e, incr_bb);
13354 edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
13355 e->probability = REG_BR_PROB_BASE;
13356 gsi = gsi_last_bb (incr_bb);
13357 gimple g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
13358 build_int_cst (unsigned_type_node, 1));
13359 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13361 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13362 struct loop *loop = alloc_loop ();
13363 cfun->has_force_vectorize_loops = true;
13364 loop->safelen = node->simdclone->simdlen;
13365 loop->force_vectorize = true;
13366 loop->header = body_bb;
13368 /* Branch around the body if the mask applies. */
13369 if (node->simdclone->inbranch)
13371 gimple_stmt_iterator gsi = gsi_last_bb (loop->header);
13372 tree mask_array
13373 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
13374 tree mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
13375 tree aref = build4 (ARRAY_REF,
13376 TREE_TYPE (TREE_TYPE (mask_array)),
13377 mask_array, iter1,
13378 NULL, NULL);
13379 g = gimple_build_assign (mask, aref);
13380 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13381 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref)));
13382 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
13384 aref = build1 (VIEW_CONVERT_EXPR,
13385 build_nonstandard_integer_type (bitsize, 0), mask);
13386 mask = make_ssa_name (TREE_TYPE (aref));
13387 g = gimple_build_assign (mask, aref);
13388 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13391 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
13392 NULL, NULL);
13393 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13394 make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
13395 FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE;
13398 /* Generate the condition. */
13399 g = gimple_build_cond (LT_EXPR,
13400 iter2,
13401 build_int_cst (unsigned_type_node,
13402 node->simdclone->simdlen),
13403 NULL, NULL);
13404 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13405 e = split_block (incr_bb, gsi_stmt (gsi));
13406 basic_block latch_bb = e->dest;
13407 basic_block new_exit_bb;
13408 new_exit_bb = split_block (latch_bb, NULL)->dest;
13409 loop->latch = latch_bb;
13411 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
13413 make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
13414 /* The successor of incr_bb is already pointing to latch_bb; just
13415 change the flags.
13416 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13417 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
13419 gphi *phi = create_phi_node (iter1, body_bb);
13420 edge preheader_edge = find_edge (entry_bb, body_bb);
13421 edge latch_edge = single_succ_edge (latch_bb);
13422 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
13423 UNKNOWN_LOCATION);
13424 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
13426 /* Generate the new return. */
13427 gsi = gsi_last_bb (new_exit_bb);
13428 if (retval
13429 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
13430 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
13431 retval = TREE_OPERAND (retval, 0);
13432 else if (retval)
13434 retval = build1 (VIEW_CONVERT_EXPR,
13435 TREE_TYPE (TREE_TYPE (node->decl)),
13436 retval);
13437 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
13438 false, GSI_CONTINUE_LINKING);
13440 g = gimple_build_return (retval);
13441 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13443 /* Handle aligned clauses by replacing default defs of the aligned
13444 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13445 lhs. Handle linear by adding PHIs. */
13446 for (unsigned i = 0; i < node->simdclone->nargs; i++)
13447 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
13448 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
13449 || !is_gimple_reg_type
13450 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
13452 tree orig_arg = node->simdclone->args[i].orig_arg;
13453 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
13454 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
13455 else
13457 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
13458 gimple_add_tmp_var (iter1);
13460 gsi = gsi_after_labels (entry_bb);
13461 g = gimple_build_assign (iter1, orig_arg);
13462 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
13463 gsi = gsi_after_labels (body_bb);
13464 g = gimple_build_assign (orig_arg, iter1);
13465 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
13467 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
13468 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
13469 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
13470 == REFERENCE_TYPE
13471 && TREE_ADDRESSABLE
13472 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
13474 tree orig_arg = node->simdclone->args[i].orig_arg;
13475 tree def = ssa_default_def (cfun, orig_arg);
13476 if (def && !has_zero_uses (def))
13478 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
13479 gimple_add_tmp_var (iter1);
13480 gsi = gsi_after_labels (entry_bb);
13481 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
13482 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
13483 gsi = gsi_after_labels (body_bb);
13484 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
13485 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
13488 else if (node->simdclone->args[i].alignment
13489 && node->simdclone->args[i].arg_type
13490 == SIMD_CLONE_ARG_TYPE_UNIFORM
13491 && (node->simdclone->args[i].alignment
13492 & (node->simdclone->args[i].alignment - 1)) == 0
13493 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
13494 == POINTER_TYPE)
13496 unsigned int alignment = node->simdclone->args[i].alignment;
13497 tree orig_arg = node->simdclone->args[i].orig_arg;
13498 tree def = ssa_default_def (cfun, orig_arg);
13499 if (def && !has_zero_uses (def))
13501 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
13502 gimple_seq seq = NULL;
13503 bool need_cvt = false;
13504 gcall *call
13505 = gimple_build_call (fn, 2, def, size_int (alignment));
13506 g = call;
13507 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
13508 ptr_type_node))
13509 need_cvt = true;
13510 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
13511 gimple_call_set_lhs (g, t);
13512 gimple_seq_add_stmt_without_update (&seq, g);
13513 if (need_cvt)
13515 t = make_ssa_name (orig_arg);
13516 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
13517 gimple_seq_add_stmt_without_update (&seq, g);
13519 gsi_insert_seq_on_edge_immediate
13520 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
13522 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
13523 int freq = compute_call_stmt_bb_frequency (current_function_decl,
13524 entry_bb);
13525 node->create_edge (cgraph_node::get_create (fn),
13526 call, entry_bb->count, freq);
13528 imm_use_iterator iter;
13529 use_operand_p use_p;
13530 gimple use_stmt;
13531 tree repl = gimple_get_lhs (g);
13532 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
13533 if (is_gimple_debug (use_stmt) || use_stmt == call)
13534 continue;
13535 else
13536 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
13537 SET_USE (use_p, repl);
13540 else if (node->simdclone->args[i].arg_type
13541 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
13543 tree orig_arg = node->simdclone->args[i].orig_arg;
13544 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
13545 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
13546 tree def = NULL_TREE;
13547 if (TREE_ADDRESSABLE (orig_arg))
13549 def = make_ssa_name (TREE_TYPE (orig_arg));
13550 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
13551 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
13552 gsi = gsi_after_labels (entry_bb);
13553 g = gimple_build_assign (def, orig_arg);
13554 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
13556 else
13558 def = ssa_default_def (cfun, orig_arg);
13559 if (!def || has_zero_uses (def))
13560 def = NULL_TREE;
13561 else
13563 iter1 = make_ssa_name (orig_arg);
13564 iter2 = make_ssa_name (orig_arg);
13567 if (def)
13569 phi = create_phi_node (iter1, body_bb);
13570 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
13571 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
13572 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
13573 ? PLUS_EXPR : POINTER_PLUS_EXPR;
13574 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
13575 ? TREE_TYPE (orig_arg) : sizetype;
13576 tree addcst
13577 = build_int_cst (addtype, node->simdclone->args[i].linear_step);
13578 g = gimple_build_assign (iter2, code, iter1, addcst);
13579 gsi = gsi_last_bb (incr_bb);
13580 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
13582 imm_use_iterator iter;
13583 use_operand_p use_p;
13584 gimple use_stmt;
13585 if (TREE_ADDRESSABLE (orig_arg))
13587 gsi = gsi_after_labels (body_bb);
13588 g = gimple_build_assign (orig_arg, iter1);
13589 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
13591 else
13592 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
13593 if (use_stmt == phi)
13594 continue;
13595 else
13596 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
13597 SET_USE (use_p, iter1);
13601 calculate_dominance_info (CDI_DOMINATORS);
13602 add_loop (loop, loop->header->loop_father);
13603 update_ssa (TODO_update_ssa);
13605 pop_cfun ();
13608 /* If the function in NODE is tagged as an elemental SIMD function,
13609 create the appropriate SIMD clones. */
13611 static void
13612 expand_simd_clones (struct cgraph_node *node)
13614 tree attr = lookup_attribute ("omp declare simd",
13615 DECL_ATTRIBUTES (node->decl));
13616 if (attr == NULL_TREE
13617 || node->global.inlined_to
13618 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
13619 return;
13621 /* Ignore
13622 #pragma omp declare simd
13623 extern int foo ();
13624 in C, there we don't know the argument types at all. */
13625 if (!node->definition
13626 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
13627 return;
13629 /* Call this before creating clone_info, as it might ggc_collect. */
13630 if (node->definition && node->has_gimple_body_p ())
13631 node->get_body ();
13635 /* Start with parsing the "omp declare simd" attribute(s). */
13636 bool inbranch_clause_specified;
13637 struct cgraph_simd_clone *clone_info
13638 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
13639 &inbranch_clause_specified);
13640 if (clone_info == NULL)
13641 continue;
13643 int orig_simdlen = clone_info->simdlen;
13644 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
13645 /* The target can return 0 (no simd clones should be created),
13646 1 (just one ISA of simd clones should be created) or higher
13647 count of ISA variants. In that case, clone_info is initialized
13648 for the first ISA variant. */
13649 int count
13650 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
13651 base_type, 0);
13652 if (count == 0)
13653 continue;
13655 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13656 also create one inbranch and one !inbranch clone of it. */
13657 for (int i = 0; i < count * 2; i++)
13659 struct cgraph_simd_clone *clone = clone_info;
13660 if (inbranch_clause_specified && (i & 1) != 0)
13661 continue;
13663 if (i != 0)
13665 clone = simd_clone_struct_alloc (clone_info->nargs
13666 + ((i & 1) != 0));
13667 simd_clone_struct_copy (clone, clone_info);
13668 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13669 and simd_clone_adjust_argument_types did to the first
13670 clone's info. */
13671 clone->nargs -= clone_info->inbranch;
13672 clone->simdlen = orig_simdlen;
13673 /* And call the target hook again to get the right ISA. */
13674 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
13675 base_type,
13676 i / 2);
13677 if ((i & 1) != 0)
13678 clone->inbranch = 1;
13681 /* simd_clone_mangle might fail if such a clone has been created
13682 already. */
13683 tree id = simd_clone_mangle (node, clone);
13684 if (id == NULL_TREE)
13685 continue;
13687 /* Only when we are sure we want to create the clone actually
13688 clone the function (or definitions) or create another
13689 extern FUNCTION_DECL (for prototypes without definitions). */
13690 struct cgraph_node *n = simd_clone_create (node);
13691 if (n == NULL)
13692 continue;
13694 n->simdclone = clone;
13695 clone->origin = node;
13696 clone->next_clone = NULL;
13697 if (node->simd_clones == NULL)
13699 clone->prev_clone = n;
13700 node->simd_clones = n;
13702 else
13704 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
13705 clone->prev_clone->simdclone->next_clone = n;
13706 node->simd_clones->simdclone->prev_clone = n;
13708 symtab->change_decl_assembler_name (n->decl, id);
13709 /* And finally adjust the return type, parameters and for
13710 definitions also function body. */
13711 if (node->definition)
13712 simd_clone_adjust (n);
13713 else
13715 simd_clone_adjust_return_type (n);
13716 simd_clone_adjust_argument_types (n);
13720 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
13723 /* Entry point for IPA simd clone creation pass. */
13725 static unsigned int
13726 ipa_omp_simd_clone (void)
13728 struct cgraph_node *node;
13729 FOR_EACH_FUNCTION (node)
13730 expand_simd_clones (node);
13731 return 0;
13734 namespace {
13736 const pass_data pass_data_omp_simd_clone =
13738 SIMPLE_IPA_PASS, /* type */
13739 "simdclone", /* name */
13740 OPTGROUP_NONE, /* optinfo_flags */
13741 TV_NONE, /* tv_id */
13742 ( PROP_ssa | PROP_cfg ), /* properties_required */
13743 0, /* properties_provided */
13744 0, /* properties_destroyed */
13745 0, /* todo_flags_start */
13746 0, /* todo_flags_finish */
13749 class pass_omp_simd_clone : public simple_ipa_opt_pass
13751 public:
13752 pass_omp_simd_clone(gcc::context *ctxt)
13753 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
13756 /* opt_pass methods: */
13757 virtual bool gate (function *);
13758 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
13761 bool
13762 pass_omp_simd_clone::gate (function *)
13764 return ((flag_openmp || flag_openmp_simd
13765 || flag_cilkplus
13766 || (in_lto_p && !flag_wpa))
13767 && (targetm.simd_clone.compute_vecsize_and_simdlen != NULL));
13770 } // anon namespace
13772 simple_ipa_opt_pass *
13773 make_pass_omp_simd_clone (gcc::context *ctxt)
13775 return new pass_omp_simd_clone (ctxt);
13778 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
13779 adds their addresses and sizes to constructor-vector V_CTOR. */
13780 static void
13781 add_decls_addresses_to_decl_constructor (vec<tree, va_gc> *v_decls,
13782 vec<constructor_elt, va_gc> *v_ctor)
13784 unsigned len = vec_safe_length (v_decls);
13785 for (unsigned i = 0; i < len; i++)
13787 tree it = (*v_decls)[i];
13788 bool is_function = TREE_CODE (it) != VAR_DECL;
13790 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, build_fold_addr_expr (it));
13791 if (!is_function)
13792 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE,
13793 fold_convert (const_ptr_type_node,
13794 DECL_SIZE_UNIT (it)));
13798 /* Create new symbols containing (address, size) pairs for global variables,
13799 marked with "omp declare target" attribute, as well as addresses for the
13800 functions, which are outlined offloading regions. */
13801 void
13802 omp_finish_file (void)
13804 unsigned num_funcs = vec_safe_length (offload_funcs);
13805 unsigned num_vars = vec_safe_length (offload_vars);
13807 if (num_funcs == 0 && num_vars == 0)
13808 return;
13810 if (targetm_common.have_named_sections)
13812 vec<constructor_elt, va_gc> *v_f, *v_v;
13813 vec_alloc (v_f, num_funcs);
13814 vec_alloc (v_v, num_vars * 2);
13816 add_decls_addresses_to_decl_constructor (offload_funcs, v_f);
13817 add_decls_addresses_to_decl_constructor (offload_vars, v_v);
13819 tree vars_decl_type = build_array_type_nelts (pointer_sized_int_node,
13820 num_vars * 2);
13821 tree funcs_decl_type = build_array_type_nelts (pointer_sized_int_node,
13822 num_funcs);
13823 TYPE_ALIGN (vars_decl_type) = TYPE_ALIGN (pointer_sized_int_node);
13824 TYPE_ALIGN (funcs_decl_type) = TYPE_ALIGN (pointer_sized_int_node);
13825 tree ctor_v = build_constructor (vars_decl_type, v_v);
13826 tree ctor_f = build_constructor (funcs_decl_type, v_f);
13827 TREE_CONSTANT (ctor_v) = TREE_CONSTANT (ctor_f) = 1;
13828 TREE_STATIC (ctor_v) = TREE_STATIC (ctor_f) = 1;
13829 tree funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
13830 get_identifier (".offload_func_table"),
13831 funcs_decl_type);
13832 tree vars_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
13833 get_identifier (".offload_var_table"),
13834 vars_decl_type);
13835 TREE_STATIC (funcs_decl) = TREE_STATIC (vars_decl) = 1;
13836 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
13837 otherwise a joint table in a binary will contain padding between
13838 tables from multiple object files. */
13839 DECL_USER_ALIGN (funcs_decl) = DECL_USER_ALIGN (vars_decl) = 1;
13840 DECL_ALIGN (funcs_decl) = TYPE_ALIGN (funcs_decl_type);
13841 DECL_ALIGN (vars_decl) = TYPE_ALIGN (vars_decl_type);
13842 DECL_INITIAL (funcs_decl) = ctor_f;
13843 DECL_INITIAL (vars_decl) = ctor_v;
13844 set_decl_section_name (funcs_decl, OFFLOAD_FUNC_TABLE_SECTION_NAME);
13845 set_decl_section_name (vars_decl, OFFLOAD_VAR_TABLE_SECTION_NAME);
13847 varpool_node::finalize_decl (vars_decl);
13848 varpool_node::finalize_decl (funcs_decl);
13850 else
13852 for (unsigned i = 0; i < num_funcs; i++)
13854 tree it = (*offload_funcs)[i];
13855 targetm.record_offload_symbol (it);
13857 for (unsigned i = 0; i < num_vars; i++)
13859 tree it = (*offload_vars)[i];
13860 targetm.record_offload_symbol (it);
13865 #include "gt-omp-low.h"