PR sanitizer/65400
[official-gcc.git] / gcc / omp-low.c
blob48d73cbc02110b015203f4bde3c625197107e2bb
1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
14 version.
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 for more details.
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
25 #include "config.h"
26 #include "system.h"
27 #include "coretypes.h"
28 #include "tm.h"
29 #include "hash-set.h"
30 #include "machmode.h"
31 #include "vec.h"
32 #include "double-int.h"
33 #include "input.h"
34 #include "alias.h"
35 #include "symtab.h"
36 #include "wide-int.h"
37 #include "inchash.h"
38 #include "tree.h"
39 #include "fold-const.h"
40 #include "stringpool.h"
41 #include "stor-layout.h"
42 #include "rtl.h"
43 #include "predict.h"
44 #include "hard-reg-set.h"
45 #include "function.h"
46 #include "dominance.h"
47 #include "cfg.h"
48 #include "cfganal.h"
49 #include "basic-block.h"
50 #include "tree-ssa-alias.h"
51 #include "internal-fn.h"
52 #include "gimple-fold.h"
53 #include "gimple-expr.h"
54 #include "is-a.h"
55 #include "gimple.h"
56 #include "gimplify.h"
57 #include "gimple-iterator.h"
58 #include "gimplify-me.h"
59 #include "gimple-walk.h"
60 #include "tree-iterator.h"
61 #include "tree-inline.h"
62 #include "langhooks.h"
63 #include "diagnostic-core.h"
64 #include "gimple-ssa.h"
65 #include "hash-map.h"
66 #include "plugin-api.h"
67 #include "ipa-ref.h"
68 #include "cgraph.h"
69 #include "tree-cfg.h"
70 #include "tree-phinodes.h"
71 #include "ssa-iterators.h"
72 #include "tree-ssanames.h"
73 #include "tree-into-ssa.h"
74 #include "hashtab.h"
75 #include "flags.h"
76 #include "statistics.h"
77 #include "real.h"
78 #include "fixed-value.h"
79 #include "insn-config.h"
80 #include "expmed.h"
81 #include "dojump.h"
82 #include "explow.h"
83 #include "calls.h"
84 #include "emit-rtl.h"
85 #include "varasm.h"
86 #include "stmt.h"
87 #include "expr.h"
88 #include "tree-dfa.h"
89 #include "tree-ssa.h"
90 #include "tree-pass.h"
91 #include "except.h"
92 #include "splay-tree.h"
93 #include "insn-codes.h"
94 #include "optabs.h"
95 #include "cfgloop.h"
96 #include "target.h"
97 #include "common/common-target.h"
98 #include "omp-low.h"
99 #include "gimple-low.h"
100 #include "tree-cfgcleanup.h"
101 #include "pretty-print.h"
102 #include "alloc-pool.h"
103 #include "symbol-summary.h"
104 #include "ipa-prop.h"
105 #include "tree-nested.h"
106 #include "tree-eh.h"
107 #include "cilk.h"
108 #include "context.h"
109 #include "lto-section-names.h"
110 #include "gomp-constants.h"
113 /* Lowering of OMP parallel and workshare constructs proceeds in two
114 phases. The first phase scans the function looking for OMP statements
115 and then for variables that must be replaced to satisfy data sharing
116 clauses. The second phase expands code for the constructs, as well as
117 re-gimplifying things when variables have been replaced with complex
118 expressions.
120 Final code generation is done by pass_expand_omp. The flowgraph is
121 scanned for regions which are then moved to a new
122 function, to be invoked by the thread library, or offloaded. */
124 /* OMP region information. Every parallel and workshare
125 directive is enclosed between two markers, the OMP_* directive
126 and a corresponding OMP_RETURN statement. */
128 struct omp_region
130 /* The enclosing region. */
131 struct omp_region *outer;
133 /* First child region. */
134 struct omp_region *inner;
136 /* Next peer region. */
137 struct omp_region *next;
139 /* Block containing the omp directive as its last stmt. */
140 basic_block entry;
142 /* Block containing the OMP_RETURN as its last stmt. */
143 basic_block exit;
145 /* Block containing the OMP_CONTINUE as its last stmt. */
146 basic_block cont;
148 /* If this is a combined parallel+workshare region, this is a list
149 of additional arguments needed by the combined parallel+workshare
150 library call. */
151 vec<tree, va_gc> *ws_args;
153 /* The code for the omp directive of this region. */
154 enum gimple_code type;
156 /* Schedule kind, only used for OMP_FOR type regions. */
157 enum omp_clause_schedule_kind sched_kind;
159 /* True if this is a combined parallel+workshare region. */
160 bool is_combined_parallel;
163 /* Levels of parallelism as defined by OpenACC. Increasing numbers
164 correspond to deeper loop nesting levels. */
165 #define MASK_GANG 1
166 #define MASK_WORKER 2
167 #define MASK_VECTOR 4
169 /* Context structure. Used to store information about each parallel
170 directive in the code. */
172 typedef struct omp_context
174 /* This field must be at the beginning, as we do "inheritance": Some
175 callback functions for tree-inline.c (e.g., omp_copy_decl)
176 receive a copy_body_data pointer that is up-casted to an
177 omp_context pointer. */
178 copy_body_data cb;
180 /* The tree of contexts corresponding to the encountered constructs. */
181 struct omp_context *outer;
182 gimple stmt;
184 /* Map variables to fields in a structure that allows communication
185 between sending and receiving threads. */
186 splay_tree field_map;
187 tree record_type;
188 tree sender_decl;
189 tree receiver_decl;
191 /* These are used just by task contexts, if task firstprivate fn is
192 needed. srecord_type is used to communicate from the thread
193 that encountered the task construct to task firstprivate fn,
194 record_type is allocated by GOMP_task, initialized by task firstprivate
195 fn and passed to the task body fn. */
196 splay_tree sfield_map;
197 tree srecord_type;
199 /* A chain of variables to add to the top-level block surrounding the
200 construct. In the case of a parallel, this is in the child function. */
201 tree block_vars;
203 /* A map of reduction pointer variables. For accelerators, each
204 reduction variable is replaced with an array. Each thread, in turn,
205 is assigned to a slot on that array. */
206 splay_tree reduction_map;
208 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
209 barriers should jump to during omplower pass. */
210 tree cancel_label;
212 /* What to do with variables with implicitly determined sharing
213 attributes. */
214 enum omp_clause_default_kind default_kind;
216 /* Nesting depth of this context. Used to beautify error messages re
217 invalid gotos. The outermost ctx is depth 1, with depth 0 being
218 reserved for the main body of the function. */
219 int depth;
221 /* True if this parallel directive is nested within another. */
222 bool is_nested;
224 /* True if this construct can be cancelled. */
225 bool cancellable;
227 /* For OpenACC loops, a mask of gang, worker and vector used at
228 levels below this one. */
229 int gwv_below;
230 /* For OpenACC loops, a mask of gang, worker and vector used at
231 this level and above. For parallel and kernels clauses, a mask
232 indicating which of num_gangs/num_workers/num_vectors was used. */
233 int gwv_this;
234 } omp_context;
236 /* A structure holding the elements of:
237 for (V = N1; V cond N2; V += STEP) [...] */
239 struct omp_for_data_loop
241 tree v, n1, n2, step;
242 enum tree_code cond_code;
245 /* A structure describing the main elements of a parallel loop. */
247 struct omp_for_data
249 struct omp_for_data_loop loop;
250 tree chunk_size;
251 gomp_for *for_stmt;
252 tree pre, iter_type;
253 int collapse;
254 bool have_nowait, have_ordered;
255 enum omp_clause_schedule_kind sched_kind;
256 struct omp_for_data_loop *loops;
260 static splay_tree all_contexts;
261 static int taskreg_nesting_level;
262 static int target_nesting_level;
263 static struct omp_region *root_omp_region;
264 static bitmap task_shared_vars;
265 static vec<omp_context *> taskreg_contexts;
267 static void scan_omp (gimple_seq *, omp_context *);
268 static tree scan_omp_1_op (tree *, int *, void *);
270 #define WALK_SUBSTMTS \
271 case GIMPLE_BIND: \
272 case GIMPLE_TRY: \
273 case GIMPLE_CATCH: \
274 case GIMPLE_EH_FILTER: \
275 case GIMPLE_TRANSACTION: \
276 /* The sub-statements for these should be walked. */ \
277 *handled_ops_p = false; \
278 break;
280 /* Helper function to get the name of the array containing the partial
281 reductions for OpenACC reductions. */
282 static const char *
283 oacc_get_reduction_array_id (tree node)
285 const char *id = IDENTIFIER_POINTER (DECL_NAME (node));
286 int len = strlen ("OACC") + strlen (id);
287 char *temp_name = XALLOCAVEC (char, len + 1);
288 snprintf (temp_name, len + 1, "OACC%s", id);
289 return IDENTIFIER_POINTER (get_identifier (temp_name));
292 /* Determine the number of threads OpenACC threads used to determine the
293 size of the array of partial reductions. Currently, this is num_gangs
294 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
295 because it is independed of the device used. */
297 static tree
298 oacc_max_threads (omp_context *ctx)
300 tree nthreads, vector_length, gangs, clauses;
302 gangs = fold_convert (sizetype, integer_one_node);
303 vector_length = gangs;
305 /* The reduction clause may be nested inside a loop directive.
306 Scan for the innermost vector_length clause. */
307 for (omp_context *oc = ctx; oc; oc = oc->outer)
309 if (gimple_code (oc->stmt) != GIMPLE_OMP_TARGET
310 || (gimple_omp_target_kind (oc->stmt)
311 != GF_OMP_TARGET_KIND_OACC_PARALLEL))
312 continue;
314 clauses = gimple_omp_target_clauses (oc->stmt);
316 vector_length = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH);
317 if (vector_length)
318 vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length),
319 sizetype,
320 OMP_CLAUSE_VECTOR_LENGTH_EXPR
321 (vector_length));
322 else
323 vector_length = fold_convert (sizetype, integer_one_node);
325 gangs = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS);
326 if (gangs)
327 gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (gangs), sizetype,
328 OMP_CLAUSE_NUM_GANGS_EXPR (gangs));
329 else
330 gangs = fold_convert (sizetype, integer_one_node);
332 break;
335 nthreads = fold_build2 (MULT_EXPR, sizetype, gangs, vector_length);
337 return nthreads;
340 /* Holds offload tables with decls. */
341 vec<tree, va_gc> *offload_funcs, *offload_vars;
343 /* Convenience function for calling scan_omp_1_op on tree operands. */
345 static inline tree
346 scan_omp_op (tree *tp, omp_context *ctx)
348 struct walk_stmt_info wi;
350 memset (&wi, 0, sizeof (wi));
351 wi.info = ctx;
352 wi.want_locations = true;
354 return walk_tree (tp, scan_omp_1_op, &wi, NULL);
357 static void lower_omp (gimple_seq *, omp_context *);
358 static tree lookup_decl_in_outer_ctx (tree, omp_context *);
359 static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
361 /* Find an OMP clause of type KIND within CLAUSES. */
363 tree
364 find_omp_clause (tree clauses, enum omp_clause_code kind)
366 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
367 if (OMP_CLAUSE_CODE (clauses) == kind)
368 return clauses;
370 return NULL_TREE;
373 /* Return true if CTX is for an omp parallel. */
375 static inline bool
376 is_parallel_ctx (omp_context *ctx)
378 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
382 /* Return true if CTX is for an omp task. */
384 static inline bool
385 is_task_ctx (omp_context *ctx)
387 return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
391 /* Return true if CTX is for an omp parallel or omp task. */
393 static inline bool
394 is_taskreg_ctx (omp_context *ctx)
396 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL
397 || gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
401 /* Return true if REGION is a combined parallel+workshare region. */
403 static inline bool
404 is_combined_parallel (struct omp_region *region)
406 return region->is_combined_parallel;
410 /* Extract the header elements of parallel loop FOR_STMT and store
411 them into *FD. */
413 static void
414 extract_omp_for_data (gomp_for *for_stmt, struct omp_for_data *fd,
415 struct omp_for_data_loop *loops)
417 tree t, var, *collapse_iter, *collapse_count;
418 tree count = NULL_TREE, iter_type = long_integer_type_node;
419 struct omp_for_data_loop *loop;
420 int i;
421 struct omp_for_data_loop dummy_loop;
422 location_t loc = gimple_location (for_stmt);
423 bool simd = gimple_omp_for_kind (for_stmt) & GF_OMP_FOR_SIMD;
424 bool distribute = gimple_omp_for_kind (for_stmt)
425 == GF_OMP_FOR_KIND_DISTRIBUTE;
427 fd->for_stmt = for_stmt;
428 fd->pre = NULL;
429 fd->collapse = gimple_omp_for_collapse (for_stmt);
430 if (fd->collapse > 1)
431 fd->loops = loops;
432 else
433 fd->loops = &fd->loop;
435 fd->have_nowait = distribute || simd;
436 fd->have_ordered = false;
437 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
438 fd->chunk_size = NULL_TREE;
439 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_CILKFOR)
440 fd->sched_kind = OMP_CLAUSE_SCHEDULE_CILKFOR;
441 collapse_iter = NULL;
442 collapse_count = NULL;
444 for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t))
445 switch (OMP_CLAUSE_CODE (t))
447 case OMP_CLAUSE_NOWAIT:
448 fd->have_nowait = true;
449 break;
450 case OMP_CLAUSE_ORDERED:
451 fd->have_ordered = true;
452 break;
453 case OMP_CLAUSE_SCHEDULE:
454 gcc_assert (!distribute);
455 fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t);
456 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t);
457 break;
458 case OMP_CLAUSE_DIST_SCHEDULE:
459 gcc_assert (distribute);
460 fd->chunk_size = OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t);
461 break;
462 case OMP_CLAUSE_COLLAPSE:
463 if (fd->collapse > 1)
465 collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t);
466 collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t);
468 break;
469 default:
470 break;
473 /* FIXME: for now map schedule(auto) to schedule(static).
474 There should be analysis to determine whether all iterations
475 are approximately the same amount of work (then schedule(static)
476 is best) or if it varies (then schedule(dynamic,N) is better). */
477 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO)
479 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
480 gcc_assert (fd->chunk_size == NULL);
482 gcc_assert (fd->collapse == 1 || collapse_iter != NULL);
483 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
484 gcc_assert (fd->chunk_size == NULL);
485 else if (fd->chunk_size == NULL)
487 /* We only need to compute a default chunk size for ordered
488 static loops and dynamic loops. */
489 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
490 || fd->have_ordered)
491 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
492 ? integer_zero_node : integer_one_node;
495 for (i = 0; i < fd->collapse; i++)
497 if (fd->collapse == 1)
498 loop = &fd->loop;
499 else if (loops != NULL)
500 loop = loops + i;
501 else
502 loop = &dummy_loop;
504 loop->v = gimple_omp_for_index (for_stmt, i);
505 gcc_assert (SSA_VAR_P (loop->v));
506 gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
507 || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE);
508 var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v;
509 loop->n1 = gimple_omp_for_initial (for_stmt, i);
511 loop->cond_code = gimple_omp_for_cond (for_stmt, i);
512 loop->n2 = gimple_omp_for_final (for_stmt, i);
513 switch (loop->cond_code)
515 case LT_EXPR:
516 case GT_EXPR:
517 break;
518 case NE_EXPR:
519 gcc_assert (gimple_omp_for_kind (for_stmt)
520 == GF_OMP_FOR_KIND_CILKSIMD
521 || (gimple_omp_for_kind (for_stmt)
522 == GF_OMP_FOR_KIND_CILKFOR));
523 break;
524 case LE_EXPR:
525 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
526 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, 1);
527 else
528 loop->n2 = fold_build2_loc (loc,
529 PLUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
530 build_int_cst (TREE_TYPE (loop->n2), 1));
531 loop->cond_code = LT_EXPR;
532 break;
533 case GE_EXPR:
534 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
535 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, -1);
536 else
537 loop->n2 = fold_build2_loc (loc,
538 MINUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
539 build_int_cst (TREE_TYPE (loop->n2), 1));
540 loop->cond_code = GT_EXPR;
541 break;
542 default:
543 gcc_unreachable ();
546 t = gimple_omp_for_incr (for_stmt, i);
547 gcc_assert (TREE_OPERAND (t, 0) == var);
548 switch (TREE_CODE (t))
550 case PLUS_EXPR:
551 loop->step = TREE_OPERAND (t, 1);
552 break;
553 case POINTER_PLUS_EXPR:
554 loop->step = fold_convert (ssizetype, TREE_OPERAND (t, 1));
555 break;
556 case MINUS_EXPR:
557 loop->step = TREE_OPERAND (t, 1);
558 loop->step = fold_build1_loc (loc,
559 NEGATE_EXPR, TREE_TYPE (loop->step),
560 loop->step);
561 break;
562 default:
563 gcc_unreachable ();
566 if (simd
567 || (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
568 && !fd->have_ordered))
570 if (fd->collapse == 1)
571 iter_type = TREE_TYPE (loop->v);
572 else if (i == 0
573 || TYPE_PRECISION (iter_type)
574 < TYPE_PRECISION (TREE_TYPE (loop->v)))
575 iter_type
576 = build_nonstandard_integer_type
577 (TYPE_PRECISION (TREE_TYPE (loop->v)), 1);
579 else if (iter_type != long_long_unsigned_type_node)
581 if (POINTER_TYPE_P (TREE_TYPE (loop->v)))
582 iter_type = long_long_unsigned_type_node;
583 else if (TYPE_UNSIGNED (TREE_TYPE (loop->v))
584 && TYPE_PRECISION (TREE_TYPE (loop->v))
585 >= TYPE_PRECISION (iter_type))
587 tree n;
589 if (loop->cond_code == LT_EXPR)
590 n = fold_build2_loc (loc,
591 PLUS_EXPR, TREE_TYPE (loop->v),
592 loop->n2, loop->step);
593 else
594 n = loop->n1;
595 if (TREE_CODE (n) != INTEGER_CST
596 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n))
597 iter_type = long_long_unsigned_type_node;
599 else if (TYPE_PRECISION (TREE_TYPE (loop->v))
600 > TYPE_PRECISION (iter_type))
602 tree n1, n2;
604 if (loop->cond_code == LT_EXPR)
606 n1 = loop->n1;
607 n2 = fold_build2_loc (loc,
608 PLUS_EXPR, TREE_TYPE (loop->v),
609 loop->n2, loop->step);
611 else
613 n1 = fold_build2_loc (loc,
614 MINUS_EXPR, TREE_TYPE (loop->v),
615 loop->n2, loop->step);
616 n2 = loop->n1;
618 if (TREE_CODE (n1) != INTEGER_CST
619 || TREE_CODE (n2) != INTEGER_CST
620 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1)
621 || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type)))
622 iter_type = long_long_unsigned_type_node;
626 if (collapse_count && *collapse_count == NULL)
628 t = fold_binary (loop->cond_code, boolean_type_node,
629 fold_convert (TREE_TYPE (loop->v), loop->n1),
630 fold_convert (TREE_TYPE (loop->v), loop->n2));
631 if (t && integer_zerop (t))
632 count = build_zero_cst (long_long_unsigned_type_node);
633 else if ((i == 0 || count != NULL_TREE)
634 && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
635 && TREE_CONSTANT (loop->n1)
636 && TREE_CONSTANT (loop->n2)
637 && TREE_CODE (loop->step) == INTEGER_CST)
639 tree itype = TREE_TYPE (loop->v);
641 if (POINTER_TYPE_P (itype))
642 itype = signed_type_for (itype);
643 t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1));
644 t = fold_build2_loc (loc,
645 PLUS_EXPR, itype,
646 fold_convert_loc (loc, itype, loop->step), t);
647 t = fold_build2_loc (loc, PLUS_EXPR, itype, t,
648 fold_convert_loc (loc, itype, loop->n2));
649 t = fold_build2_loc (loc, MINUS_EXPR, itype, t,
650 fold_convert_loc (loc, itype, loop->n1));
651 if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR)
652 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype,
653 fold_build1_loc (loc, NEGATE_EXPR, itype, t),
654 fold_build1_loc (loc, NEGATE_EXPR, itype,
655 fold_convert_loc (loc, itype,
656 loop->step)));
657 else
658 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t,
659 fold_convert_loc (loc, itype, loop->step));
660 t = fold_convert_loc (loc, long_long_unsigned_type_node, t);
661 if (count != NULL_TREE)
662 count = fold_build2_loc (loc,
663 MULT_EXPR, long_long_unsigned_type_node,
664 count, t);
665 else
666 count = t;
667 if (TREE_CODE (count) != INTEGER_CST)
668 count = NULL_TREE;
670 else if (count && !integer_zerop (count))
671 count = NULL_TREE;
675 if (count
676 && !simd
677 && (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
678 || fd->have_ordered))
680 if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node)))
681 iter_type = long_long_unsigned_type_node;
682 else
683 iter_type = long_integer_type_node;
685 else if (collapse_iter && *collapse_iter != NULL)
686 iter_type = TREE_TYPE (*collapse_iter);
687 fd->iter_type = iter_type;
688 if (collapse_iter && *collapse_iter == NULL)
689 *collapse_iter = create_tmp_var (iter_type, ".iter");
690 if (collapse_count && *collapse_count == NULL)
692 if (count)
693 *collapse_count = fold_convert_loc (loc, iter_type, count);
694 else
695 *collapse_count = create_tmp_var (iter_type, ".count");
698 if (fd->collapse > 1)
700 fd->loop.v = *collapse_iter;
701 fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0);
702 fd->loop.n2 = *collapse_count;
703 fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1);
704 fd->loop.cond_code = LT_EXPR;
707 /* For OpenACC loops, force a chunk size of one, as this avoids the default
708 scheduling where several subsequent iterations are being executed by the
709 same thread. */
710 if (gimple_omp_for_kind (for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
712 gcc_assert (fd->chunk_size == NULL_TREE);
713 fd->chunk_size = build_int_cst (TREE_TYPE (fd->loop.v), 1);
718 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
719 is the immediate dominator of PAR_ENTRY_BB, return true if there
720 are no data dependencies that would prevent expanding the parallel
721 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
723 When expanding a combined parallel+workshare region, the call to
724 the child function may need additional arguments in the case of
725 GIMPLE_OMP_FOR regions. In some cases, these arguments are
726 computed out of variables passed in from the parent to the child
727 via 'struct .omp_data_s'. For instance:
729 #pragma omp parallel for schedule (guided, i * 4)
730 for (j ...)
732 Is lowered into:
734 # BLOCK 2 (PAR_ENTRY_BB)
735 .omp_data_o.i = i;
736 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
738 # BLOCK 3 (WS_ENTRY_BB)
739 .omp_data_i = &.omp_data_o;
740 D.1667 = .omp_data_i->i;
741 D.1598 = D.1667 * 4;
742 #pragma omp for schedule (guided, D.1598)
744 When we outline the parallel region, the call to the child function
745 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
746 that value is computed *after* the call site. So, in principle we
747 cannot do the transformation.
749 To see whether the code in WS_ENTRY_BB blocks the combined
750 parallel+workshare call, we collect all the variables used in the
751 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
752 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
753 call.
755 FIXME. If we had the SSA form built at this point, we could merely
756 hoist the code in block 3 into block 2 and be done with it. But at
757 this point we don't have dataflow information and though we could
758 hack something up here, it is really not worth the aggravation. */
760 static bool
761 workshare_safe_to_combine_p (basic_block ws_entry_bb)
763 struct omp_for_data fd;
764 gimple ws_stmt = last_stmt (ws_entry_bb);
766 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
767 return true;
769 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
771 extract_omp_for_data (as_a <gomp_for *> (ws_stmt), &fd, NULL);
773 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
774 return false;
775 if (fd.iter_type != long_integer_type_node)
776 return false;
778 /* FIXME. We give up too easily here. If any of these arguments
779 are not constants, they will likely involve variables that have
780 been mapped into fields of .omp_data_s for sharing with the child
781 function. With appropriate data flow, it would be possible to
782 see through this. */
783 if (!is_gimple_min_invariant (fd.loop.n1)
784 || !is_gimple_min_invariant (fd.loop.n2)
785 || !is_gimple_min_invariant (fd.loop.step)
786 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
787 return false;
789 return true;
793 /* Collect additional arguments needed to emit a combined
794 parallel+workshare call. WS_STMT is the workshare directive being
795 expanded. */
797 static vec<tree, va_gc> *
798 get_ws_args_for (gimple par_stmt, gimple ws_stmt)
800 tree t;
801 location_t loc = gimple_location (ws_stmt);
802 vec<tree, va_gc> *ws_args;
804 if (gomp_for *for_stmt = dyn_cast <gomp_for *> (ws_stmt))
806 struct omp_for_data fd;
807 tree n1, n2;
809 extract_omp_for_data (for_stmt, &fd, NULL);
810 n1 = fd.loop.n1;
811 n2 = fd.loop.n2;
813 if (gimple_omp_for_combined_into_p (for_stmt))
815 tree innerc
816 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt),
817 OMP_CLAUSE__LOOPTEMP_);
818 gcc_assert (innerc);
819 n1 = OMP_CLAUSE_DECL (innerc);
820 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
821 OMP_CLAUSE__LOOPTEMP_);
822 gcc_assert (innerc);
823 n2 = OMP_CLAUSE_DECL (innerc);
826 vec_alloc (ws_args, 3 + (fd.chunk_size != 0));
828 t = fold_convert_loc (loc, long_integer_type_node, n1);
829 ws_args->quick_push (t);
831 t = fold_convert_loc (loc, long_integer_type_node, n2);
832 ws_args->quick_push (t);
834 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step);
835 ws_args->quick_push (t);
837 if (fd.chunk_size)
839 t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size);
840 ws_args->quick_push (t);
843 return ws_args;
845 else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
847 /* Number of sections is equal to the number of edges from the
848 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
849 the exit of the sections region. */
850 basic_block bb = single_succ (gimple_bb (ws_stmt));
851 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
852 vec_alloc (ws_args, 1);
853 ws_args->quick_push (t);
854 return ws_args;
857 gcc_unreachable ();
861 /* Discover whether REGION is a combined parallel+workshare region. */
863 static void
864 determine_parallel_type (struct omp_region *region)
866 basic_block par_entry_bb, par_exit_bb;
867 basic_block ws_entry_bb, ws_exit_bb;
869 if (region == NULL || region->inner == NULL
870 || region->exit == NULL || region->inner->exit == NULL
871 || region->inner->cont == NULL)
872 return;
874 /* We only support parallel+for and parallel+sections. */
875 if (region->type != GIMPLE_OMP_PARALLEL
876 || (region->inner->type != GIMPLE_OMP_FOR
877 && region->inner->type != GIMPLE_OMP_SECTIONS))
878 return;
880 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
881 WS_EXIT_BB -> PAR_EXIT_BB. */
882 par_entry_bb = region->entry;
883 par_exit_bb = region->exit;
884 ws_entry_bb = region->inner->entry;
885 ws_exit_bb = region->inner->exit;
887 if (single_succ (par_entry_bb) == ws_entry_bb
888 && single_succ (ws_exit_bb) == par_exit_bb
889 && workshare_safe_to_combine_p (ws_entry_bb)
890 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb))
891 || (last_and_only_stmt (ws_entry_bb)
892 && last_and_only_stmt (par_exit_bb))))
894 gimple par_stmt = last_stmt (par_entry_bb);
895 gimple ws_stmt = last_stmt (ws_entry_bb);
897 if (region->inner->type == GIMPLE_OMP_FOR)
899 /* If this is a combined parallel loop, we need to determine
900 whether or not to use the combined library calls. There
901 are two cases where we do not apply the transformation:
902 static loops and any kind of ordered loop. In the first
903 case, we already open code the loop so there is no need
904 to do anything else. In the latter case, the combined
905 parallel loop call would still need extra synchronization
906 to implement ordered semantics, so there would not be any
907 gain in using the combined call. */
908 tree clauses = gimple_omp_for_clauses (ws_stmt);
909 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE);
910 if (c == NULL
911 || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC
912 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED))
914 region->is_combined_parallel = false;
915 region->inner->is_combined_parallel = false;
916 return;
920 region->is_combined_parallel = true;
921 region->inner->is_combined_parallel = true;
922 region->ws_args = get_ws_args_for (par_stmt, ws_stmt);
927 /* Return true if EXPR is variable sized. */
929 static inline bool
930 is_variable_sized (const_tree expr)
932 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
935 /* Return true if DECL is a reference type. */
937 static inline bool
938 is_reference (tree decl)
940 return lang_hooks.decls.omp_privatize_by_reference (decl);
943 /* Return the type of a decl. If the decl is reference type,
944 return its base type. */
945 static inline tree
946 get_base_type (tree decl)
948 tree type = TREE_TYPE (decl);
949 if (is_reference (decl))
950 type = TREE_TYPE (type);
951 return type;
954 /* Lookup variables. The "maybe" form
955 allows for the variable form to not have been entered, otherwise we
956 assert that the variable must have been entered. */
958 static inline tree
959 lookup_decl (tree var, omp_context *ctx)
961 tree *n = ctx->cb.decl_map->get (var);
962 return *n;
965 static inline tree
966 maybe_lookup_decl (const_tree var, omp_context *ctx)
968 tree *n = ctx->cb.decl_map->get (const_cast<tree> (var));
969 return n ? *n : NULL_TREE;
972 static inline tree
973 lookup_field (tree var, omp_context *ctx)
975 splay_tree_node n;
976 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
977 return (tree) n->value;
980 static inline tree
981 lookup_sfield (tree var, omp_context *ctx)
983 splay_tree_node n;
984 n = splay_tree_lookup (ctx->sfield_map
985 ? ctx->sfield_map : ctx->field_map,
986 (splay_tree_key) var);
987 return (tree) n->value;
990 static inline tree
991 maybe_lookup_field (tree var, omp_context *ctx)
993 splay_tree_node n;
994 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
995 return n ? (tree) n->value : NULL_TREE;
998 static inline tree
999 lookup_oacc_reduction (const char *id, omp_context *ctx)
1001 splay_tree_node n;
1002 n = splay_tree_lookup (ctx->reduction_map, (splay_tree_key) id);
1003 return (tree) n->value;
1006 static inline tree
1007 maybe_lookup_oacc_reduction (tree var, omp_context *ctx)
1009 splay_tree_node n = NULL;
1010 if (ctx->reduction_map)
1011 n = splay_tree_lookup (ctx->reduction_map, (splay_tree_key) var);
1012 return n ? (tree) n->value : NULL_TREE;
1015 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1016 the parallel context if DECL is to be shared. */
1018 static bool
1019 use_pointer_for_field (tree decl, omp_context *shared_ctx)
1021 if (AGGREGATE_TYPE_P (TREE_TYPE (decl)))
1022 return true;
1024 /* We can only use copy-in/copy-out semantics for shared variables
1025 when we know the value is not accessible from an outer scope. */
1026 if (shared_ctx)
1028 gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt));
1030 /* ??? Trivially accessible from anywhere. But why would we even
1031 be passing an address in this case? Should we simply assert
1032 this to be false, or should we have a cleanup pass that removes
1033 these from the list of mappings? */
1034 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
1035 return true;
1037 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1038 without analyzing the expression whether or not its location
1039 is accessible to anyone else. In the case of nested parallel
1040 regions it certainly may be. */
1041 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
1042 return true;
1044 /* Do not use copy-in/copy-out for variables that have their
1045 address taken. */
1046 if (TREE_ADDRESSABLE (decl))
1047 return true;
1049 /* lower_send_shared_vars only uses copy-in, but not copy-out
1050 for these. */
1051 if (TREE_READONLY (decl)
1052 || ((TREE_CODE (decl) == RESULT_DECL
1053 || TREE_CODE (decl) == PARM_DECL)
1054 && DECL_BY_REFERENCE (decl)))
1055 return false;
1057 /* Disallow copy-in/out in nested parallel if
1058 decl is shared in outer parallel, otherwise
1059 each thread could store the shared variable
1060 in its own copy-in location, making the
1061 variable no longer really shared. */
1062 if (shared_ctx->is_nested)
1064 omp_context *up;
1066 for (up = shared_ctx->outer; up; up = up->outer)
1067 if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up))
1068 break;
1070 if (up)
1072 tree c;
1074 for (c = gimple_omp_taskreg_clauses (up->stmt);
1075 c; c = OMP_CLAUSE_CHAIN (c))
1076 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
1077 && OMP_CLAUSE_DECL (c) == decl)
1078 break;
1080 if (c)
1081 goto maybe_mark_addressable_and_ret;
1085 /* For tasks avoid using copy-in/out. As tasks can be
1086 deferred or executed in different thread, when GOMP_task
1087 returns, the task hasn't necessarily terminated. */
1088 if (is_task_ctx (shared_ctx))
1090 tree outer;
1091 maybe_mark_addressable_and_ret:
1092 outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
1093 if (is_gimple_reg (outer))
1095 /* Taking address of OUTER in lower_send_shared_vars
1096 might need regimplification of everything that uses the
1097 variable. */
1098 if (!task_shared_vars)
1099 task_shared_vars = BITMAP_ALLOC (NULL);
1100 bitmap_set_bit (task_shared_vars, DECL_UID (outer));
1101 TREE_ADDRESSABLE (outer) = 1;
1103 return true;
1107 return false;
1110 /* Construct a new automatic decl similar to VAR. */
1112 static tree
1113 omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
1115 tree copy = copy_var_decl (var, name, type);
1117 DECL_CONTEXT (copy) = current_function_decl;
1118 DECL_CHAIN (copy) = ctx->block_vars;
1119 ctx->block_vars = copy;
1121 return copy;
1124 static tree
1125 omp_copy_decl_1 (tree var, omp_context *ctx)
1127 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
1130 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1131 as appropriate. */
1132 static tree
1133 omp_build_component_ref (tree obj, tree field)
1135 tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL);
1136 if (TREE_THIS_VOLATILE (field))
1137 TREE_THIS_VOLATILE (ret) |= 1;
1138 if (TREE_READONLY (field))
1139 TREE_READONLY (ret) |= 1;
1140 return ret;
1143 /* Build tree nodes to access the field for VAR on the receiver side. */
1145 static tree
1146 build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
1148 tree x, field = lookup_field (var, ctx);
1150 /* If the receiver record type was remapped in the child function,
1151 remap the field into the new record type. */
1152 x = maybe_lookup_field (field, ctx);
1153 if (x != NULL)
1154 field = x;
1156 x = build_simple_mem_ref (ctx->receiver_decl);
1157 x = omp_build_component_ref (x, field);
1158 if (by_ref)
1159 x = build_simple_mem_ref (x);
1161 return x;
1164 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1165 of a parallel, this is a component reference; for workshare constructs
1166 this is some variable. */
1168 static tree
1169 build_outer_var_ref (tree var, omp_context *ctx)
1171 tree x;
1173 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
1174 x = var;
1175 else if (is_variable_sized (var))
1177 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
1178 x = build_outer_var_ref (x, ctx);
1179 x = build_simple_mem_ref (x);
1181 else if (is_taskreg_ctx (ctx))
1183 bool by_ref = use_pointer_for_field (var, NULL);
1184 x = build_receiver_ref (var, by_ref, ctx);
1186 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1187 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
1189 /* #pragma omp simd isn't a worksharing construct, and can reference even
1190 private vars in its linear etc. clauses. */
1191 x = NULL_TREE;
1192 if (ctx->outer && is_taskreg_ctx (ctx))
1193 x = lookup_decl (var, ctx->outer);
1194 else if (ctx->outer)
1195 x = maybe_lookup_decl_in_outer_ctx (var, ctx);
1196 if (x == NULL_TREE)
1197 x = var;
1199 else if (ctx->outer)
1200 x = lookup_decl (var, ctx->outer);
1201 else if (is_reference (var))
1202 /* This can happen with orphaned constructs. If var is reference, it is
1203 possible it is shared and as such valid. */
1204 x = var;
1205 else
1206 gcc_unreachable ();
1208 if (is_reference (var))
1209 x = build_simple_mem_ref (x);
1211 return x;
1214 /* Build tree nodes to access the field for VAR on the sender side. */
1216 static tree
1217 build_sender_ref (tree var, omp_context *ctx)
1219 tree field = lookup_sfield (var, ctx);
1220 return omp_build_component_ref (ctx->sender_decl, field);
1223 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1225 static void
1226 install_var_field (tree var, bool by_ref, int mask, omp_context *ctx)
1228 tree field, type, sfield = NULL_TREE;
1230 gcc_assert ((mask & 1) == 0
1231 || !splay_tree_lookup (ctx->field_map, (splay_tree_key) var));
1232 gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
1233 || !splay_tree_lookup (ctx->sfield_map, (splay_tree_key) var));
1234 gcc_assert ((mask & 3) == 3
1235 || !is_gimple_omp_oacc (ctx->stmt));
1237 type = TREE_TYPE (var);
1238 if (mask & 4)
1240 gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
1241 type = build_pointer_type (build_pointer_type (type));
1243 else if (by_ref)
1244 type = build_pointer_type (type);
1245 else if ((mask & 3) == 1 && is_reference (var))
1246 type = TREE_TYPE (type);
1248 field = build_decl (DECL_SOURCE_LOCATION (var),
1249 FIELD_DECL, DECL_NAME (var), type);
1251 /* Remember what variable this field was created for. This does have a
1252 side effect of making dwarf2out ignore this member, so for helpful
1253 debugging we clear it later in delete_omp_context. */
1254 DECL_ABSTRACT_ORIGIN (field) = var;
1255 if (type == TREE_TYPE (var))
1257 DECL_ALIGN (field) = DECL_ALIGN (var);
1258 DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
1259 TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
1261 else
1262 DECL_ALIGN (field) = TYPE_ALIGN (type);
1264 if ((mask & 3) == 3)
1266 insert_field_into_struct (ctx->record_type, field);
1267 if (ctx->srecord_type)
1269 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1270 FIELD_DECL, DECL_NAME (var), type);
1271 DECL_ABSTRACT_ORIGIN (sfield) = var;
1272 DECL_ALIGN (sfield) = DECL_ALIGN (field);
1273 DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
1274 TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
1275 insert_field_into_struct (ctx->srecord_type, sfield);
1278 else
1280 if (ctx->srecord_type == NULL_TREE)
1282 tree t;
1284 ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
1285 ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1286 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
1288 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1289 FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
1290 DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
1291 insert_field_into_struct (ctx->srecord_type, sfield);
1292 splay_tree_insert (ctx->sfield_map,
1293 (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
1294 (splay_tree_value) sfield);
1297 sfield = field;
1298 insert_field_into_struct ((mask & 1) ? ctx->record_type
1299 : ctx->srecord_type, field);
1302 if (mask & 1)
1303 splay_tree_insert (ctx->field_map, (splay_tree_key) var,
1304 (splay_tree_value) field);
1305 if ((mask & 2) && ctx->sfield_map)
1306 splay_tree_insert (ctx->sfield_map, (splay_tree_key) var,
1307 (splay_tree_value) sfield);
1310 static tree
1311 install_var_local (tree var, omp_context *ctx)
1313 tree new_var = omp_copy_decl_1 (var, ctx);
1314 insert_decl_map (&ctx->cb, var, new_var);
1315 return new_var;
1318 /* Adjust the replacement for DECL in CTX for the new context. This means
1319 copying the DECL_VALUE_EXPR, and fixing up the type. */
1321 static void
1322 fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
1324 tree new_decl, size;
1326 new_decl = lookup_decl (decl, ctx);
1328 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
1330 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
1331 && DECL_HAS_VALUE_EXPR_P (decl))
1333 tree ve = DECL_VALUE_EXPR (decl);
1334 walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
1335 SET_DECL_VALUE_EXPR (new_decl, ve);
1336 DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
1339 if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
1341 size = remap_decl (DECL_SIZE (decl), &ctx->cb);
1342 if (size == error_mark_node)
1343 size = TYPE_SIZE (TREE_TYPE (new_decl));
1344 DECL_SIZE (new_decl) = size;
1346 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
1347 if (size == error_mark_node)
1348 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
1349 DECL_SIZE_UNIT (new_decl) = size;
1353 /* The callback for remap_decl. Search all containing contexts for a
1354 mapping of the variable; this avoids having to duplicate the splay
1355 tree ahead of time. We know a mapping doesn't already exist in the
1356 given context. Create new mappings to implement default semantics. */
1358 static tree
1359 omp_copy_decl (tree var, copy_body_data *cb)
1361 omp_context *ctx = (omp_context *) cb;
1362 tree new_var;
1364 if (TREE_CODE (var) == LABEL_DECL)
1366 new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
1367 DECL_CONTEXT (new_var) = current_function_decl;
1368 insert_decl_map (&ctx->cb, var, new_var);
1369 return new_var;
1372 while (!is_taskreg_ctx (ctx))
1374 ctx = ctx->outer;
1375 if (ctx == NULL)
1376 return var;
1377 new_var = maybe_lookup_decl (var, ctx);
1378 if (new_var)
1379 return new_var;
1382 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
1383 return var;
1385 return error_mark_node;
1389 /* Debugging dumps for parallel regions. */
1390 void dump_omp_region (FILE *, struct omp_region *, int);
1391 void debug_omp_region (struct omp_region *);
1392 void debug_all_omp_regions (void);
1394 /* Dump the parallel region tree rooted at REGION. */
1396 void
1397 dump_omp_region (FILE *file, struct omp_region *region, int indent)
1399 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
1400 gimple_code_name[region->type]);
1402 if (region->inner)
1403 dump_omp_region (file, region->inner, indent + 4);
1405 if (region->cont)
1407 fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "",
1408 region->cont->index);
1411 if (region->exit)
1412 fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "",
1413 region->exit->index);
1414 else
1415 fprintf (file, "%*s[no exit marker]\n", indent, "");
1417 if (region->next)
1418 dump_omp_region (file, region->next, indent);
1421 DEBUG_FUNCTION void
1422 debug_omp_region (struct omp_region *region)
1424 dump_omp_region (stderr, region, 0);
1427 DEBUG_FUNCTION void
1428 debug_all_omp_regions (void)
1430 dump_omp_region (stderr, root_omp_region, 0);
1434 /* Create a new parallel region starting at STMT inside region PARENT. */
1436 static struct omp_region *
1437 new_omp_region (basic_block bb, enum gimple_code type,
1438 struct omp_region *parent)
1440 struct omp_region *region = XCNEW (struct omp_region);
1442 region->outer = parent;
1443 region->entry = bb;
1444 region->type = type;
1446 if (parent)
1448 /* This is a nested region. Add it to the list of inner
1449 regions in PARENT. */
1450 region->next = parent->inner;
1451 parent->inner = region;
1453 else
1455 /* This is a toplevel region. Add it to the list of toplevel
1456 regions in ROOT_OMP_REGION. */
1457 region->next = root_omp_region;
1458 root_omp_region = region;
1461 return region;
1464 /* Release the memory associated with the region tree rooted at REGION. */
1466 static void
1467 free_omp_region_1 (struct omp_region *region)
1469 struct omp_region *i, *n;
1471 for (i = region->inner; i ; i = n)
1473 n = i->next;
1474 free_omp_region_1 (i);
1477 free (region);
1480 /* Release the memory for the entire omp region tree. */
1482 void
1483 free_omp_regions (void)
1485 struct omp_region *r, *n;
1486 for (r = root_omp_region; r ; r = n)
1488 n = r->next;
1489 free_omp_region_1 (r);
1491 root_omp_region = NULL;
1495 /* Create a new context, with OUTER_CTX being the surrounding context. */
1497 static omp_context *
1498 new_omp_context (gimple stmt, omp_context *outer_ctx)
1500 omp_context *ctx = XCNEW (omp_context);
1502 splay_tree_insert (all_contexts, (splay_tree_key) stmt,
1503 (splay_tree_value) ctx);
1504 ctx->stmt = stmt;
1506 if (outer_ctx)
1508 ctx->outer = outer_ctx;
1509 ctx->cb = outer_ctx->cb;
1510 ctx->cb.block = NULL;
1511 ctx->depth = outer_ctx->depth + 1;
1512 ctx->reduction_map = outer_ctx->reduction_map;
1514 else
1516 ctx->cb.src_fn = current_function_decl;
1517 ctx->cb.dst_fn = current_function_decl;
1518 ctx->cb.src_node = cgraph_node::get (current_function_decl);
1519 gcc_checking_assert (ctx->cb.src_node);
1520 ctx->cb.dst_node = ctx->cb.src_node;
1521 ctx->cb.src_cfun = cfun;
1522 ctx->cb.copy_decl = omp_copy_decl;
1523 ctx->cb.eh_lp_nr = 0;
1524 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1525 ctx->depth = 1;
1528 ctx->cb.decl_map = new hash_map<tree, tree>;
1530 return ctx;
1533 static gimple_seq maybe_catch_exception (gimple_seq);
1535 /* Finalize task copyfn. */
1537 static void
1538 finalize_task_copyfn (gomp_task *task_stmt)
1540 struct function *child_cfun;
1541 tree child_fn;
1542 gimple_seq seq = NULL, new_seq;
1543 gbind *bind;
1545 child_fn = gimple_omp_task_copy_fn (task_stmt);
1546 if (child_fn == NULL_TREE)
1547 return;
1549 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1550 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1552 push_cfun (child_cfun);
1553 bind = gimplify_body (child_fn, false);
1554 gimple_seq_add_stmt (&seq, bind);
1555 new_seq = maybe_catch_exception (seq);
1556 if (new_seq != seq)
1558 bind = gimple_build_bind (NULL, new_seq, NULL);
1559 seq = NULL;
1560 gimple_seq_add_stmt (&seq, bind);
1562 gimple_set_body (child_fn, seq);
1563 pop_cfun ();
1565 /* Inform the callgraph about the new function. */
1566 cgraph_node::add_new_function (child_fn, false);
1569 /* Destroy a omp_context data structures. Called through the splay tree
1570 value delete callback. */
1572 static void
1573 delete_omp_context (splay_tree_value value)
1575 omp_context *ctx = (omp_context *) value;
1577 delete ctx->cb.decl_map;
1579 if (ctx->field_map)
1580 splay_tree_delete (ctx->field_map);
1581 if (ctx->sfield_map)
1582 splay_tree_delete (ctx->sfield_map);
1583 /* Reduction map is copied to nested contexts, so only delete it in the
1584 owner. */
1585 if (ctx->reduction_map
1586 && gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
1587 && is_gimple_omp_offloaded (ctx->stmt)
1588 && is_gimple_omp_oacc (ctx->stmt))
1589 splay_tree_delete (ctx->reduction_map);
1591 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1592 it produces corrupt debug information. */
1593 if (ctx->record_type)
1595 tree t;
1596 for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1597 DECL_ABSTRACT_ORIGIN (t) = NULL;
1599 if (ctx->srecord_type)
1601 tree t;
1602 for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1603 DECL_ABSTRACT_ORIGIN (t) = NULL;
1606 if (is_task_ctx (ctx))
1607 finalize_task_copyfn (as_a <gomp_task *> (ctx->stmt));
1609 XDELETE (ctx);
1612 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1613 context. */
1615 static void
1616 fixup_child_record_type (omp_context *ctx)
1618 tree f, type = ctx->record_type;
1620 /* ??? It isn't sufficient to just call remap_type here, because
1621 variably_modified_type_p doesn't work the way we expect for
1622 record types. Testing each field for whether it needs remapping
1623 and creating a new record by hand works, however. */
1624 for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1625 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1626 break;
1627 if (f)
1629 tree name, new_fields = NULL;
1631 type = lang_hooks.types.make_type (RECORD_TYPE);
1632 name = DECL_NAME (TYPE_NAME (ctx->record_type));
1633 name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1634 TYPE_DECL, name, type);
1635 TYPE_NAME (type) = name;
1637 for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1639 tree new_f = copy_node (f);
1640 DECL_CONTEXT (new_f) = type;
1641 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1642 DECL_CHAIN (new_f) = new_fields;
1643 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1644 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1645 &ctx->cb, NULL);
1646 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1647 &ctx->cb, NULL);
1648 new_fields = new_f;
1650 /* Arrange to be able to look up the receiver field
1651 given the sender field. */
1652 splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1653 (splay_tree_value) new_f);
1655 TYPE_FIELDS (type) = nreverse (new_fields);
1656 layout_type (type);
1659 TREE_TYPE (ctx->receiver_decl)
1660 = build_qualified_type (build_reference_type (type), TYPE_QUAL_RESTRICT);
1663 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1664 specified by CLAUSES. */
1666 static void
1667 scan_sharing_clauses (tree clauses, omp_context *ctx)
1669 tree c, decl;
1670 bool scan_array_reductions = false;
1672 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1674 bool by_ref;
1676 switch (OMP_CLAUSE_CODE (c))
1678 case OMP_CLAUSE_PRIVATE:
1679 decl = OMP_CLAUSE_DECL (c);
1680 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1681 goto do_private;
1682 else if (!is_variable_sized (decl))
1683 install_var_local (decl, ctx);
1684 break;
1686 case OMP_CLAUSE_SHARED:
1687 decl = OMP_CLAUSE_DECL (c);
1688 /* Ignore shared directives in teams construct. */
1689 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1691 /* Global variables don't need to be copied,
1692 the receiver side will use them directly. */
1693 tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx);
1694 if (is_global_var (odecl))
1695 break;
1696 insert_decl_map (&ctx->cb, decl, odecl);
1697 break;
1699 gcc_assert (is_taskreg_ctx (ctx));
1700 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1701 || !is_variable_sized (decl));
1702 /* Global variables don't need to be copied,
1703 the receiver side will use them directly. */
1704 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1705 break;
1706 by_ref = use_pointer_for_field (decl, ctx);
1707 if (! TREE_READONLY (decl)
1708 || TREE_ADDRESSABLE (decl)
1709 || by_ref
1710 || is_reference (decl))
1712 install_var_field (decl, by_ref, 3, ctx);
1713 install_var_local (decl, ctx);
1714 break;
1716 /* We don't need to copy const scalar vars back. */
1717 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1718 goto do_private;
1720 case OMP_CLAUSE_LASTPRIVATE:
1721 /* Let the corresponding firstprivate clause create
1722 the variable. */
1723 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1724 break;
1725 /* FALLTHRU */
1727 case OMP_CLAUSE_FIRSTPRIVATE:
1728 if (is_gimple_omp_oacc (ctx->stmt))
1730 sorry ("clause not supported yet");
1731 break;
1733 /* FALLTHRU */
1734 case OMP_CLAUSE_REDUCTION:
1735 case OMP_CLAUSE_LINEAR:
1736 decl = OMP_CLAUSE_DECL (c);
1737 do_private:
1738 if (is_variable_sized (decl))
1740 if (is_task_ctx (ctx))
1741 install_var_field (decl, false, 1, ctx);
1742 break;
1744 else if (is_taskreg_ctx (ctx))
1746 bool global
1747 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1748 by_ref = use_pointer_for_field (decl, NULL);
1750 if (is_task_ctx (ctx)
1751 && (global || by_ref || is_reference (decl)))
1753 install_var_field (decl, false, 1, ctx);
1754 if (!global)
1755 install_var_field (decl, by_ref, 2, ctx);
1757 else if (!global)
1758 install_var_field (decl, by_ref, 3, ctx);
1760 install_var_local (decl, ctx);
1761 if (is_gimple_omp_oacc (ctx->stmt)
1762 && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
1764 /* Create a decl for the reduction array. */
1765 tree var = OMP_CLAUSE_DECL (c);
1766 tree type = get_base_type (var);
1767 tree ptype = build_pointer_type (type);
1768 tree array = create_tmp_var (ptype,
1769 oacc_get_reduction_array_id (var));
1770 omp_context *c = (ctx->field_map ? ctx : ctx->outer);
1771 install_var_field (array, true, 3, c);
1772 install_var_local (array, c);
1774 /* Insert it into the current context. */
1775 splay_tree_insert (ctx->reduction_map, (splay_tree_key)
1776 oacc_get_reduction_array_id (var),
1777 (splay_tree_value) array);
1778 splay_tree_insert (ctx->reduction_map,
1779 (splay_tree_key) array,
1780 (splay_tree_value) array);
1782 break;
1784 case OMP_CLAUSE__LOOPTEMP_:
1785 gcc_assert (is_parallel_ctx (ctx));
1786 decl = OMP_CLAUSE_DECL (c);
1787 install_var_field (decl, false, 3, ctx);
1788 install_var_local (decl, ctx);
1789 break;
1791 case OMP_CLAUSE_COPYPRIVATE:
1792 case OMP_CLAUSE_COPYIN:
1793 decl = OMP_CLAUSE_DECL (c);
1794 by_ref = use_pointer_for_field (decl, NULL);
1795 install_var_field (decl, by_ref, 3, ctx);
1796 break;
1798 case OMP_CLAUSE_DEFAULT:
1799 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c);
1800 break;
1802 case OMP_CLAUSE_FINAL:
1803 case OMP_CLAUSE_IF:
1804 case OMP_CLAUSE_NUM_THREADS:
1805 case OMP_CLAUSE_NUM_TEAMS:
1806 case OMP_CLAUSE_THREAD_LIMIT:
1807 case OMP_CLAUSE_DEVICE:
1808 case OMP_CLAUSE_SCHEDULE:
1809 case OMP_CLAUSE_DIST_SCHEDULE:
1810 case OMP_CLAUSE_DEPEND:
1811 case OMP_CLAUSE__CILK_FOR_COUNT_:
1812 case OMP_CLAUSE_NUM_GANGS:
1813 case OMP_CLAUSE_NUM_WORKERS:
1814 case OMP_CLAUSE_VECTOR_LENGTH:
1815 if (ctx->outer)
1816 scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1817 break;
1819 case OMP_CLAUSE_TO:
1820 case OMP_CLAUSE_FROM:
1821 case OMP_CLAUSE_MAP:
1822 if (ctx->outer)
1823 scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1824 decl = OMP_CLAUSE_DECL (c);
1825 /* Global variables with "omp declare target" attribute
1826 don't need to be copied, the receiver side will use them
1827 directly. */
1828 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1829 && DECL_P (decl)
1830 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1831 && varpool_node::get_create (decl)->offloadable)
1832 break;
1833 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1834 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER)
1836 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1837 not offloaded; there is nothing to map for those. */
1838 if (!is_gimple_omp_offloaded (ctx->stmt)
1839 && !POINTER_TYPE_P (TREE_TYPE (decl))
1840 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
1841 break;
1843 if (DECL_P (decl))
1845 if (DECL_SIZE (decl)
1846 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1848 tree decl2 = DECL_VALUE_EXPR (decl);
1849 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
1850 decl2 = TREE_OPERAND (decl2, 0);
1851 gcc_assert (DECL_P (decl2));
1852 install_var_field (decl2, true, 3, ctx);
1853 install_var_local (decl2, ctx);
1854 install_var_local (decl, ctx);
1856 else
1858 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1859 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1860 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1861 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1862 install_var_field (decl, true, 7, ctx);
1863 else
1864 install_var_field (decl, true, 3, ctx);
1865 if (is_gimple_omp_offloaded (ctx->stmt))
1866 install_var_local (decl, ctx);
1869 else
1871 tree base = get_base_address (decl);
1872 tree nc = OMP_CLAUSE_CHAIN (c);
1873 if (DECL_P (base)
1874 && nc != NULL_TREE
1875 && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1876 && OMP_CLAUSE_DECL (nc) == base
1877 && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER
1878 && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1880 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1881 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1883 else
1885 if (ctx->outer)
1887 scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer);
1888 decl = OMP_CLAUSE_DECL (c);
1890 gcc_assert (!splay_tree_lookup (ctx->field_map,
1891 (splay_tree_key) decl));
1892 tree field
1893 = build_decl (OMP_CLAUSE_LOCATION (c),
1894 FIELD_DECL, NULL_TREE, ptr_type_node);
1895 DECL_ALIGN (field) = TYPE_ALIGN (ptr_type_node);
1896 insert_field_into_struct (ctx->record_type, field);
1897 splay_tree_insert (ctx->field_map, (splay_tree_key) decl,
1898 (splay_tree_value) field);
1901 break;
1903 case OMP_CLAUSE_NOWAIT:
1904 case OMP_CLAUSE_ORDERED:
1905 case OMP_CLAUSE_COLLAPSE:
1906 case OMP_CLAUSE_UNTIED:
1907 case OMP_CLAUSE_MERGEABLE:
1908 case OMP_CLAUSE_PROC_BIND:
1909 case OMP_CLAUSE_SAFELEN:
1910 case OMP_CLAUSE_ASYNC:
1911 case OMP_CLAUSE_WAIT:
1912 case OMP_CLAUSE_GANG:
1913 case OMP_CLAUSE_WORKER:
1914 case OMP_CLAUSE_VECTOR:
1915 break;
1917 case OMP_CLAUSE_ALIGNED:
1918 decl = OMP_CLAUSE_DECL (c);
1919 if (is_global_var (decl)
1920 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1921 install_var_local (decl, ctx);
1922 break;
1924 case OMP_CLAUSE_DEVICE_RESIDENT:
1925 case OMP_CLAUSE_USE_DEVICE:
1926 case OMP_CLAUSE__CACHE_:
1927 case OMP_CLAUSE_INDEPENDENT:
1928 case OMP_CLAUSE_AUTO:
1929 case OMP_CLAUSE_SEQ:
1930 sorry ("Clause not supported yet");
1931 break;
1933 default:
1934 gcc_unreachable ();
1938 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1940 switch (OMP_CLAUSE_CODE (c))
1942 case OMP_CLAUSE_LASTPRIVATE:
1943 /* Let the corresponding firstprivate clause create
1944 the variable. */
1945 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1946 scan_array_reductions = true;
1947 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1948 break;
1949 /* FALLTHRU */
1951 case OMP_CLAUSE_FIRSTPRIVATE:
1952 if (is_gimple_omp_oacc (ctx->stmt))
1954 sorry ("clause not supported yet");
1955 break;
1957 /* FALLTHRU */
1958 case OMP_CLAUSE_PRIVATE:
1959 case OMP_CLAUSE_REDUCTION:
1960 case OMP_CLAUSE_LINEAR:
1961 decl = OMP_CLAUSE_DECL (c);
1962 if (is_variable_sized (decl))
1963 install_var_local (decl, ctx);
1964 fixup_remapped_decl (decl, ctx,
1965 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1966 && OMP_CLAUSE_PRIVATE_DEBUG (c));
1967 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1968 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1969 scan_array_reductions = true;
1970 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
1971 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
1972 scan_array_reductions = true;
1973 break;
1975 case OMP_CLAUSE_SHARED:
1976 /* Ignore shared directives in teams construct. */
1977 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1978 break;
1979 decl = OMP_CLAUSE_DECL (c);
1980 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1981 fixup_remapped_decl (decl, ctx, false);
1982 break;
1984 case OMP_CLAUSE_MAP:
1985 if (!is_gimple_omp_offloaded (ctx->stmt))
1986 break;
1987 decl = OMP_CLAUSE_DECL (c);
1988 if (DECL_P (decl)
1989 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1990 && varpool_node::get_create (decl)->offloadable)
1991 break;
1992 if (DECL_P (decl))
1994 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
1995 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
1996 && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
1998 tree new_decl = lookup_decl (decl, ctx);
1999 TREE_TYPE (new_decl)
2000 = remap_type (TREE_TYPE (decl), &ctx->cb);
2002 else if (DECL_SIZE (decl)
2003 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
2005 tree decl2 = DECL_VALUE_EXPR (decl);
2006 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
2007 decl2 = TREE_OPERAND (decl2, 0);
2008 gcc_assert (DECL_P (decl2));
2009 fixup_remapped_decl (decl2, ctx, false);
2010 fixup_remapped_decl (decl, ctx, true);
2012 else
2013 fixup_remapped_decl (decl, ctx, false);
2015 break;
2017 case OMP_CLAUSE_COPYPRIVATE:
2018 case OMP_CLAUSE_COPYIN:
2019 case OMP_CLAUSE_DEFAULT:
2020 case OMP_CLAUSE_IF:
2021 case OMP_CLAUSE_NUM_THREADS:
2022 case OMP_CLAUSE_NUM_TEAMS:
2023 case OMP_CLAUSE_THREAD_LIMIT:
2024 case OMP_CLAUSE_DEVICE:
2025 case OMP_CLAUSE_SCHEDULE:
2026 case OMP_CLAUSE_DIST_SCHEDULE:
2027 case OMP_CLAUSE_NOWAIT:
2028 case OMP_CLAUSE_ORDERED:
2029 case OMP_CLAUSE_COLLAPSE:
2030 case OMP_CLAUSE_UNTIED:
2031 case OMP_CLAUSE_FINAL:
2032 case OMP_CLAUSE_MERGEABLE:
2033 case OMP_CLAUSE_PROC_BIND:
2034 case OMP_CLAUSE_SAFELEN:
2035 case OMP_CLAUSE_ALIGNED:
2036 case OMP_CLAUSE_DEPEND:
2037 case OMP_CLAUSE__LOOPTEMP_:
2038 case OMP_CLAUSE_TO:
2039 case OMP_CLAUSE_FROM:
2040 case OMP_CLAUSE__CILK_FOR_COUNT_:
2041 case OMP_CLAUSE_ASYNC:
2042 case OMP_CLAUSE_WAIT:
2043 case OMP_CLAUSE_NUM_GANGS:
2044 case OMP_CLAUSE_NUM_WORKERS:
2045 case OMP_CLAUSE_VECTOR_LENGTH:
2046 case OMP_CLAUSE_GANG:
2047 case OMP_CLAUSE_WORKER:
2048 case OMP_CLAUSE_VECTOR:
2049 break;
2051 case OMP_CLAUSE_DEVICE_RESIDENT:
2052 case OMP_CLAUSE_USE_DEVICE:
2053 case OMP_CLAUSE__CACHE_:
2054 case OMP_CLAUSE_INDEPENDENT:
2055 case OMP_CLAUSE_AUTO:
2056 case OMP_CLAUSE_SEQ:
2057 sorry ("Clause not supported yet");
2058 break;
2060 default:
2061 gcc_unreachable ();
2065 gcc_checking_assert (!scan_array_reductions
2066 || !is_gimple_omp_oacc (ctx->stmt));
2067 if (scan_array_reductions)
2068 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2069 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
2070 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2072 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx);
2073 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
2075 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
2076 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2077 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
2078 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2079 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2080 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
2083 /* Create a new name for omp child function. Returns an identifier. If
2084 IS_CILK_FOR is true then the suffix for the child function is
2085 "_cilk_for_fn." */
2087 static tree
2088 create_omp_child_function_name (bool task_copy, bool is_cilk_for)
2090 if (is_cilk_for)
2091 return clone_function_name (current_function_decl, "_cilk_for_fn");
2092 return clone_function_name (current_function_decl,
2093 task_copy ? "_omp_cpyfn" : "_omp_fn");
2096 /* Returns the type of the induction variable for the child function for
2097 _Cilk_for and the types for _high and _low variables based on TYPE. */
2099 static tree
2100 cilk_for_check_loop_diff_type (tree type)
2102 if (TYPE_PRECISION (type) <= TYPE_PRECISION (uint32_type_node))
2104 if (TYPE_UNSIGNED (type))
2105 return uint32_type_node;
2106 else
2107 return integer_type_node;
2109 else
2111 if (TYPE_UNSIGNED (type))
2112 return uint64_type_node;
2113 else
2114 return long_long_integer_type_node;
2118 /* Build a decl for the omp child function. It'll not contain a body
2119 yet, just the bare decl. */
2121 static void
2122 create_omp_child_function (omp_context *ctx, bool task_copy)
2124 tree decl, type, name, t;
2126 tree cilk_for_count
2127 = (flag_cilkplus && gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2128 ? find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
2129 OMP_CLAUSE__CILK_FOR_COUNT_) : NULL_TREE;
2130 tree cilk_var_type = NULL_TREE;
2132 name = create_omp_child_function_name (task_copy,
2133 cilk_for_count != NULL_TREE);
2134 if (task_copy)
2135 type = build_function_type_list (void_type_node, ptr_type_node,
2136 ptr_type_node, NULL_TREE);
2137 else if (cilk_for_count)
2139 type = TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count, 0));
2140 cilk_var_type = cilk_for_check_loop_diff_type (type);
2141 type = build_function_type_list (void_type_node, ptr_type_node,
2142 cilk_var_type, cilk_var_type, NULL_TREE);
2144 else
2145 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
2147 decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type);
2149 gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt)
2150 || !task_copy);
2151 if (!task_copy)
2152 ctx->cb.dst_fn = decl;
2153 else
2154 gimple_omp_task_set_copy_fn (ctx->stmt, decl);
2156 TREE_STATIC (decl) = 1;
2157 TREE_USED (decl) = 1;
2158 DECL_ARTIFICIAL (decl) = 1;
2159 DECL_IGNORED_P (decl) = 0;
2160 TREE_PUBLIC (decl) = 0;
2161 DECL_UNINLINABLE (decl) = 1;
2162 DECL_EXTERNAL (decl) = 0;
2163 DECL_CONTEXT (decl) = NULL_TREE;
2164 DECL_INITIAL (decl) = make_node (BLOCK);
2165 if (cgraph_node::get (current_function_decl)->offloadable)
2166 cgraph_node::get_create (decl)->offloadable = 1;
2167 else
2169 omp_context *octx;
2170 for (octx = ctx; octx; octx = octx->outer)
2171 if (is_gimple_omp_offloaded (octx->stmt))
2173 cgraph_node::get_create (decl)->offloadable = 1;
2174 #ifdef ENABLE_OFFLOADING
2175 g->have_offload = true;
2176 #endif
2177 break;
2181 if (cgraph_node::get_create (decl)->offloadable
2182 && !lookup_attribute ("omp declare target",
2183 DECL_ATTRIBUTES (current_function_decl)))
2184 DECL_ATTRIBUTES (decl)
2185 = tree_cons (get_identifier ("omp target entrypoint"),
2186 NULL_TREE, DECL_ATTRIBUTES (decl));
2188 t = build_decl (DECL_SOURCE_LOCATION (decl),
2189 RESULT_DECL, NULL_TREE, void_type_node);
2190 DECL_ARTIFICIAL (t) = 1;
2191 DECL_IGNORED_P (t) = 1;
2192 DECL_CONTEXT (t) = decl;
2193 DECL_RESULT (decl) = t;
2195 /* _Cilk_for's child function requires two extra parameters called
2196 __low and __high that are set the by Cilk runtime when it calls this
2197 function. */
2198 if (cilk_for_count)
2200 t = build_decl (DECL_SOURCE_LOCATION (decl),
2201 PARM_DECL, get_identifier ("__high"), cilk_var_type);
2202 DECL_ARTIFICIAL (t) = 1;
2203 DECL_NAMELESS (t) = 1;
2204 DECL_ARG_TYPE (t) = ptr_type_node;
2205 DECL_CONTEXT (t) = current_function_decl;
2206 TREE_USED (t) = 1;
2207 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2208 DECL_ARGUMENTS (decl) = t;
2210 t = build_decl (DECL_SOURCE_LOCATION (decl),
2211 PARM_DECL, get_identifier ("__low"), cilk_var_type);
2212 DECL_ARTIFICIAL (t) = 1;
2213 DECL_NAMELESS (t) = 1;
2214 DECL_ARG_TYPE (t) = ptr_type_node;
2215 DECL_CONTEXT (t) = current_function_decl;
2216 TREE_USED (t) = 1;
2217 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2218 DECL_ARGUMENTS (decl) = t;
2221 tree data_name = get_identifier (".omp_data_i");
2222 t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name,
2223 ptr_type_node);
2224 DECL_ARTIFICIAL (t) = 1;
2225 DECL_NAMELESS (t) = 1;
2226 DECL_ARG_TYPE (t) = ptr_type_node;
2227 DECL_CONTEXT (t) = current_function_decl;
2228 TREE_USED (t) = 1;
2229 if (cilk_for_count)
2230 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2231 DECL_ARGUMENTS (decl) = t;
2232 if (!task_copy)
2233 ctx->receiver_decl = t;
2234 else
2236 t = build_decl (DECL_SOURCE_LOCATION (decl),
2237 PARM_DECL, get_identifier (".omp_data_o"),
2238 ptr_type_node);
2239 DECL_ARTIFICIAL (t) = 1;
2240 DECL_NAMELESS (t) = 1;
2241 DECL_ARG_TYPE (t) = ptr_type_node;
2242 DECL_CONTEXT (t) = current_function_decl;
2243 TREE_USED (t) = 1;
2244 TREE_ADDRESSABLE (t) = 1;
2245 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2246 DECL_ARGUMENTS (decl) = t;
2249 /* Allocate memory for the function structure. The call to
2250 allocate_struct_function clobbers CFUN, so we need to restore
2251 it afterward. */
2252 push_struct_function (decl);
2253 cfun->function_end_locus = gimple_location (ctx->stmt);
2254 pop_cfun ();
2257 /* Callback for walk_gimple_seq. Check if combined parallel
2258 contains gimple_omp_for_combined_into_p OMP_FOR. */
2260 static tree
2261 find_combined_for (gimple_stmt_iterator *gsi_p,
2262 bool *handled_ops_p,
2263 struct walk_stmt_info *wi)
2265 gimple stmt = gsi_stmt (*gsi_p);
2267 *handled_ops_p = true;
2268 switch (gimple_code (stmt))
2270 WALK_SUBSTMTS;
2272 case GIMPLE_OMP_FOR:
2273 if (gimple_omp_for_combined_into_p (stmt)
2274 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
2276 wi->info = stmt;
2277 return integer_zero_node;
2279 break;
2280 default:
2281 break;
2283 return NULL;
2286 /* Scan an OpenMP parallel directive. */
2288 static void
2289 scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2291 omp_context *ctx;
2292 tree name;
2293 gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi));
2295 /* Ignore parallel directives with empty bodies, unless there
2296 are copyin clauses. */
2297 if (optimize > 0
2298 && empty_body_p (gimple_omp_body (stmt))
2299 && find_omp_clause (gimple_omp_parallel_clauses (stmt),
2300 OMP_CLAUSE_COPYIN) == NULL)
2302 gsi_replace (gsi, gimple_build_nop (), false);
2303 return;
2306 if (gimple_omp_parallel_combined_p (stmt))
2308 struct walk_stmt_info wi;
2310 memset (&wi, 0, sizeof (wi));
2311 wi.val_only = true;
2312 walk_gimple_seq (gimple_omp_body (stmt),
2313 find_combined_for, NULL, &wi);
2314 if (wi.info)
2316 gomp_for *for_stmt = as_a <gomp_for *> ((gimple) wi.info);
2317 struct omp_for_data fd;
2318 extract_omp_for_data (for_stmt, &fd, NULL);
2319 /* We need two temporaries with fd.loop.v type (istart/iend)
2320 and then (fd.collapse - 1) temporaries with the same
2321 type for count2 ... countN-1 vars if not constant. */
2322 size_t count = 2, i;
2323 tree type = fd.iter_type;
2324 if (fd.collapse > 1
2325 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
2326 count += fd.collapse - 1;
2327 for (i = 0; i < count; i++)
2329 tree temp = create_tmp_var (type);
2330 tree c = build_omp_clause (UNKNOWN_LOCATION,
2331 OMP_CLAUSE__LOOPTEMP_);
2332 insert_decl_map (&outer_ctx->cb, temp, temp);
2333 OMP_CLAUSE_DECL (c) = temp;
2334 OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
2335 gimple_omp_parallel_set_clauses (stmt, c);
2340 ctx = new_omp_context (stmt, outer_ctx);
2341 taskreg_contexts.safe_push (ctx);
2342 if (taskreg_nesting_level > 1)
2343 ctx->is_nested = true;
2344 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2345 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2346 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2347 name = create_tmp_var_name (".omp_data_s");
2348 name = build_decl (gimple_location (stmt),
2349 TYPE_DECL, name, ctx->record_type);
2350 DECL_ARTIFICIAL (name) = 1;
2351 DECL_NAMELESS (name) = 1;
2352 TYPE_NAME (ctx->record_type) = name;
2353 create_omp_child_function (ctx, false);
2354 gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2356 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2357 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2359 if (TYPE_FIELDS (ctx->record_type) == NULL)
2360 ctx->record_type = ctx->receiver_decl = NULL;
2363 /* Scan an OpenMP task directive. */
2365 static void
2366 scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2368 omp_context *ctx;
2369 tree name, t;
2370 gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi));
2372 /* Ignore task directives with empty bodies. */
2373 if (optimize > 0
2374 && empty_body_p (gimple_omp_body (stmt)))
2376 gsi_replace (gsi, gimple_build_nop (), false);
2377 return;
2380 ctx = new_omp_context (stmt, outer_ctx);
2381 taskreg_contexts.safe_push (ctx);
2382 if (taskreg_nesting_level > 1)
2383 ctx->is_nested = true;
2384 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2385 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2386 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2387 name = create_tmp_var_name (".omp_data_s");
2388 name = build_decl (gimple_location (stmt),
2389 TYPE_DECL, name, ctx->record_type);
2390 DECL_ARTIFICIAL (name) = 1;
2391 DECL_NAMELESS (name) = 1;
2392 TYPE_NAME (ctx->record_type) = name;
2393 create_omp_child_function (ctx, false);
2394 gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2396 scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2398 if (ctx->srecord_type)
2400 name = create_tmp_var_name (".omp_data_a");
2401 name = build_decl (gimple_location (stmt),
2402 TYPE_DECL, name, ctx->srecord_type);
2403 DECL_ARTIFICIAL (name) = 1;
2404 DECL_NAMELESS (name) = 1;
2405 TYPE_NAME (ctx->srecord_type) = name;
2406 create_omp_child_function (ctx, true);
2409 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2411 if (TYPE_FIELDS (ctx->record_type) == NULL)
2413 ctx->record_type = ctx->receiver_decl = NULL;
2414 t = build_int_cst (long_integer_type_node, 0);
2415 gimple_omp_task_set_arg_size (stmt, t);
2416 t = build_int_cst (long_integer_type_node, 1);
2417 gimple_omp_task_set_arg_align (stmt, t);
2422 /* If any decls have been made addressable during scan_omp,
2423 adjust their fields if needed, and layout record types
2424 of parallel/task constructs. */
2426 static void
2427 finish_taskreg_scan (omp_context *ctx)
2429 if (ctx->record_type == NULL_TREE)
2430 return;
2432 /* If any task_shared_vars were needed, verify all
2433 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2434 statements if use_pointer_for_field hasn't changed
2435 because of that. If it did, update field types now. */
2436 if (task_shared_vars)
2438 tree c;
2440 for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2441 c; c = OMP_CLAUSE_CHAIN (c))
2442 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED)
2444 tree decl = OMP_CLAUSE_DECL (c);
2446 /* Global variables don't need to be copied,
2447 the receiver side will use them directly. */
2448 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2449 continue;
2450 if (!bitmap_bit_p (task_shared_vars, DECL_UID (decl))
2451 || !use_pointer_for_field (decl, ctx))
2452 continue;
2453 tree field = lookup_field (decl, ctx);
2454 if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE
2455 && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl))
2456 continue;
2457 TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl));
2458 TREE_THIS_VOLATILE (field) = 0;
2459 DECL_USER_ALIGN (field) = 0;
2460 DECL_ALIGN (field) = TYPE_ALIGN (TREE_TYPE (field));
2461 if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field))
2462 TYPE_ALIGN (ctx->record_type) = DECL_ALIGN (field);
2463 if (ctx->srecord_type)
2465 tree sfield = lookup_sfield (decl, ctx);
2466 TREE_TYPE (sfield) = TREE_TYPE (field);
2467 TREE_THIS_VOLATILE (sfield) = 0;
2468 DECL_USER_ALIGN (sfield) = 0;
2469 DECL_ALIGN (sfield) = DECL_ALIGN (field);
2470 if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield))
2471 TYPE_ALIGN (ctx->srecord_type) = DECL_ALIGN (sfield);
2476 if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2478 layout_type (ctx->record_type);
2479 fixup_child_record_type (ctx);
2481 else
2483 location_t loc = gimple_location (ctx->stmt);
2484 tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2485 /* Move VLA fields to the end. */
2486 p = &TYPE_FIELDS (ctx->record_type);
2487 while (*p)
2488 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2489 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2491 *q = *p;
2492 *p = TREE_CHAIN (*p);
2493 TREE_CHAIN (*q) = NULL_TREE;
2494 q = &TREE_CHAIN (*q);
2496 else
2497 p = &DECL_CHAIN (*p);
2498 *p = vla_fields;
2499 layout_type (ctx->record_type);
2500 fixup_child_record_type (ctx);
2501 if (ctx->srecord_type)
2502 layout_type (ctx->srecord_type);
2503 tree t = fold_convert_loc (loc, long_integer_type_node,
2504 TYPE_SIZE_UNIT (ctx->record_type));
2505 gimple_omp_task_set_arg_size (ctx->stmt, t);
2506 t = build_int_cst (long_integer_type_node,
2507 TYPE_ALIGN_UNIT (ctx->record_type));
2508 gimple_omp_task_set_arg_align (ctx->stmt, t);
2513 static omp_context *
2514 enclosing_target_ctx (omp_context *ctx)
2516 while (ctx != NULL
2517 && gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
2518 ctx = ctx->outer;
2519 gcc_assert (ctx != NULL);
2520 return ctx;
2523 static bool
2524 oacc_loop_or_target_p (gimple stmt)
2526 enum gimple_code outer_type = gimple_code (stmt);
2527 return ((outer_type == GIMPLE_OMP_TARGET
2528 && ((gimple_omp_target_kind (stmt)
2529 == GF_OMP_TARGET_KIND_OACC_PARALLEL)
2530 || (gimple_omp_target_kind (stmt)
2531 == GF_OMP_TARGET_KIND_OACC_KERNELS)))
2532 || (outer_type == GIMPLE_OMP_FOR
2533 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP));
2536 /* Scan a GIMPLE_OMP_FOR. */
2538 static void
2539 scan_omp_for (gomp_for *stmt, omp_context *outer_ctx)
2541 enum gimple_code outer_type = GIMPLE_ERROR_MARK;
2542 omp_context *ctx;
2543 size_t i;
2544 tree clauses = gimple_omp_for_clauses (stmt);
2546 if (outer_ctx)
2547 outer_type = gimple_code (outer_ctx->stmt);
2549 ctx = new_omp_context (stmt, outer_ctx);
2551 if (is_gimple_omp_oacc (stmt))
2553 if (outer_ctx && outer_type == GIMPLE_OMP_FOR)
2554 ctx->gwv_this = outer_ctx->gwv_this;
2555 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2557 int val;
2558 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_GANG)
2559 val = MASK_GANG;
2560 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WORKER)
2561 val = MASK_WORKER;
2562 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR)
2563 val = MASK_VECTOR;
2564 else
2565 continue;
2566 ctx->gwv_this |= val;
2567 if (!outer_ctx)
2569 /* Skip; not nested inside a region. */
2570 continue;
2572 if (!oacc_loop_or_target_p (outer_ctx->stmt))
2574 /* Skip; not nested inside an OpenACC region. */
2575 continue;
2577 if (outer_type == GIMPLE_OMP_FOR)
2578 outer_ctx->gwv_below |= val;
2579 if (OMP_CLAUSE_OPERAND (c, 0) != NULL_TREE)
2581 omp_context *enclosing = enclosing_target_ctx (outer_ctx);
2582 if (gimple_omp_target_kind (enclosing->stmt)
2583 == GF_OMP_TARGET_KIND_OACC_PARALLEL)
2584 error_at (gimple_location (stmt),
2585 "no arguments allowed to gang, worker and vector clauses inside parallel");
2590 scan_sharing_clauses (clauses, ctx);
2592 scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
2593 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
2595 scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
2596 scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
2597 scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
2598 scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
2600 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2602 if (is_gimple_omp_oacc (stmt))
2604 if (ctx->gwv_this & ctx->gwv_below)
2605 error_at (gimple_location (stmt),
2606 "gang, worker and vector may occur only once in a loop nest");
2607 else if (ctx->gwv_below != 0
2608 && ctx->gwv_this > ctx->gwv_below)
2609 error_at (gimple_location (stmt),
2610 "gang, worker and vector must occur in this order in a loop nest");
2611 if (outer_ctx && outer_type == GIMPLE_OMP_FOR)
2612 outer_ctx->gwv_below |= ctx->gwv_below;
2616 /* Scan an OpenMP sections directive. */
2618 static void
2619 scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx)
2621 omp_context *ctx;
2623 ctx = new_omp_context (stmt, outer_ctx);
2624 scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
2625 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2628 /* Scan an OpenMP single directive. */
2630 static void
2631 scan_omp_single (gomp_single *stmt, omp_context *outer_ctx)
2633 omp_context *ctx;
2634 tree name;
2636 ctx = new_omp_context (stmt, outer_ctx);
2637 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2638 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2639 name = create_tmp_var_name (".omp_copy_s");
2640 name = build_decl (gimple_location (stmt),
2641 TYPE_DECL, name, ctx->record_type);
2642 TYPE_NAME (ctx->record_type) = name;
2644 scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
2645 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2647 if (TYPE_FIELDS (ctx->record_type) == NULL)
2648 ctx->record_type = NULL;
2649 else
2650 layout_type (ctx->record_type);
2653 /* Scan a GIMPLE_OMP_TARGET. */
2655 static void
2656 scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
2658 omp_context *ctx;
2659 tree name;
2660 bool offloaded = is_gimple_omp_offloaded (stmt);
2661 tree clauses = gimple_omp_target_clauses (stmt);
2663 ctx = new_omp_context (stmt, outer_ctx);
2664 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2665 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2666 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2667 name = create_tmp_var_name (".omp_data_t");
2668 name = build_decl (gimple_location (stmt),
2669 TYPE_DECL, name, ctx->record_type);
2670 DECL_ARTIFICIAL (name) = 1;
2671 DECL_NAMELESS (name) = 1;
2672 TYPE_NAME (ctx->record_type) = name;
2673 if (offloaded)
2675 if (is_gimple_omp_oacc (stmt))
2676 ctx->reduction_map = splay_tree_new (splay_tree_compare_pointers,
2677 0, 0);
2679 create_omp_child_function (ctx, false);
2680 gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
2683 if (is_gimple_omp_oacc (stmt))
2685 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2687 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_GANGS)
2688 ctx->gwv_this |= MASK_GANG;
2689 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_WORKERS)
2690 ctx->gwv_this |= MASK_WORKER;
2691 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR_LENGTH)
2692 ctx->gwv_this |= MASK_VECTOR;
2696 scan_sharing_clauses (clauses, ctx);
2697 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2699 if (TYPE_FIELDS (ctx->record_type) == NULL)
2700 ctx->record_type = ctx->receiver_decl = NULL;
2701 else
2703 TYPE_FIELDS (ctx->record_type)
2704 = nreverse (TYPE_FIELDS (ctx->record_type));
2705 #ifdef ENABLE_CHECKING
2706 tree field;
2707 unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
2708 for (field = TYPE_FIELDS (ctx->record_type);
2709 field;
2710 field = DECL_CHAIN (field))
2711 gcc_assert (DECL_ALIGN (field) == align);
2712 #endif
2713 layout_type (ctx->record_type);
2714 if (offloaded)
2715 fixup_child_record_type (ctx);
2719 /* Scan an OpenMP teams directive. */
2721 static void
2722 scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
2724 omp_context *ctx = new_omp_context (stmt, outer_ctx);
2725 scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
2726 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2729 /* Check nesting restrictions. */
2730 static bool
2731 check_omp_nesting_restrictions (gimple stmt, omp_context *ctx)
2733 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2734 inside an OpenACC CTX. */
2735 if (!(is_gimple_omp (stmt)
2736 && is_gimple_omp_oacc (stmt)))
2738 for (omp_context *ctx_ = ctx; ctx_ != NULL; ctx_ = ctx_->outer)
2739 if (is_gimple_omp (ctx_->stmt)
2740 && is_gimple_omp_oacc (ctx_->stmt))
2742 error_at (gimple_location (stmt),
2743 "non-OpenACC construct inside of OpenACC region");
2744 return false;
2748 if (ctx != NULL)
2750 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
2751 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
2753 error_at (gimple_location (stmt),
2754 "OpenMP constructs may not be nested inside simd region");
2755 return false;
2757 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2759 if ((gimple_code (stmt) != GIMPLE_OMP_FOR
2760 || (gimple_omp_for_kind (stmt)
2761 != GF_OMP_FOR_KIND_DISTRIBUTE))
2762 && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
2764 error_at (gimple_location (stmt),
2765 "only distribute or parallel constructs are allowed to "
2766 "be closely nested inside teams construct");
2767 return false;
2771 switch (gimple_code (stmt))
2773 case GIMPLE_OMP_FOR:
2774 if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_SIMD)
2775 return true;
2776 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
2778 if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
2780 error_at (gimple_location (stmt),
2781 "distribute construct must be closely nested inside "
2782 "teams construct");
2783 return false;
2785 return true;
2787 /* FALLTHRU */
2788 case GIMPLE_CALL:
2789 if (is_gimple_call (stmt)
2790 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2791 == BUILT_IN_GOMP_CANCEL
2792 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2793 == BUILT_IN_GOMP_CANCELLATION_POINT))
2795 const char *bad = NULL;
2796 const char *kind = NULL;
2797 if (ctx == NULL)
2799 error_at (gimple_location (stmt), "orphaned %qs construct",
2800 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2801 == BUILT_IN_GOMP_CANCEL
2802 ? "#pragma omp cancel"
2803 : "#pragma omp cancellation point");
2804 return false;
2806 switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
2807 ? tree_to_shwi (gimple_call_arg (stmt, 0))
2808 : 0)
2810 case 1:
2811 if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
2812 bad = "#pragma omp parallel";
2813 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2814 == BUILT_IN_GOMP_CANCEL
2815 && !integer_zerop (gimple_call_arg (stmt, 1)))
2816 ctx->cancellable = true;
2817 kind = "parallel";
2818 break;
2819 case 2:
2820 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
2821 || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
2822 bad = "#pragma omp for";
2823 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2824 == BUILT_IN_GOMP_CANCEL
2825 && !integer_zerop (gimple_call_arg (stmt, 1)))
2827 ctx->cancellable = true;
2828 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2829 OMP_CLAUSE_NOWAIT))
2830 warning_at (gimple_location (stmt), 0,
2831 "%<#pragma omp cancel for%> inside "
2832 "%<nowait%> for construct");
2833 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2834 OMP_CLAUSE_ORDERED))
2835 warning_at (gimple_location (stmt), 0,
2836 "%<#pragma omp cancel for%> inside "
2837 "%<ordered%> for construct");
2839 kind = "for";
2840 break;
2841 case 4:
2842 if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
2843 && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
2844 bad = "#pragma omp sections";
2845 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2846 == BUILT_IN_GOMP_CANCEL
2847 && !integer_zerop (gimple_call_arg (stmt, 1)))
2849 if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
2851 ctx->cancellable = true;
2852 if (find_omp_clause (gimple_omp_sections_clauses
2853 (ctx->stmt),
2854 OMP_CLAUSE_NOWAIT))
2855 warning_at (gimple_location (stmt), 0,
2856 "%<#pragma omp cancel sections%> inside "
2857 "%<nowait%> sections construct");
2859 else
2861 gcc_assert (ctx->outer
2862 && gimple_code (ctx->outer->stmt)
2863 == GIMPLE_OMP_SECTIONS);
2864 ctx->outer->cancellable = true;
2865 if (find_omp_clause (gimple_omp_sections_clauses
2866 (ctx->outer->stmt),
2867 OMP_CLAUSE_NOWAIT))
2868 warning_at (gimple_location (stmt), 0,
2869 "%<#pragma omp cancel sections%> inside "
2870 "%<nowait%> sections construct");
2873 kind = "sections";
2874 break;
2875 case 8:
2876 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK)
2877 bad = "#pragma omp task";
2878 else
2879 ctx->cancellable = true;
2880 kind = "taskgroup";
2881 break;
2882 default:
2883 error_at (gimple_location (stmt), "invalid arguments");
2884 return false;
2886 if (bad)
2888 error_at (gimple_location (stmt),
2889 "%<%s %s%> construct not closely nested inside of %qs",
2890 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2891 == BUILT_IN_GOMP_CANCEL
2892 ? "#pragma omp cancel"
2893 : "#pragma omp cancellation point", kind, bad);
2894 return false;
2897 /* FALLTHRU */
2898 case GIMPLE_OMP_SECTIONS:
2899 case GIMPLE_OMP_SINGLE:
2900 for (; ctx != NULL; ctx = ctx->outer)
2901 switch (gimple_code (ctx->stmt))
2903 case GIMPLE_OMP_FOR:
2904 case GIMPLE_OMP_SECTIONS:
2905 case GIMPLE_OMP_SINGLE:
2906 case GIMPLE_OMP_ORDERED:
2907 case GIMPLE_OMP_MASTER:
2908 case GIMPLE_OMP_TASK:
2909 case GIMPLE_OMP_CRITICAL:
2910 if (is_gimple_call (stmt))
2912 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2913 != BUILT_IN_GOMP_BARRIER)
2914 return true;
2915 error_at (gimple_location (stmt),
2916 "barrier region may not be closely nested inside "
2917 "of work-sharing, critical, ordered, master or "
2918 "explicit task region");
2919 return false;
2921 error_at (gimple_location (stmt),
2922 "work-sharing region may not be closely nested inside "
2923 "of work-sharing, critical, ordered, master or explicit "
2924 "task region");
2925 return false;
2926 case GIMPLE_OMP_PARALLEL:
2927 return true;
2928 default:
2929 break;
2931 break;
2932 case GIMPLE_OMP_MASTER:
2933 for (; ctx != NULL; ctx = ctx->outer)
2934 switch (gimple_code (ctx->stmt))
2936 case GIMPLE_OMP_FOR:
2937 case GIMPLE_OMP_SECTIONS:
2938 case GIMPLE_OMP_SINGLE:
2939 case GIMPLE_OMP_TASK:
2940 error_at (gimple_location (stmt),
2941 "master region may not be closely nested inside "
2942 "of work-sharing or explicit task region");
2943 return false;
2944 case GIMPLE_OMP_PARALLEL:
2945 return true;
2946 default:
2947 break;
2949 break;
2950 case GIMPLE_OMP_ORDERED:
2951 for (; ctx != NULL; ctx = ctx->outer)
2952 switch (gimple_code (ctx->stmt))
2954 case GIMPLE_OMP_CRITICAL:
2955 case GIMPLE_OMP_TASK:
2956 error_at (gimple_location (stmt),
2957 "ordered region may not be closely nested inside "
2958 "of critical or explicit task region");
2959 return false;
2960 case GIMPLE_OMP_FOR:
2961 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2962 OMP_CLAUSE_ORDERED) == NULL)
2964 error_at (gimple_location (stmt),
2965 "ordered region must be closely nested inside "
2966 "a loop region with an ordered clause");
2967 return false;
2969 return true;
2970 case GIMPLE_OMP_PARALLEL:
2971 error_at (gimple_location (stmt),
2972 "ordered region must be closely nested inside "
2973 "a loop region with an ordered clause");
2974 return false;
2975 default:
2976 break;
2978 break;
2979 case GIMPLE_OMP_CRITICAL:
2981 tree this_stmt_name
2982 = gimple_omp_critical_name (as_a <gomp_critical *> (stmt));
2983 for (; ctx != NULL; ctx = ctx->outer)
2984 if (gomp_critical *other_crit
2985 = dyn_cast <gomp_critical *> (ctx->stmt))
2986 if (this_stmt_name == gimple_omp_critical_name (other_crit))
2988 error_at (gimple_location (stmt),
2989 "critical region may not be nested inside a critical "
2990 "region with the same name");
2991 return false;
2994 break;
2995 case GIMPLE_OMP_TEAMS:
2996 if (ctx == NULL
2997 || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
2998 || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION)
3000 error_at (gimple_location (stmt),
3001 "teams construct not closely nested inside of target "
3002 "region");
3003 return false;
3005 break;
3006 case GIMPLE_OMP_TARGET:
3007 for (; ctx != NULL; ctx = ctx->outer)
3009 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET)
3011 if (is_gimple_omp (stmt)
3012 && is_gimple_omp_oacc (stmt)
3013 && is_gimple_omp (ctx->stmt))
3015 error_at (gimple_location (stmt),
3016 "OpenACC construct inside of non-OpenACC region");
3017 return false;
3019 continue;
3022 const char *stmt_name, *ctx_stmt_name;
3023 switch (gimple_omp_target_kind (stmt))
3025 case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break;
3026 case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break;
3027 case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break;
3028 case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break;
3029 case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break;
3030 case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break;
3031 case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break;
3032 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: stmt_name = "enter/exit data"; break;
3033 default: gcc_unreachable ();
3035 switch (gimple_omp_target_kind (ctx->stmt))
3037 case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break;
3038 case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break;
3039 case GF_OMP_TARGET_KIND_OACC_PARALLEL: ctx_stmt_name = "parallel"; break;
3040 case GF_OMP_TARGET_KIND_OACC_KERNELS: ctx_stmt_name = "kernels"; break;
3041 case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break;
3042 default: gcc_unreachable ();
3045 /* OpenACC/OpenMP mismatch? */
3046 if (is_gimple_omp_oacc (stmt)
3047 != is_gimple_omp_oacc (ctx->stmt))
3049 error_at (gimple_location (stmt),
3050 "%s %s construct inside of %s %s region",
3051 (is_gimple_omp_oacc (stmt)
3052 ? "OpenACC" : "OpenMP"), stmt_name,
3053 (is_gimple_omp_oacc (ctx->stmt)
3054 ? "OpenACC" : "OpenMP"), ctx_stmt_name);
3055 return false;
3057 if (is_gimple_omp_offloaded (ctx->stmt))
3059 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3060 if (is_gimple_omp_oacc (ctx->stmt))
3062 error_at (gimple_location (stmt),
3063 "%s construct inside of %s region",
3064 stmt_name, ctx_stmt_name);
3065 return false;
3067 else
3069 gcc_checking_assert (!is_gimple_omp_oacc (stmt));
3070 warning_at (gimple_location (stmt), 0,
3071 "%s construct inside of %s region",
3072 stmt_name, ctx_stmt_name);
3076 break;
3077 default:
3078 break;
3080 return true;
3084 /* Helper function scan_omp.
3086 Callback for walk_tree or operators in walk_gimple_stmt used to
3087 scan for OMP directives in TP. */
3089 static tree
3090 scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
3092 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
3093 omp_context *ctx = (omp_context *) wi->info;
3094 tree t = *tp;
3096 switch (TREE_CODE (t))
3098 case VAR_DECL:
3099 case PARM_DECL:
3100 case LABEL_DECL:
3101 case RESULT_DECL:
3102 if (ctx)
3103 *tp = remap_decl (t, &ctx->cb);
3104 break;
3106 default:
3107 if (ctx && TYPE_P (t))
3108 *tp = remap_type (t, &ctx->cb);
3109 else if (!DECL_P (t))
3111 *walk_subtrees = 1;
3112 if (ctx)
3114 tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
3115 if (tem != TREE_TYPE (t))
3117 if (TREE_CODE (t) == INTEGER_CST)
3118 *tp = wide_int_to_tree (tem, t);
3119 else
3120 TREE_TYPE (t) = tem;
3124 break;
3127 return NULL_TREE;
3130 /* Return true if FNDECL is a setjmp or a longjmp. */
3132 static bool
3133 setjmp_or_longjmp_p (const_tree fndecl)
3135 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
3136 && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP
3137 || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP))
3138 return true;
3140 tree declname = DECL_NAME (fndecl);
3141 if (!declname)
3142 return false;
3143 const char *name = IDENTIFIER_POINTER (declname);
3144 return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
3148 /* Helper function for scan_omp.
3150 Callback for walk_gimple_stmt used to scan for OMP directives in
3151 the current statement in GSI. */
3153 static tree
3154 scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
3155 struct walk_stmt_info *wi)
3157 gimple stmt = gsi_stmt (*gsi);
3158 omp_context *ctx = (omp_context *) wi->info;
3160 if (gimple_has_location (stmt))
3161 input_location = gimple_location (stmt);
3163 /* Check the nesting restrictions. */
3164 bool remove = false;
3165 if (is_gimple_omp (stmt))
3166 remove = !check_omp_nesting_restrictions (stmt, ctx);
3167 else if (is_gimple_call (stmt))
3169 tree fndecl = gimple_call_fndecl (stmt);
3170 if (fndecl)
3172 if (setjmp_or_longjmp_p (fndecl)
3173 && ctx
3174 && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3175 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
3177 remove = true;
3178 error_at (gimple_location (stmt),
3179 "setjmp/longjmp inside simd construct");
3181 else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
3182 switch (DECL_FUNCTION_CODE (fndecl))
3184 case BUILT_IN_GOMP_BARRIER:
3185 case BUILT_IN_GOMP_CANCEL:
3186 case BUILT_IN_GOMP_CANCELLATION_POINT:
3187 case BUILT_IN_GOMP_TASKYIELD:
3188 case BUILT_IN_GOMP_TASKWAIT:
3189 case BUILT_IN_GOMP_TASKGROUP_START:
3190 case BUILT_IN_GOMP_TASKGROUP_END:
3191 remove = !check_omp_nesting_restrictions (stmt, ctx);
3192 break;
3193 default:
3194 break;
3198 if (remove)
3200 stmt = gimple_build_nop ();
3201 gsi_replace (gsi, stmt, false);
3204 *handled_ops_p = true;
3206 switch (gimple_code (stmt))
3208 case GIMPLE_OMP_PARALLEL:
3209 taskreg_nesting_level++;
3210 scan_omp_parallel (gsi, ctx);
3211 taskreg_nesting_level--;
3212 break;
3214 case GIMPLE_OMP_TASK:
3215 taskreg_nesting_level++;
3216 scan_omp_task (gsi, ctx);
3217 taskreg_nesting_level--;
3218 break;
3220 case GIMPLE_OMP_FOR:
3221 scan_omp_for (as_a <gomp_for *> (stmt), ctx);
3222 break;
3224 case GIMPLE_OMP_SECTIONS:
3225 scan_omp_sections (as_a <gomp_sections *> (stmt), ctx);
3226 break;
3228 case GIMPLE_OMP_SINGLE:
3229 scan_omp_single (as_a <gomp_single *> (stmt), ctx);
3230 break;
3232 case GIMPLE_OMP_SECTION:
3233 case GIMPLE_OMP_MASTER:
3234 case GIMPLE_OMP_TASKGROUP:
3235 case GIMPLE_OMP_ORDERED:
3236 case GIMPLE_OMP_CRITICAL:
3237 ctx = new_omp_context (stmt, ctx);
3238 scan_omp (gimple_omp_body_ptr (stmt), ctx);
3239 break;
3241 case GIMPLE_OMP_TARGET:
3242 scan_omp_target (as_a <gomp_target *> (stmt), ctx);
3243 break;
3245 case GIMPLE_OMP_TEAMS:
3246 scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
3247 break;
3249 case GIMPLE_BIND:
3251 tree var;
3253 *handled_ops_p = false;
3254 if (ctx)
3255 for (var = gimple_bind_vars (as_a <gbind *> (stmt));
3256 var ;
3257 var = DECL_CHAIN (var))
3258 insert_decl_map (&ctx->cb, var, var);
3260 break;
3261 default:
3262 *handled_ops_p = false;
3263 break;
3266 return NULL_TREE;
3270 /* Scan all the statements starting at the current statement. CTX
3271 contains context information about the OMP directives and
3272 clauses found during the scan. */
3274 static void
3275 scan_omp (gimple_seq *body_p, omp_context *ctx)
3277 location_t saved_location;
3278 struct walk_stmt_info wi;
3280 memset (&wi, 0, sizeof (wi));
3281 wi.info = ctx;
3282 wi.want_locations = true;
3284 saved_location = input_location;
3285 walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
3286 input_location = saved_location;
3289 /* Re-gimplification and code generation routines. */
3291 /* Build a call to GOMP_barrier. */
3293 static gimple
3294 build_omp_barrier (tree lhs)
3296 tree fndecl = builtin_decl_explicit (lhs ? BUILT_IN_GOMP_BARRIER_CANCEL
3297 : BUILT_IN_GOMP_BARRIER);
3298 gcall *g = gimple_build_call (fndecl, 0);
3299 if (lhs)
3300 gimple_call_set_lhs (g, lhs);
3301 return g;
3304 /* If a context was created for STMT when it was scanned, return it. */
3306 static omp_context *
3307 maybe_lookup_ctx (gimple stmt)
3309 splay_tree_node n;
3310 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
3311 return n ? (omp_context *) n->value : NULL;
3315 /* Find the mapping for DECL in CTX or the immediately enclosing
3316 context that has a mapping for DECL.
3318 If CTX is a nested parallel directive, we may have to use the decl
3319 mappings created in CTX's parent context. Suppose that we have the
3320 following parallel nesting (variable UIDs showed for clarity):
3322 iD.1562 = 0;
3323 #omp parallel shared(iD.1562) -> outer parallel
3324 iD.1562 = iD.1562 + 1;
3326 #omp parallel shared (iD.1562) -> inner parallel
3327 iD.1562 = iD.1562 - 1;
3329 Each parallel structure will create a distinct .omp_data_s structure
3330 for copying iD.1562 in/out of the directive:
3332 outer parallel .omp_data_s.1.i -> iD.1562
3333 inner parallel .omp_data_s.2.i -> iD.1562
3335 A shared variable mapping will produce a copy-out operation before
3336 the parallel directive and a copy-in operation after it. So, in
3337 this case we would have:
3339 iD.1562 = 0;
3340 .omp_data_o.1.i = iD.1562;
3341 #omp parallel shared(iD.1562) -> outer parallel
3342 .omp_data_i.1 = &.omp_data_o.1
3343 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3345 .omp_data_o.2.i = iD.1562; -> **
3346 #omp parallel shared(iD.1562) -> inner parallel
3347 .omp_data_i.2 = &.omp_data_o.2
3348 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3351 ** This is a problem. The symbol iD.1562 cannot be referenced
3352 inside the body of the outer parallel region. But since we are
3353 emitting this copy operation while expanding the inner parallel
3354 directive, we need to access the CTX structure of the outer
3355 parallel directive to get the correct mapping:
3357 .omp_data_o.2.i = .omp_data_i.1->i
3359 Since there may be other workshare or parallel directives enclosing
3360 the parallel directive, it may be necessary to walk up the context
3361 parent chain. This is not a problem in general because nested
3362 parallelism happens only rarely. */
3364 static tree
3365 lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
3367 tree t;
3368 omp_context *up;
3370 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
3371 t = maybe_lookup_decl (decl, up);
3373 gcc_assert (!ctx->is_nested || t || is_global_var (decl));
3375 return t ? t : decl;
3379 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3380 in outer contexts. */
3382 static tree
3383 maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
3385 tree t = NULL;
3386 omp_context *up;
3388 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
3389 t = maybe_lookup_decl (decl, up);
3391 return t ? t : decl;
3395 /* Construct the initialization value for reduction CLAUSE. */
3397 tree
3398 omp_reduction_init (tree clause, tree type)
3400 location_t loc = OMP_CLAUSE_LOCATION (clause);
3401 switch (OMP_CLAUSE_REDUCTION_CODE (clause))
3403 case PLUS_EXPR:
3404 case MINUS_EXPR:
3405 case BIT_IOR_EXPR:
3406 case BIT_XOR_EXPR:
3407 case TRUTH_OR_EXPR:
3408 case TRUTH_ORIF_EXPR:
3409 case TRUTH_XOR_EXPR:
3410 case NE_EXPR:
3411 return build_zero_cst (type);
3413 case MULT_EXPR:
3414 case TRUTH_AND_EXPR:
3415 case TRUTH_ANDIF_EXPR:
3416 case EQ_EXPR:
3417 return fold_convert_loc (loc, type, integer_one_node);
3419 case BIT_AND_EXPR:
3420 return fold_convert_loc (loc, type, integer_minus_one_node);
3422 case MAX_EXPR:
3423 if (SCALAR_FLOAT_TYPE_P (type))
3425 REAL_VALUE_TYPE max, min;
3426 if (HONOR_INFINITIES (type))
3428 real_inf (&max);
3429 real_arithmetic (&min, NEGATE_EXPR, &max, NULL);
3431 else
3432 real_maxval (&min, 1, TYPE_MODE (type));
3433 return build_real (type, min);
3435 else
3437 gcc_assert (INTEGRAL_TYPE_P (type));
3438 return TYPE_MIN_VALUE (type);
3441 case MIN_EXPR:
3442 if (SCALAR_FLOAT_TYPE_P (type))
3444 REAL_VALUE_TYPE max;
3445 if (HONOR_INFINITIES (type))
3446 real_inf (&max);
3447 else
3448 real_maxval (&max, 0, TYPE_MODE (type));
3449 return build_real (type, max);
3451 else
3453 gcc_assert (INTEGRAL_TYPE_P (type));
3454 return TYPE_MAX_VALUE (type);
3457 default:
3458 gcc_unreachable ();
3462 /* Return alignment to be assumed for var in CLAUSE, which should be
3463 OMP_CLAUSE_ALIGNED. */
3465 static tree
3466 omp_clause_aligned_alignment (tree clause)
3468 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
3469 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
3471 /* Otherwise return implementation defined alignment. */
3472 unsigned int al = 1;
3473 machine_mode mode, vmode;
3474 int vs = targetm.vectorize.autovectorize_vector_sizes ();
3475 if (vs)
3476 vs = 1 << floor_log2 (vs);
3477 static enum mode_class classes[]
3478 = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
3479 for (int i = 0; i < 4; i += 2)
3480 for (mode = GET_CLASS_NARROWEST_MODE (classes[i]);
3481 mode != VOIDmode;
3482 mode = GET_MODE_WIDER_MODE (mode))
3484 vmode = targetm.vectorize.preferred_simd_mode (mode);
3485 if (GET_MODE_CLASS (vmode) != classes[i + 1])
3486 continue;
3487 while (vs
3488 && GET_MODE_SIZE (vmode) < vs
3489 && GET_MODE_2XWIDER_MODE (vmode) != VOIDmode)
3490 vmode = GET_MODE_2XWIDER_MODE (vmode);
3492 tree type = lang_hooks.types.type_for_mode (mode, 1);
3493 if (type == NULL_TREE || TYPE_MODE (type) != mode)
3494 continue;
3495 type = build_vector_type (type, GET_MODE_SIZE (vmode)
3496 / GET_MODE_SIZE (mode));
3497 if (TYPE_MODE (type) != vmode)
3498 continue;
3499 if (TYPE_ALIGN_UNIT (type) > al)
3500 al = TYPE_ALIGN_UNIT (type);
3502 return build_int_cst (integer_type_node, al);
3505 /* Return maximum possible vectorization factor for the target. */
3507 static int
3508 omp_max_vf (void)
3510 if (!optimize
3511 || optimize_debug
3512 || !flag_tree_loop_optimize
3513 || (!flag_tree_loop_vectorize
3514 && (global_options_set.x_flag_tree_loop_vectorize
3515 || global_options_set.x_flag_tree_vectorize)))
3516 return 1;
3518 int vs = targetm.vectorize.autovectorize_vector_sizes ();
3519 if (vs)
3521 vs = 1 << floor_log2 (vs);
3522 return vs;
3524 machine_mode vqimode = targetm.vectorize.preferred_simd_mode (QImode);
3525 if (GET_MODE_CLASS (vqimode) == MODE_VECTOR_INT)
3526 return GET_MODE_NUNITS (vqimode);
3527 return 1;
3530 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3531 privatization. */
3533 static bool
3534 lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, int &max_vf,
3535 tree &idx, tree &lane, tree &ivar, tree &lvar)
3537 if (max_vf == 0)
3539 max_vf = omp_max_vf ();
3540 if (max_vf > 1)
3542 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3543 OMP_CLAUSE_SAFELEN);
3544 if (c && TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) != INTEGER_CST)
3545 max_vf = 1;
3546 else if (c && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
3547 max_vf) == -1)
3548 max_vf = tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c));
3550 if (max_vf > 1)
3552 idx = create_tmp_var (unsigned_type_node);
3553 lane = create_tmp_var (unsigned_type_node);
3556 if (max_vf == 1)
3557 return false;
3559 tree atype = build_array_type_nelts (TREE_TYPE (new_var), max_vf);
3560 tree avar = create_tmp_var_raw (atype);
3561 if (TREE_ADDRESSABLE (new_var))
3562 TREE_ADDRESSABLE (avar) = 1;
3563 DECL_ATTRIBUTES (avar)
3564 = tree_cons (get_identifier ("omp simd array"), NULL,
3565 DECL_ATTRIBUTES (avar));
3566 gimple_add_tmp_var (avar);
3567 ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, idx,
3568 NULL_TREE, NULL_TREE);
3569 lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, lane,
3570 NULL_TREE, NULL_TREE);
3571 if (DECL_P (new_var))
3573 SET_DECL_VALUE_EXPR (new_var, lvar);
3574 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3576 return true;
3579 /* Helper function of lower_rec_input_clauses. For a reference
3580 in simd reduction, add an underlying variable it will reference. */
3582 static void
3583 handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
3585 tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
3586 if (TREE_CONSTANT (z))
3588 const char *name = NULL;
3589 if (DECL_NAME (new_vard))
3590 name = IDENTIFIER_POINTER (DECL_NAME (new_vard));
3592 z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)), name);
3593 gimple_add_tmp_var (z);
3594 TREE_ADDRESSABLE (z) = 1;
3595 z = build_fold_addr_expr_loc (loc, z);
3596 gimplify_assign (new_vard, z, ilist);
3600 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3601 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3602 private variables. Initialization statements go in ILIST, while calls
3603 to destructors go in DLIST. */
3605 static void
3606 lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
3607 omp_context *ctx, struct omp_for_data *fd)
3609 tree c, dtor, copyin_seq, x, ptr;
3610 bool copyin_by_ref = false;
3611 bool lastprivate_firstprivate = false;
3612 bool reduction_omp_orig_ref = false;
3613 int pass;
3614 bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3615 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD);
3616 int max_vf = 0;
3617 tree lane = NULL_TREE, idx = NULL_TREE;
3618 tree ivar = NULL_TREE, lvar = NULL_TREE;
3619 gimple_seq llist[2] = { NULL, NULL };
3621 copyin_seq = NULL;
3623 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3624 with data sharing clauses referencing variable sized vars. That
3625 is unnecessarily hard to support and very unlikely to result in
3626 vectorized code anyway. */
3627 if (is_simd)
3628 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3629 switch (OMP_CLAUSE_CODE (c))
3631 case OMP_CLAUSE_LINEAR:
3632 if (OMP_CLAUSE_LINEAR_ARRAY (c))
3633 max_vf = 1;
3634 /* FALLTHRU */
3635 case OMP_CLAUSE_REDUCTION:
3636 case OMP_CLAUSE_PRIVATE:
3637 case OMP_CLAUSE_FIRSTPRIVATE:
3638 case OMP_CLAUSE_LASTPRIVATE:
3639 if (is_variable_sized (OMP_CLAUSE_DECL (c)))
3640 max_vf = 1;
3641 break;
3642 default:
3643 continue;
3646 /* Do all the fixed sized types in the first pass, and the variable sized
3647 types in the second pass. This makes sure that the scalar arguments to
3648 the variable sized types are processed before we use them in the
3649 variable sized operations. */
3650 for (pass = 0; pass < 2; ++pass)
3652 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3654 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
3655 tree var, new_var;
3656 bool by_ref;
3657 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3659 switch (c_kind)
3661 case OMP_CLAUSE_PRIVATE:
3662 if (OMP_CLAUSE_PRIVATE_DEBUG (c))
3663 continue;
3664 break;
3665 case OMP_CLAUSE_SHARED:
3666 /* Ignore shared directives in teams construct. */
3667 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3668 continue;
3669 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
3671 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c)));
3672 continue;
3674 case OMP_CLAUSE_FIRSTPRIVATE:
3675 case OMP_CLAUSE_COPYIN:
3676 case OMP_CLAUSE_LINEAR:
3677 break;
3678 case OMP_CLAUSE_REDUCTION:
3679 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
3680 reduction_omp_orig_ref = true;
3681 break;
3682 case OMP_CLAUSE__LOOPTEMP_:
3683 /* Handle _looptemp_ clauses only on parallel. */
3684 if (fd)
3685 continue;
3686 break;
3687 case OMP_CLAUSE_LASTPRIVATE:
3688 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3690 lastprivate_firstprivate = true;
3691 if (pass != 0)
3692 continue;
3694 /* Even without corresponding firstprivate, if
3695 decl is Fortran allocatable, it needs outer var
3696 reference. */
3697 else if (pass == 0
3698 && lang_hooks.decls.omp_private_outer_ref
3699 (OMP_CLAUSE_DECL (c)))
3700 lastprivate_firstprivate = true;
3701 break;
3702 case OMP_CLAUSE_ALIGNED:
3703 if (pass == 0)
3704 continue;
3705 var = OMP_CLAUSE_DECL (c);
3706 if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
3707 && !is_global_var (var))
3709 new_var = maybe_lookup_decl (var, ctx);
3710 if (new_var == NULL_TREE)
3711 new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
3712 x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3713 x = build_call_expr_loc (clause_loc, x, 2, new_var,
3714 omp_clause_aligned_alignment (c));
3715 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3716 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
3717 gimplify_and_add (x, ilist);
3719 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
3720 && is_global_var (var))
3722 tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
3723 new_var = lookup_decl (var, ctx);
3724 t = maybe_lookup_decl_in_outer_ctx (var, ctx);
3725 t = build_fold_addr_expr_loc (clause_loc, t);
3726 t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3727 t = build_call_expr_loc (clause_loc, t2, 2, t,
3728 omp_clause_aligned_alignment (c));
3729 t = fold_convert_loc (clause_loc, ptype, t);
3730 x = create_tmp_var (ptype);
3731 t = build2 (MODIFY_EXPR, ptype, x, t);
3732 gimplify_and_add (t, ilist);
3733 t = build_simple_mem_ref_loc (clause_loc, x);
3734 SET_DECL_VALUE_EXPR (new_var, t);
3735 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3737 continue;
3738 default:
3739 continue;
3742 new_var = var = OMP_CLAUSE_DECL (c);
3743 if (c_kind != OMP_CLAUSE_COPYIN)
3744 new_var = lookup_decl (var, ctx);
3746 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
3748 if (pass != 0)
3749 continue;
3751 else if (is_variable_sized (var))
3753 /* For variable sized types, we need to allocate the
3754 actual storage here. Call alloca and store the
3755 result in the pointer decl that we created elsewhere. */
3756 if (pass == 0)
3757 continue;
3759 if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
3761 gcall *stmt;
3762 tree tmp, atmp;
3764 ptr = DECL_VALUE_EXPR (new_var);
3765 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
3766 ptr = TREE_OPERAND (ptr, 0);
3767 gcc_assert (DECL_P (ptr));
3768 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
3770 /* void *tmp = __builtin_alloca */
3771 atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3772 stmt = gimple_build_call (atmp, 1, x);
3773 tmp = create_tmp_var_raw (ptr_type_node);
3774 gimple_add_tmp_var (tmp);
3775 gimple_call_set_lhs (stmt, tmp);
3777 gimple_seq_add_stmt (ilist, stmt);
3779 x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
3780 gimplify_assign (ptr, x, ilist);
3783 else if (is_reference (var))
3785 /* For references that are being privatized for Fortran,
3786 allocate new backing storage for the new pointer
3787 variable. This allows us to avoid changing all the
3788 code that expects a pointer to something that expects
3789 a direct variable. */
3790 if (pass == 0)
3791 continue;
3793 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
3794 if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
3796 x = build_receiver_ref (var, false, ctx);
3797 x = build_fold_addr_expr_loc (clause_loc, x);
3799 else if (TREE_CONSTANT (x))
3801 /* For reduction in SIMD loop, defer adding the
3802 initialization of the reference, because if we decide
3803 to use SIMD array for it, the initilization could cause
3804 expansion ICE. */
3805 if (c_kind == OMP_CLAUSE_REDUCTION && is_simd)
3806 x = NULL_TREE;
3807 else
3809 const char *name = NULL;
3810 if (DECL_NAME (var))
3811 name = IDENTIFIER_POINTER (DECL_NAME (new_var));
3813 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
3814 name);
3815 gimple_add_tmp_var (x);
3816 TREE_ADDRESSABLE (x) = 1;
3817 x = build_fold_addr_expr_loc (clause_loc, x);
3820 else
3822 tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3823 x = build_call_expr_loc (clause_loc, atmp, 1, x);
3826 if (x)
3828 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3829 gimplify_assign (new_var, x, ilist);
3832 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3834 else if (c_kind == OMP_CLAUSE_REDUCTION
3835 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3837 if (pass == 0)
3838 continue;
3840 else if (pass != 0)
3841 continue;
3843 switch (OMP_CLAUSE_CODE (c))
3845 case OMP_CLAUSE_SHARED:
3846 /* Ignore shared directives in teams construct. */
3847 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3848 continue;
3849 /* Shared global vars are just accessed directly. */
3850 if (is_global_var (new_var))
3851 break;
3852 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3853 needs to be delayed until after fixup_child_record_type so
3854 that we get the correct type during the dereference. */
3855 by_ref = use_pointer_for_field (var, ctx);
3856 x = build_receiver_ref (var, by_ref, ctx);
3857 SET_DECL_VALUE_EXPR (new_var, x);
3858 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3860 /* ??? If VAR is not passed by reference, and the variable
3861 hasn't been initialized yet, then we'll get a warning for
3862 the store into the omp_data_s structure. Ideally, we'd be
3863 able to notice this and not store anything at all, but
3864 we're generating code too early. Suppress the warning. */
3865 if (!by_ref)
3866 TREE_NO_WARNING (var) = 1;
3867 break;
3869 case OMP_CLAUSE_LASTPRIVATE:
3870 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3871 break;
3872 /* FALLTHRU */
3874 case OMP_CLAUSE_PRIVATE:
3875 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
3876 x = build_outer_var_ref (var, ctx);
3877 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
3879 if (is_task_ctx (ctx))
3880 x = build_receiver_ref (var, false, ctx);
3881 else
3882 x = build_outer_var_ref (var, ctx);
3884 else
3885 x = NULL;
3886 do_private:
3887 tree nx;
3888 nx = lang_hooks.decls.omp_clause_default_ctor (c, new_var, x);
3889 if (is_simd)
3891 tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
3892 if ((TREE_ADDRESSABLE (new_var) || nx || y
3893 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
3894 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3895 idx, lane, ivar, lvar))
3897 if (nx)
3898 x = lang_hooks.decls.omp_clause_default_ctor
3899 (c, unshare_expr (ivar), x);
3900 if (nx && x)
3901 gimplify_and_add (x, &llist[0]);
3902 if (y)
3904 y = lang_hooks.decls.omp_clause_dtor (c, ivar);
3905 if (y)
3907 gimple_seq tseq = NULL;
3909 dtor = y;
3910 gimplify_stmt (&dtor, &tseq);
3911 gimple_seq_add_seq (&llist[1], tseq);
3914 break;
3917 if (nx)
3918 gimplify_and_add (nx, ilist);
3919 /* FALLTHRU */
3921 do_dtor:
3922 x = lang_hooks.decls.omp_clause_dtor (c, new_var);
3923 if (x)
3925 gimple_seq tseq = NULL;
3927 dtor = x;
3928 gimplify_stmt (&dtor, &tseq);
3929 gimple_seq_add_seq (dlist, tseq);
3931 break;
3933 case OMP_CLAUSE_LINEAR:
3934 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
3935 goto do_firstprivate;
3936 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
3937 x = NULL;
3938 else
3939 x = build_outer_var_ref (var, ctx);
3940 goto do_private;
3942 case OMP_CLAUSE_FIRSTPRIVATE:
3943 if (is_task_ctx (ctx))
3945 if (is_reference (var) || is_variable_sized (var))
3946 goto do_dtor;
3947 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
3948 ctx))
3949 || use_pointer_for_field (var, NULL))
3951 x = build_receiver_ref (var, false, ctx);
3952 SET_DECL_VALUE_EXPR (new_var, x);
3953 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3954 goto do_dtor;
3957 do_firstprivate:
3958 x = build_outer_var_ref (var, ctx);
3959 if (is_simd)
3961 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
3962 && gimple_omp_for_combined_into_p (ctx->stmt))
3964 tree t = OMP_CLAUSE_LINEAR_STEP (c);
3965 tree stept = TREE_TYPE (t);
3966 tree ct = find_omp_clause (clauses,
3967 OMP_CLAUSE__LOOPTEMP_);
3968 gcc_assert (ct);
3969 tree l = OMP_CLAUSE_DECL (ct);
3970 tree n1 = fd->loop.n1;
3971 tree step = fd->loop.step;
3972 tree itype = TREE_TYPE (l);
3973 if (POINTER_TYPE_P (itype))
3974 itype = signed_type_for (itype);
3975 l = fold_build2 (MINUS_EXPR, itype, l, n1);
3976 if (TYPE_UNSIGNED (itype)
3977 && fd->loop.cond_code == GT_EXPR)
3978 l = fold_build2 (TRUNC_DIV_EXPR, itype,
3979 fold_build1 (NEGATE_EXPR, itype, l),
3980 fold_build1 (NEGATE_EXPR,
3981 itype, step));
3982 else
3983 l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
3984 t = fold_build2 (MULT_EXPR, stept,
3985 fold_convert (stept, l), t);
3987 if (OMP_CLAUSE_LINEAR_ARRAY (c))
3989 x = lang_hooks.decls.omp_clause_linear_ctor
3990 (c, new_var, x, t);
3991 gimplify_and_add (x, ilist);
3992 goto do_dtor;
3995 if (POINTER_TYPE_P (TREE_TYPE (x)))
3996 x = fold_build2 (POINTER_PLUS_EXPR,
3997 TREE_TYPE (x), x, t);
3998 else
3999 x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, t);
4002 if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
4003 || TREE_ADDRESSABLE (new_var))
4004 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
4005 idx, lane, ivar, lvar))
4007 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
4009 tree iv = create_tmp_var (TREE_TYPE (new_var));
4010 x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
4011 gimplify_and_add (x, ilist);
4012 gimple_stmt_iterator gsi
4013 = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
4014 gassign *g
4015 = gimple_build_assign (unshare_expr (lvar), iv);
4016 gsi_insert_before_without_update (&gsi, g,
4017 GSI_SAME_STMT);
4018 tree t = OMP_CLAUSE_LINEAR_STEP (c);
4019 enum tree_code code = PLUS_EXPR;
4020 if (POINTER_TYPE_P (TREE_TYPE (new_var)))
4021 code = POINTER_PLUS_EXPR;
4022 g = gimple_build_assign (iv, code, iv, t);
4023 gsi_insert_before_without_update (&gsi, g,
4024 GSI_SAME_STMT);
4025 break;
4027 x = lang_hooks.decls.omp_clause_copy_ctor
4028 (c, unshare_expr (ivar), x);
4029 gimplify_and_add (x, &llist[0]);
4030 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
4031 if (x)
4033 gimple_seq tseq = NULL;
4035 dtor = x;
4036 gimplify_stmt (&dtor, &tseq);
4037 gimple_seq_add_seq (&llist[1], tseq);
4039 break;
4042 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
4043 gimplify_and_add (x, ilist);
4044 goto do_dtor;
4046 case OMP_CLAUSE__LOOPTEMP_:
4047 gcc_assert (is_parallel_ctx (ctx));
4048 x = build_outer_var_ref (var, ctx);
4049 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
4050 gimplify_and_add (x, ilist);
4051 break;
4053 case OMP_CLAUSE_COPYIN:
4054 by_ref = use_pointer_for_field (var, NULL);
4055 x = build_receiver_ref (var, by_ref, ctx);
4056 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
4057 append_to_statement_list (x, &copyin_seq);
4058 copyin_by_ref |= by_ref;
4059 break;
4061 case OMP_CLAUSE_REDUCTION:
4062 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4064 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
4065 gimple tseq;
4066 x = build_outer_var_ref (var, ctx);
4068 if (is_reference (var)
4069 && !useless_type_conversion_p (TREE_TYPE (placeholder),
4070 TREE_TYPE (x)))
4071 x = build_fold_addr_expr_loc (clause_loc, x);
4072 SET_DECL_VALUE_EXPR (placeholder, x);
4073 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
4074 tree new_vard = new_var;
4075 if (is_reference (var))
4077 gcc_assert (TREE_CODE (new_var) == MEM_REF);
4078 new_vard = TREE_OPERAND (new_var, 0);
4079 gcc_assert (DECL_P (new_vard));
4081 if (is_simd
4082 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
4083 idx, lane, ivar, lvar))
4085 if (new_vard == new_var)
4087 gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
4088 SET_DECL_VALUE_EXPR (new_var, ivar);
4090 else
4092 SET_DECL_VALUE_EXPR (new_vard,
4093 build_fold_addr_expr (ivar));
4094 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
4096 x = lang_hooks.decls.omp_clause_default_ctor
4097 (c, unshare_expr (ivar),
4098 build_outer_var_ref (var, ctx));
4099 if (x)
4100 gimplify_and_add (x, &llist[0]);
4101 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
4103 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
4104 lower_omp (&tseq, ctx);
4105 gimple_seq_add_seq (&llist[0], tseq);
4107 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
4108 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
4109 lower_omp (&tseq, ctx);
4110 gimple_seq_add_seq (&llist[1], tseq);
4111 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
4112 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
4113 if (new_vard == new_var)
4114 SET_DECL_VALUE_EXPR (new_var, lvar);
4115 else
4116 SET_DECL_VALUE_EXPR (new_vard,
4117 build_fold_addr_expr (lvar));
4118 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
4119 if (x)
4121 tseq = NULL;
4122 dtor = x;
4123 gimplify_stmt (&dtor, &tseq);
4124 gimple_seq_add_seq (&llist[1], tseq);
4126 break;
4128 /* If this is a reference to constant size reduction var
4129 with placeholder, we haven't emitted the initializer
4130 for it because it is undesirable if SIMD arrays are used.
4131 But if they aren't used, we need to emit the deferred
4132 initialization now. */
4133 else if (is_reference (var) && is_simd)
4134 handle_simd_reference (clause_loc, new_vard, ilist);
4135 x = lang_hooks.decls.omp_clause_default_ctor
4136 (c, unshare_expr (new_var),
4137 build_outer_var_ref (var, ctx));
4138 if (x)
4139 gimplify_and_add (x, ilist);
4140 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
4142 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
4143 lower_omp (&tseq, ctx);
4144 gimple_seq_add_seq (ilist, tseq);
4146 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
4147 if (is_simd)
4149 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
4150 lower_omp (&tseq, ctx);
4151 gimple_seq_add_seq (dlist, tseq);
4152 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
4154 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
4155 goto do_dtor;
4157 else
4159 x = omp_reduction_init (c, TREE_TYPE (new_var));
4160 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
4161 enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
4163 /* reduction(-:var) sums up the partial results, so it
4164 acts identically to reduction(+:var). */
4165 if (code == MINUS_EXPR)
4166 code = PLUS_EXPR;
4168 tree new_vard = new_var;
4169 if (is_simd && is_reference (var))
4171 gcc_assert (TREE_CODE (new_var) == MEM_REF);
4172 new_vard = TREE_OPERAND (new_var, 0);
4173 gcc_assert (DECL_P (new_vard));
4175 if (is_simd
4176 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
4177 idx, lane, ivar, lvar))
4179 tree ref = build_outer_var_ref (var, ctx);
4181 gimplify_assign (unshare_expr (ivar), x, &llist[0]);
4183 x = build2 (code, TREE_TYPE (ref), ref, ivar);
4184 ref = build_outer_var_ref (var, ctx);
4185 gimplify_assign (ref, x, &llist[1]);
4187 if (new_vard != new_var)
4189 SET_DECL_VALUE_EXPR (new_vard,
4190 build_fold_addr_expr (lvar));
4191 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
4194 else
4196 if (is_reference (var) && is_simd)
4197 handle_simd_reference (clause_loc, new_vard, ilist);
4198 gimplify_assign (new_var, x, ilist);
4199 if (is_simd)
4201 tree ref = build_outer_var_ref (var, ctx);
4203 x = build2 (code, TREE_TYPE (ref), ref, new_var);
4204 ref = build_outer_var_ref (var, ctx);
4205 gimplify_assign (ref, x, dlist);
4209 break;
4211 default:
4212 gcc_unreachable ();
4217 if (lane)
4219 tree uid = create_tmp_var (ptr_type_node, "simduid");
4220 /* Don't want uninit warnings on simduid, it is always uninitialized,
4221 but we use it not for the value, but for the DECL_UID only. */
4222 TREE_NO_WARNING (uid) = 1;
4223 gimple g
4224 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid);
4225 gimple_call_set_lhs (g, lane);
4226 gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
4227 gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
4228 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
4229 OMP_CLAUSE__SIMDUID__DECL (c) = uid;
4230 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
4231 gimple_omp_for_set_clauses (ctx->stmt, c);
4232 g = gimple_build_assign (lane, INTEGER_CST,
4233 build_int_cst (unsigned_type_node, 0));
4234 gimple_seq_add_stmt (ilist, g);
4235 for (int i = 0; i < 2; i++)
4236 if (llist[i])
4238 tree vf = create_tmp_var (unsigned_type_node);
4239 g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
4240 gimple_call_set_lhs (g, vf);
4241 gimple_seq *seq = i == 0 ? ilist : dlist;
4242 gimple_seq_add_stmt (seq, g);
4243 tree t = build_int_cst (unsigned_type_node, 0);
4244 g = gimple_build_assign (idx, INTEGER_CST, t);
4245 gimple_seq_add_stmt (seq, g);
4246 tree body = create_artificial_label (UNKNOWN_LOCATION);
4247 tree header = create_artificial_label (UNKNOWN_LOCATION);
4248 tree end = create_artificial_label (UNKNOWN_LOCATION);
4249 gimple_seq_add_stmt (seq, gimple_build_goto (header));
4250 gimple_seq_add_stmt (seq, gimple_build_label (body));
4251 gimple_seq_add_seq (seq, llist[i]);
4252 t = build_int_cst (unsigned_type_node, 1);
4253 g = gimple_build_assign (idx, PLUS_EXPR, idx, t);
4254 gimple_seq_add_stmt (seq, g);
4255 gimple_seq_add_stmt (seq, gimple_build_label (header));
4256 g = gimple_build_cond (LT_EXPR, idx, vf, body, end);
4257 gimple_seq_add_stmt (seq, g);
4258 gimple_seq_add_stmt (seq, gimple_build_label (end));
4262 /* The copyin sequence is not to be executed by the main thread, since
4263 that would result in self-copies. Perhaps not visible to scalars,
4264 but it certainly is to C++ operator=. */
4265 if (copyin_seq)
4267 x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
4269 x = build2 (NE_EXPR, boolean_type_node, x,
4270 build_int_cst (TREE_TYPE (x), 0));
4271 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
4272 gimplify_and_add (x, ilist);
4275 /* If any copyin variable is passed by reference, we must ensure the
4276 master thread doesn't modify it before it is copied over in all
4277 threads. Similarly for variables in both firstprivate and
4278 lastprivate clauses we need to ensure the lastprivate copying
4279 happens after firstprivate copying in all threads. And similarly
4280 for UDRs if initializer expression refers to omp_orig. */
4281 if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref)
4283 /* Don't add any barrier for #pragma omp simd or
4284 #pragma omp distribute. */
4285 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
4286 || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR)
4287 gimple_seq_add_stmt (ilist, build_omp_barrier (NULL_TREE));
4290 /* If max_vf is non-zero, then we can use only a vectorization factor
4291 up to the max_vf we chose. So stick it into the safelen clause. */
4292 if (max_vf)
4294 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
4295 OMP_CLAUSE_SAFELEN);
4296 if (c == NULL_TREE
4297 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) == INTEGER_CST
4298 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
4299 max_vf) == 1))
4301 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
4302 OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
4303 max_vf);
4304 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
4305 gimple_omp_for_set_clauses (ctx->stmt, c);
4311 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4312 both parallel and workshare constructs. PREDICATE may be NULL if it's
4313 always true. */
4315 static void
4316 lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list,
4317 omp_context *ctx)
4319 tree x, c, label = NULL, orig_clauses = clauses;
4320 bool par_clauses = false;
4321 tree simduid = NULL, lastlane = NULL;
4323 /* Early exit if there are no lastprivate or linear clauses. */
4324 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
4325 if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
4326 || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
4327 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
4328 break;
4329 if (clauses == NULL)
4331 /* If this was a workshare clause, see if it had been combined
4332 with its parallel. In that case, look for the clauses on the
4333 parallel statement itself. */
4334 if (is_parallel_ctx (ctx))
4335 return;
4337 ctx = ctx->outer;
4338 if (ctx == NULL || !is_parallel_ctx (ctx))
4339 return;
4341 clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
4342 OMP_CLAUSE_LASTPRIVATE);
4343 if (clauses == NULL)
4344 return;
4345 par_clauses = true;
4348 if (predicate)
4350 gcond *stmt;
4351 tree label_true, arm1, arm2;
4353 label = create_artificial_label (UNKNOWN_LOCATION);
4354 label_true = create_artificial_label (UNKNOWN_LOCATION);
4355 arm1 = TREE_OPERAND (predicate, 0);
4356 arm2 = TREE_OPERAND (predicate, 1);
4357 gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
4358 gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
4359 stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2,
4360 label_true, label);
4361 gimple_seq_add_stmt (stmt_list, stmt);
4362 gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
4365 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4366 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
4368 simduid = find_omp_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
4369 if (simduid)
4370 simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
4373 for (c = clauses; c ;)
4375 tree var, new_var;
4376 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4378 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
4379 || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
4380 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
4382 var = OMP_CLAUSE_DECL (c);
4383 new_var = lookup_decl (var, ctx);
4385 if (simduid && DECL_HAS_VALUE_EXPR_P (new_var))
4387 tree val = DECL_VALUE_EXPR (new_var);
4388 if (TREE_CODE (val) == ARRAY_REF
4389 && VAR_P (TREE_OPERAND (val, 0))
4390 && lookup_attribute ("omp simd array",
4391 DECL_ATTRIBUTES (TREE_OPERAND (val,
4392 0))))
4394 if (lastlane == NULL)
4396 lastlane = create_tmp_var (unsigned_type_node);
4397 gcall *g
4398 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
4399 2, simduid,
4400 TREE_OPERAND (val, 1));
4401 gimple_call_set_lhs (g, lastlane);
4402 gimple_seq_add_stmt (stmt_list, g);
4404 new_var = build4 (ARRAY_REF, TREE_TYPE (val),
4405 TREE_OPERAND (val, 0), lastlane,
4406 NULL_TREE, NULL_TREE);
4410 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
4411 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
4413 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
4414 gimple_seq_add_seq (stmt_list,
4415 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
4416 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
4418 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
4419 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
4421 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
4422 gimple_seq_add_seq (stmt_list,
4423 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
4424 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
4427 x = build_outer_var_ref (var, ctx);
4428 if (is_reference (var))
4429 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4430 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
4431 gimplify_and_add (x, stmt_list);
4433 c = OMP_CLAUSE_CHAIN (c);
4434 if (c == NULL && !par_clauses)
4436 /* If this was a workshare clause, see if it had been combined
4437 with its parallel. In that case, continue looking for the
4438 clauses also on the parallel statement itself. */
4439 if (is_parallel_ctx (ctx))
4440 break;
4442 ctx = ctx->outer;
4443 if (ctx == NULL || !is_parallel_ctx (ctx))
4444 break;
4446 c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
4447 OMP_CLAUSE_LASTPRIVATE);
4448 par_clauses = true;
4452 if (label)
4453 gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
4456 static void
4457 oacc_lower_reduction_var_helper (gimple_seq *stmt_seqp, omp_context *ctx,
4458 tree tid, tree var, tree new_var)
4460 /* The atomic add at the end of the sum creates unnecessary
4461 write contention on accelerators. To work around this,
4462 create an array to store the partial reductions. Later, in
4463 lower_omp_for (for openacc), the values of array will be
4464 combined. */
4466 tree t = NULL_TREE, array, x;
4467 tree type = get_base_type (var);
4468 gimple stmt;
4470 /* Now insert the partial reductions into the array. */
4472 /* Find the reduction array. */
4474 tree ptype = build_pointer_type (type);
4476 t = lookup_oacc_reduction (oacc_get_reduction_array_id (var), ctx);
4477 t = build_receiver_ref (t, false, ctx->outer);
4479 array = create_tmp_var (ptype);
4480 gimplify_assign (array, t, stmt_seqp);
4482 tree ptr = create_tmp_var (TREE_TYPE (array));
4484 /* Find the reduction array. */
4486 /* testing a unary conversion. */
4487 tree offset = create_tmp_var (sizetype);
4488 gimplify_assign (offset, TYPE_SIZE_UNIT (type),
4489 stmt_seqp);
4490 t = create_tmp_var (sizetype);
4491 gimplify_assign (t, unshare_expr (fold_build1 (NOP_EXPR, sizetype, tid)),
4492 stmt_seqp);
4493 stmt = gimple_build_assign (offset, MULT_EXPR, offset, t);
4494 gimple_seq_add_stmt (stmt_seqp, stmt);
4496 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4497 of adding sizeof(var) to the array? */
4498 ptr = create_tmp_var (ptype);
4499 stmt = gimple_build_assign (unshare_expr (ptr), POINTER_PLUS_EXPR, array,
4500 offset);
4501 gimple_seq_add_stmt (stmt_seqp, stmt);
4503 /* Move the local sum to gfc$sum[i]. */
4504 x = unshare_expr (build_simple_mem_ref (ptr));
4505 stmt = gimplify_assign (x, new_var, stmt_seqp);
4508 /* Generate code to implement the REDUCTION clauses. */
4510 static void
4511 lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx)
4513 gimple_seq sub_seq = NULL;
4514 gimple stmt;
4515 tree x, c, tid = NULL_TREE;
4516 int count = 0;
4518 /* SIMD reductions are handled in lower_rec_input_clauses. */
4519 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4520 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
4521 return;
4523 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4524 update in that case, otherwise use a lock. */
4525 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
4526 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
4528 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4530 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4531 count = -1;
4532 break;
4534 count++;
4537 if (count == 0)
4538 return;
4540 /* Initialize thread info for OpenACC. */
4541 if (is_gimple_omp_oacc (ctx->stmt))
4543 /* Get the current thread id. */
4544 tree call = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
4545 tid = create_tmp_var (TREE_TYPE (TREE_TYPE (call)));
4546 gimple stmt = gimple_build_call (call, 0);
4547 gimple_call_set_lhs (stmt, tid);
4548 gimple_seq_add_stmt (stmt_seqp, stmt);
4551 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4553 tree var, ref, new_var;
4554 enum tree_code code;
4555 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4557 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
4558 continue;
4560 var = OMP_CLAUSE_DECL (c);
4561 new_var = lookup_decl (var, ctx);
4562 if (is_reference (var))
4563 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4564 ref = build_outer_var_ref (var, ctx);
4565 code = OMP_CLAUSE_REDUCTION_CODE (c);
4567 /* reduction(-:var) sums up the partial results, so it acts
4568 identically to reduction(+:var). */
4569 if (code == MINUS_EXPR)
4570 code = PLUS_EXPR;
4572 if (is_gimple_omp_oacc (ctx->stmt))
4574 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c));
4576 oacc_lower_reduction_var_helper (stmt_seqp, ctx, tid, var, new_var);
4578 else if (count == 1)
4580 tree addr = build_fold_addr_expr_loc (clause_loc, ref);
4582 addr = save_expr (addr);
4583 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
4584 x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var);
4585 x = build2 (OMP_ATOMIC, void_type_node, addr, x);
4586 gimplify_and_add (x, stmt_seqp);
4587 return;
4589 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4591 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
4593 if (is_reference (var)
4594 && !useless_type_conversion_p (TREE_TYPE (placeholder),
4595 TREE_TYPE (ref)))
4596 ref = build_fold_addr_expr_loc (clause_loc, ref);
4597 SET_DECL_VALUE_EXPR (placeholder, ref);
4598 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
4599 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
4600 gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
4601 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
4602 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
4604 else
4606 x = build2 (code, TREE_TYPE (ref), ref, new_var);
4607 ref = build_outer_var_ref (var, ctx);
4608 gimplify_assign (ref, x, &sub_seq);
4612 if (is_gimple_omp_oacc (ctx->stmt))
4613 return;
4615 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START),
4617 gimple_seq_add_stmt (stmt_seqp, stmt);
4619 gimple_seq_add_seq (stmt_seqp, sub_seq);
4621 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
4623 gimple_seq_add_stmt (stmt_seqp, stmt);
4627 /* Generate code to implement the COPYPRIVATE clauses. */
4629 static void
4630 lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
4631 omp_context *ctx)
4633 tree c;
4635 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4637 tree var, new_var, ref, x;
4638 bool by_ref;
4639 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4641 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
4642 continue;
4644 var = OMP_CLAUSE_DECL (c);
4645 by_ref = use_pointer_for_field (var, NULL);
4647 ref = build_sender_ref (var, ctx);
4648 x = new_var = lookup_decl_in_outer_ctx (var, ctx);
4649 if (by_ref)
4651 x = build_fold_addr_expr_loc (clause_loc, new_var);
4652 x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
4654 gimplify_assign (ref, x, slist);
4656 ref = build_receiver_ref (var, false, ctx);
4657 if (by_ref)
4659 ref = fold_convert_loc (clause_loc,
4660 build_pointer_type (TREE_TYPE (new_var)),
4661 ref);
4662 ref = build_fold_indirect_ref_loc (clause_loc, ref);
4664 if (is_reference (var))
4666 ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
4667 ref = build_simple_mem_ref_loc (clause_loc, ref);
4668 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4670 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
4671 gimplify_and_add (x, rlist);
4676 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4677 and REDUCTION from the sender (aka parent) side. */
4679 static void
4680 lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
4681 omp_context *ctx)
4683 tree c;
4685 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4687 tree val, ref, x, var;
4688 bool by_ref, do_in = false, do_out = false;
4689 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4691 switch (OMP_CLAUSE_CODE (c))
4693 case OMP_CLAUSE_PRIVATE:
4694 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
4695 break;
4696 continue;
4697 case OMP_CLAUSE_FIRSTPRIVATE:
4698 case OMP_CLAUSE_COPYIN:
4699 case OMP_CLAUSE_LASTPRIVATE:
4700 case OMP_CLAUSE_REDUCTION:
4701 case OMP_CLAUSE__LOOPTEMP_:
4702 break;
4703 default:
4704 continue;
4707 val = OMP_CLAUSE_DECL (c);
4708 var = lookup_decl_in_outer_ctx (val, ctx);
4710 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
4711 && is_global_var (var))
4712 continue;
4713 if (is_variable_sized (val))
4714 continue;
4715 by_ref = use_pointer_for_field (val, NULL);
4717 switch (OMP_CLAUSE_CODE (c))
4719 case OMP_CLAUSE_PRIVATE:
4720 case OMP_CLAUSE_FIRSTPRIVATE:
4721 case OMP_CLAUSE_COPYIN:
4722 case OMP_CLAUSE__LOOPTEMP_:
4723 do_in = true;
4724 break;
4726 case OMP_CLAUSE_LASTPRIVATE:
4727 if (by_ref || is_reference (val))
4729 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
4730 continue;
4731 do_in = true;
4733 else
4735 do_out = true;
4736 if (lang_hooks.decls.omp_private_outer_ref (val))
4737 do_in = true;
4739 break;
4741 case OMP_CLAUSE_REDUCTION:
4742 do_in = true;
4743 do_out = !(by_ref || is_reference (val));
4744 break;
4746 default:
4747 gcc_unreachable ();
4750 if (do_in)
4752 ref = build_sender_ref (val, ctx);
4753 x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
4754 gimplify_assign (ref, x, ilist);
4755 if (is_task_ctx (ctx))
4756 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
4759 if (do_out)
4761 ref = build_sender_ref (val, ctx);
4762 gimplify_assign (var, ref, olist);
4767 /* Generate code to implement SHARED from the sender (aka parent)
4768 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4769 list things that got automatically shared. */
4771 static void
4772 lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
4774 tree var, ovar, nvar, f, x, record_type;
4776 if (ctx->record_type == NULL)
4777 return;
4779 record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
4780 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
4782 ovar = DECL_ABSTRACT_ORIGIN (f);
4783 nvar = maybe_lookup_decl (ovar, ctx);
4784 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar))
4785 continue;
4787 /* If CTX is a nested parallel directive. Find the immediately
4788 enclosing parallel or workshare construct that contains a
4789 mapping for OVAR. */
4790 var = lookup_decl_in_outer_ctx (ovar, ctx);
4792 if (use_pointer_for_field (ovar, ctx))
4794 x = build_sender_ref (ovar, ctx);
4795 var = build_fold_addr_expr (var);
4796 gimplify_assign (x, var, ilist);
4798 else
4800 x = build_sender_ref (ovar, ctx);
4801 gimplify_assign (x, var, ilist);
4803 if (!TREE_READONLY (var)
4804 /* We don't need to receive a new reference to a result
4805 or parm decl. In fact we may not store to it as we will
4806 invalidate any pending RSO and generate wrong gimple
4807 during inlining. */
4808 && !((TREE_CODE (var) == RESULT_DECL
4809 || TREE_CODE (var) == PARM_DECL)
4810 && DECL_BY_REFERENCE (var)))
4812 x = build_sender_ref (ovar, ctx);
4813 gimplify_assign (var, x, olist);
4820 /* A convenience function to build an empty GIMPLE_COND with just the
4821 condition. */
4823 static gcond *
4824 gimple_build_cond_empty (tree cond)
4826 enum tree_code pred_code;
4827 tree lhs, rhs;
4829 gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs);
4830 return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE);
4834 /* Build the function calls to GOMP_parallel_start etc to actually
4835 generate the parallel operation. REGION is the parallel region
4836 being expanded. BB is the block where to insert the code. WS_ARGS
4837 will be set if this is a call to a combined parallel+workshare
4838 construct, it contains the list of additional arguments needed by
4839 the workshare construct. */
4841 static void
4842 expand_parallel_call (struct omp_region *region, basic_block bb,
4843 gomp_parallel *entry_stmt,
4844 vec<tree, va_gc> *ws_args)
4846 tree t, t1, t2, val, cond, c, clauses, flags;
4847 gimple_stmt_iterator gsi;
4848 gimple stmt;
4849 enum built_in_function start_ix;
4850 int start_ix2;
4851 location_t clause_loc;
4852 vec<tree, va_gc> *args;
4854 clauses = gimple_omp_parallel_clauses (entry_stmt);
4856 /* Determine what flavor of GOMP_parallel we will be
4857 emitting. */
4858 start_ix = BUILT_IN_GOMP_PARALLEL;
4859 if (is_combined_parallel (region))
4861 switch (region->inner->type)
4863 case GIMPLE_OMP_FOR:
4864 gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
4865 start_ix2 = ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4866 + (region->inner->sched_kind
4867 == OMP_CLAUSE_SCHEDULE_RUNTIME
4868 ? 3 : region->inner->sched_kind));
4869 start_ix = (enum built_in_function)start_ix2;
4870 break;
4871 case GIMPLE_OMP_SECTIONS:
4872 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS;
4873 break;
4874 default:
4875 gcc_unreachable ();
4879 /* By default, the value of NUM_THREADS is zero (selected at run time)
4880 and there is no conditional. */
4881 cond = NULL_TREE;
4882 val = build_int_cst (unsigned_type_node, 0);
4883 flags = build_int_cst (unsigned_type_node, 0);
4885 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
4886 if (c)
4887 cond = OMP_CLAUSE_IF_EXPR (c);
4889 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS);
4890 if (c)
4892 val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
4893 clause_loc = OMP_CLAUSE_LOCATION (c);
4895 else
4896 clause_loc = gimple_location (entry_stmt);
4898 c = find_omp_clause (clauses, OMP_CLAUSE_PROC_BIND);
4899 if (c)
4900 flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c));
4902 /* Ensure 'val' is of the correct type. */
4903 val = fold_convert_loc (clause_loc, unsigned_type_node, val);
4905 /* If we found the clause 'if (cond)', build either
4906 (cond != 0) or (cond ? val : 1u). */
4907 if (cond)
4909 cond = gimple_boolify (cond);
4911 if (integer_zerop (val))
4912 val = fold_build2_loc (clause_loc,
4913 EQ_EXPR, unsigned_type_node, cond,
4914 build_int_cst (TREE_TYPE (cond), 0));
4915 else
4917 basic_block cond_bb, then_bb, else_bb;
4918 edge e, e_then, e_else;
4919 tree tmp_then, tmp_else, tmp_join, tmp_var;
4921 tmp_var = create_tmp_var (TREE_TYPE (val));
4922 if (gimple_in_ssa_p (cfun))
4924 tmp_then = make_ssa_name (tmp_var);
4925 tmp_else = make_ssa_name (tmp_var);
4926 tmp_join = make_ssa_name (tmp_var);
4928 else
4930 tmp_then = tmp_var;
4931 tmp_else = tmp_var;
4932 tmp_join = tmp_var;
4935 e = split_block (bb, NULL);
4936 cond_bb = e->src;
4937 bb = e->dest;
4938 remove_edge (e);
4940 then_bb = create_empty_bb (cond_bb);
4941 else_bb = create_empty_bb (then_bb);
4942 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
4943 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
4945 stmt = gimple_build_cond_empty (cond);
4946 gsi = gsi_start_bb (cond_bb);
4947 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4949 gsi = gsi_start_bb (then_bb);
4950 stmt = gimple_build_assign (tmp_then, val);
4951 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4953 gsi = gsi_start_bb (else_bb);
4954 stmt = gimple_build_assign
4955 (tmp_else, build_int_cst (unsigned_type_node, 1));
4956 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4958 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
4959 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
4960 add_bb_to_loop (then_bb, cond_bb->loop_father);
4961 add_bb_to_loop (else_bb, cond_bb->loop_father);
4962 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
4963 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
4965 if (gimple_in_ssa_p (cfun))
4967 gphi *phi = create_phi_node (tmp_join, bb);
4968 add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION);
4969 add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION);
4972 val = tmp_join;
4975 gsi = gsi_start_bb (bb);
4976 val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE,
4977 false, GSI_CONTINUE_LINKING);
4980 gsi = gsi_last_bb (bb);
4981 t = gimple_omp_parallel_data_arg (entry_stmt);
4982 if (t == NULL)
4983 t1 = null_pointer_node;
4984 else
4985 t1 = build_fold_addr_expr (t);
4986 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
4988 vec_alloc (args, 4 + vec_safe_length (ws_args));
4989 args->quick_push (t2);
4990 args->quick_push (t1);
4991 args->quick_push (val);
4992 if (ws_args)
4993 args->splice (*ws_args);
4994 args->quick_push (flags);
4996 t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
4997 builtin_decl_explicit (start_ix), args);
4999 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5000 false, GSI_CONTINUE_LINKING);
5003 /* Insert a function call whose name is FUNC_NAME with the information from
5004 ENTRY_STMT into the basic_block BB. */
5006 static void
5007 expand_cilk_for_call (basic_block bb, gomp_parallel *entry_stmt,
5008 vec <tree, va_gc> *ws_args)
5010 tree t, t1, t2;
5011 gimple_stmt_iterator gsi;
5012 vec <tree, va_gc> *args;
5014 gcc_assert (vec_safe_length (ws_args) == 2);
5015 tree func_name = (*ws_args)[0];
5016 tree grain = (*ws_args)[1];
5018 tree clauses = gimple_omp_parallel_clauses (entry_stmt);
5019 tree count = find_omp_clause (clauses, OMP_CLAUSE__CILK_FOR_COUNT_);
5020 gcc_assert (count != NULL_TREE);
5021 count = OMP_CLAUSE_OPERAND (count, 0);
5023 gsi = gsi_last_bb (bb);
5024 t = gimple_omp_parallel_data_arg (entry_stmt);
5025 if (t == NULL)
5026 t1 = null_pointer_node;
5027 else
5028 t1 = build_fold_addr_expr (t);
5029 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
5031 vec_alloc (args, 4);
5032 args->quick_push (t2);
5033 args->quick_push (t1);
5034 args->quick_push (count);
5035 args->quick_push (grain);
5036 t = build_call_expr_loc_vec (UNKNOWN_LOCATION, func_name, args);
5038 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, false,
5039 GSI_CONTINUE_LINKING);
5042 /* Build the function call to GOMP_task to actually
5043 generate the task operation. BB is the block where to insert the code. */
5045 static void
5046 expand_task_call (basic_block bb, gomp_task *entry_stmt)
5048 tree t, t1, t2, t3, flags, cond, c, c2, clauses, depend;
5049 gimple_stmt_iterator gsi;
5050 location_t loc = gimple_location (entry_stmt);
5052 clauses = gimple_omp_task_clauses (entry_stmt);
5054 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
5055 if (c)
5056 cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (c));
5057 else
5058 cond = boolean_true_node;
5060 c = find_omp_clause (clauses, OMP_CLAUSE_UNTIED);
5061 c2 = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE);
5062 depend = find_omp_clause (clauses, OMP_CLAUSE_DEPEND);
5063 flags = build_int_cst (unsigned_type_node,
5064 (c ? 1 : 0) + (c2 ? 4 : 0) + (depend ? 8 : 0));
5066 c = find_omp_clause (clauses, OMP_CLAUSE_FINAL);
5067 if (c)
5069 c = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c));
5070 c = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, c,
5071 build_int_cst (unsigned_type_node, 2),
5072 build_int_cst (unsigned_type_node, 0));
5073 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, c);
5075 if (depend)
5076 depend = OMP_CLAUSE_DECL (depend);
5077 else
5078 depend = build_int_cst (ptr_type_node, 0);
5080 gsi = gsi_last_bb (bb);
5081 t = gimple_omp_task_data_arg (entry_stmt);
5082 if (t == NULL)
5083 t2 = null_pointer_node;
5084 else
5085 t2 = build_fold_addr_expr_loc (loc, t);
5086 t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt));
5087 t = gimple_omp_task_copy_fn (entry_stmt);
5088 if (t == NULL)
5089 t3 = null_pointer_node;
5090 else
5091 t3 = build_fold_addr_expr_loc (loc, t);
5093 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
5094 8, t1, t2, t3,
5095 gimple_omp_task_arg_size (entry_stmt),
5096 gimple_omp_task_arg_align (entry_stmt), cond, flags,
5097 depend);
5099 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5100 false, GSI_CONTINUE_LINKING);
5104 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5105 catch handler and return it. This prevents programs from violating the
5106 structured block semantics with throws. */
5108 static gimple_seq
5109 maybe_catch_exception (gimple_seq body)
5111 gimple g;
5112 tree decl;
5114 if (!flag_exceptions)
5115 return body;
5117 if (lang_hooks.eh_protect_cleanup_actions != NULL)
5118 decl = lang_hooks.eh_protect_cleanup_actions ();
5119 else
5120 decl = builtin_decl_explicit (BUILT_IN_TRAP);
5122 g = gimple_build_eh_must_not_throw (decl);
5123 g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
5124 GIMPLE_TRY_CATCH);
5126 return gimple_seq_alloc_with_stmt (g);
5129 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5131 static tree
5132 vec2chain (vec<tree, va_gc> *v)
5134 tree chain = NULL_TREE, t;
5135 unsigned ix;
5137 FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t)
5139 DECL_CHAIN (t) = chain;
5140 chain = t;
5143 return chain;
5147 /* Remove barriers in REGION->EXIT's block. Note that this is only
5148 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5149 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5150 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5151 removed. */
5153 static void
5154 remove_exit_barrier (struct omp_region *region)
5156 gimple_stmt_iterator gsi;
5157 basic_block exit_bb;
5158 edge_iterator ei;
5159 edge e;
5160 gimple stmt;
5161 int any_addressable_vars = -1;
5163 exit_bb = region->exit;
5165 /* If the parallel region doesn't return, we don't have REGION->EXIT
5166 block at all. */
5167 if (! exit_bb)
5168 return;
5170 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5171 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5172 statements that can appear in between are extremely limited -- no
5173 memory operations at all. Here, we allow nothing at all, so the
5174 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5175 gsi = gsi_last_bb (exit_bb);
5176 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
5177 gsi_prev (&gsi);
5178 if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
5179 return;
5181 FOR_EACH_EDGE (e, ei, exit_bb->preds)
5183 gsi = gsi_last_bb (e->src);
5184 if (gsi_end_p (gsi))
5185 continue;
5186 stmt = gsi_stmt (gsi);
5187 if (gimple_code (stmt) == GIMPLE_OMP_RETURN
5188 && !gimple_omp_return_nowait_p (stmt))
5190 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5191 in many cases. If there could be tasks queued, the barrier
5192 might be needed to let the tasks run before some local
5193 variable of the parallel that the task uses as shared
5194 runs out of scope. The task can be spawned either
5195 from within current function (this would be easy to check)
5196 or from some function it calls and gets passed an address
5197 of such a variable. */
5198 if (any_addressable_vars < 0)
5200 gomp_parallel *parallel_stmt
5201 = as_a <gomp_parallel *> (last_stmt (region->entry));
5202 tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt);
5203 tree local_decls, block, decl;
5204 unsigned ix;
5206 any_addressable_vars = 0;
5207 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl)
5208 if (TREE_ADDRESSABLE (decl))
5210 any_addressable_vars = 1;
5211 break;
5213 for (block = gimple_block (stmt);
5214 !any_addressable_vars
5215 && block
5216 && TREE_CODE (block) == BLOCK;
5217 block = BLOCK_SUPERCONTEXT (block))
5219 for (local_decls = BLOCK_VARS (block);
5220 local_decls;
5221 local_decls = DECL_CHAIN (local_decls))
5222 if (TREE_ADDRESSABLE (local_decls))
5224 any_addressable_vars = 1;
5225 break;
5227 if (block == gimple_block (parallel_stmt))
5228 break;
5231 if (!any_addressable_vars)
5232 gimple_omp_return_set_nowait (stmt);
5237 static void
5238 remove_exit_barriers (struct omp_region *region)
5240 if (region->type == GIMPLE_OMP_PARALLEL)
5241 remove_exit_barrier (region);
5243 if (region->inner)
5245 region = region->inner;
5246 remove_exit_barriers (region);
5247 while (region->next)
5249 region = region->next;
5250 remove_exit_barriers (region);
5255 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5256 calls. These can't be declared as const functions, but
5257 within one parallel body they are constant, so they can be
5258 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5259 which are declared const. Similarly for task body, except
5260 that in untied task omp_get_thread_num () can change at any task
5261 scheduling point. */
5263 static void
5264 optimize_omp_library_calls (gimple entry_stmt)
5266 basic_block bb;
5267 gimple_stmt_iterator gsi;
5268 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5269 tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree);
5270 tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
5271 tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree);
5272 bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
5273 && find_omp_clause (gimple_omp_task_clauses (entry_stmt),
5274 OMP_CLAUSE_UNTIED) != NULL);
5276 FOR_EACH_BB_FN (bb, cfun)
5277 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
5279 gimple call = gsi_stmt (gsi);
5280 tree decl;
5282 if (is_gimple_call (call)
5283 && (decl = gimple_call_fndecl (call))
5284 && DECL_EXTERNAL (decl)
5285 && TREE_PUBLIC (decl)
5286 && DECL_INITIAL (decl) == NULL)
5288 tree built_in;
5290 if (DECL_NAME (decl) == thr_num_id)
5292 /* In #pragma omp task untied omp_get_thread_num () can change
5293 during the execution of the task region. */
5294 if (untied_task)
5295 continue;
5296 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5298 else if (DECL_NAME (decl) == num_thr_id)
5299 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
5300 else
5301 continue;
5303 if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in)
5304 || gimple_call_num_args (call) != 0)
5305 continue;
5307 if (flag_exceptions && !TREE_NOTHROW (decl))
5308 continue;
5310 if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE
5311 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)),
5312 TREE_TYPE (TREE_TYPE (built_in))))
5313 continue;
5315 gimple_call_set_fndecl (call, built_in);
5320 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5321 regimplified. */
5323 static tree
5324 expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *)
5326 tree t = *tp;
5328 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5329 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t))
5330 return t;
5332 if (TREE_CODE (t) == ADDR_EXPR)
5333 recompute_tree_invariant_for_addr_expr (t);
5335 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
5336 return NULL_TREE;
5339 /* Prepend TO = FROM assignment before *GSI_P. */
5341 static void
5342 expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from)
5344 bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to);
5345 from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE,
5346 true, GSI_SAME_STMT);
5347 gimple stmt = gimple_build_assign (to, from);
5348 gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT);
5349 if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL)
5350 || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL))
5352 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5353 gimple_regimplify_operands (stmt, &gsi);
5357 /* Expand the OpenMP parallel or task directive starting at REGION. */
5359 static void
5360 expand_omp_taskreg (struct omp_region *region)
5362 basic_block entry_bb, exit_bb, new_bb;
5363 struct function *child_cfun;
5364 tree child_fn, block, t;
5365 gimple_stmt_iterator gsi;
5366 gimple entry_stmt, stmt;
5367 edge e;
5368 vec<tree, va_gc> *ws_args;
5370 entry_stmt = last_stmt (region->entry);
5371 child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
5372 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
5374 entry_bb = region->entry;
5375 exit_bb = region->exit;
5377 bool is_cilk_for
5378 = (flag_cilkplus
5379 && gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL
5380 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt),
5381 OMP_CLAUSE__CILK_FOR_COUNT_) != NULL_TREE);
5383 if (is_cilk_for)
5384 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5385 and the inner statement contains the name of the built-in function
5386 and grain. */
5387 ws_args = region->inner->ws_args;
5388 else if (is_combined_parallel (region))
5389 ws_args = region->ws_args;
5390 else
5391 ws_args = NULL;
5393 if (child_cfun->cfg)
5395 /* Due to inlining, it may happen that we have already outlined
5396 the region, in which case all we need to do is make the
5397 sub-graph unreachable and emit the parallel call. */
5398 edge entry_succ_e, exit_succ_e;
5400 entry_succ_e = single_succ_edge (entry_bb);
5402 gsi = gsi_last_bb (entry_bb);
5403 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
5404 || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
5405 gsi_remove (&gsi, true);
5407 new_bb = entry_bb;
5408 if (exit_bb)
5410 exit_succ_e = single_succ_edge (exit_bb);
5411 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
5413 remove_edge_and_dominated_blocks (entry_succ_e);
5415 else
5417 unsigned srcidx, dstidx, num;
5419 /* If the parallel region needs data sent from the parent
5420 function, then the very first statement (except possible
5421 tree profile counter updates) of the parallel body
5422 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5423 &.OMP_DATA_O is passed as an argument to the child function,
5424 we need to replace it with the argument as seen by the child
5425 function.
5427 In most cases, this will end up being the identity assignment
5428 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5429 a function call that has been inlined, the original PARM_DECL
5430 .OMP_DATA_I may have been converted into a different local
5431 variable. In which case, we need to keep the assignment. */
5432 if (gimple_omp_taskreg_data_arg (entry_stmt))
5434 basic_block entry_succ_bb = single_succ (entry_bb);
5435 tree arg, narg;
5436 gimple parcopy_stmt = NULL;
5438 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
5440 gimple stmt;
5442 gcc_assert (!gsi_end_p (gsi));
5443 stmt = gsi_stmt (gsi);
5444 if (gimple_code (stmt) != GIMPLE_ASSIGN)
5445 continue;
5447 if (gimple_num_ops (stmt) == 2)
5449 tree arg = gimple_assign_rhs1 (stmt);
5451 /* We're ignore the subcode because we're
5452 effectively doing a STRIP_NOPS. */
5454 if (TREE_CODE (arg) == ADDR_EXPR
5455 && TREE_OPERAND (arg, 0)
5456 == gimple_omp_taskreg_data_arg (entry_stmt))
5458 parcopy_stmt = stmt;
5459 break;
5464 gcc_assert (parcopy_stmt != NULL);
5465 arg = DECL_ARGUMENTS (child_fn);
5467 if (!gimple_in_ssa_p (cfun))
5469 if (gimple_assign_lhs (parcopy_stmt) == arg)
5470 gsi_remove (&gsi, true);
5471 else
5473 /* ?? Is setting the subcode really necessary ?? */
5474 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg));
5475 gimple_assign_set_rhs1 (parcopy_stmt, arg);
5478 else
5480 /* If we are in ssa form, we must load the value from the default
5481 definition of the argument. That should not be defined now,
5482 since the argument is not used uninitialized. */
5483 gcc_assert (ssa_default_def (cfun, arg) == NULL);
5484 narg = make_ssa_name (arg, gimple_build_nop ());
5485 set_ssa_default_def (cfun, arg, narg);
5486 /* ?? Is setting the subcode really necessary ?? */
5487 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (narg));
5488 gimple_assign_set_rhs1 (parcopy_stmt, narg);
5489 update_stmt (parcopy_stmt);
5493 /* Declare local variables needed in CHILD_CFUN. */
5494 block = DECL_INITIAL (child_fn);
5495 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
5496 /* The gimplifier could record temporaries in parallel/task block
5497 rather than in containing function's local_decls chain,
5498 which would mean cgraph missed finalizing them. Do it now. */
5499 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
5500 if (TREE_CODE (t) == VAR_DECL
5501 && TREE_STATIC (t)
5502 && !DECL_EXTERNAL (t))
5503 varpool_node::finalize_decl (t);
5504 DECL_SAVED_TREE (child_fn) = NULL;
5505 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5506 gimple_set_body (child_fn, NULL);
5507 TREE_USED (block) = 1;
5509 /* Reset DECL_CONTEXT on function arguments. */
5510 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
5511 DECL_CONTEXT (t) = child_fn;
5513 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5514 so that it can be moved to the child function. */
5515 gsi = gsi_last_bb (entry_bb);
5516 stmt = gsi_stmt (gsi);
5517 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
5518 || gimple_code (stmt) == GIMPLE_OMP_TASK));
5519 e = split_block (entry_bb, stmt);
5520 gsi_remove (&gsi, true);
5521 entry_bb = e->dest;
5522 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
5524 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5525 if (exit_bb)
5527 gsi = gsi_last_bb (exit_bb);
5528 gcc_assert (!gsi_end_p (gsi)
5529 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
5530 stmt = gimple_build_return (NULL);
5531 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
5532 gsi_remove (&gsi, true);
5535 /* Move the parallel region into CHILD_CFUN. */
5537 if (gimple_in_ssa_p (cfun))
5539 init_tree_ssa (child_cfun);
5540 init_ssa_operands (child_cfun);
5541 child_cfun->gimple_df->in_ssa_p = true;
5542 block = NULL_TREE;
5544 else
5545 block = gimple_block (entry_stmt);
5547 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
5548 if (exit_bb)
5549 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
5550 /* When the OMP expansion process cannot guarantee an up-to-date
5551 loop tree arrange for the child function to fixup loops. */
5552 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
5553 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
5555 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5556 num = vec_safe_length (child_cfun->local_decls);
5557 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
5559 t = (*child_cfun->local_decls)[srcidx];
5560 if (DECL_CONTEXT (t) == cfun->decl)
5561 continue;
5562 if (srcidx != dstidx)
5563 (*child_cfun->local_decls)[dstidx] = t;
5564 dstidx++;
5566 if (dstidx != num)
5567 vec_safe_truncate (child_cfun->local_decls, dstidx);
5569 /* Inform the callgraph about the new function. */
5570 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
5571 cgraph_node::add_new_function (child_fn, true);
5573 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5574 fixed in a following pass. */
5575 push_cfun (child_cfun);
5576 if (optimize)
5577 optimize_omp_library_calls (entry_stmt);
5578 cgraph_edge::rebuild_edges ();
5580 /* Some EH regions might become dead, see PR34608. If
5581 pass_cleanup_cfg isn't the first pass to happen with the
5582 new child, these dead EH edges might cause problems.
5583 Clean them up now. */
5584 if (flag_exceptions)
5586 basic_block bb;
5587 bool changed = false;
5589 FOR_EACH_BB_FN (bb, cfun)
5590 changed |= gimple_purge_dead_eh_edges (bb);
5591 if (changed)
5592 cleanup_tree_cfg ();
5594 if (gimple_in_ssa_p (cfun))
5595 update_ssa (TODO_update_ssa);
5596 pop_cfun ();
5599 /* Emit a library call to launch the children threads. */
5600 if (is_cilk_for)
5601 expand_cilk_for_call (new_bb,
5602 as_a <gomp_parallel *> (entry_stmt), ws_args);
5603 else if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
5604 expand_parallel_call (region, new_bb,
5605 as_a <gomp_parallel *> (entry_stmt), ws_args);
5606 else
5607 expand_task_call (new_bb, as_a <gomp_task *> (entry_stmt));
5608 if (gimple_in_ssa_p (cfun))
5609 update_ssa (TODO_update_ssa_only_virtuals);
5613 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5614 of the combined collapse > 1 loop constructs, generate code like:
5615 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5616 if (cond3 is <)
5617 adj = STEP3 - 1;
5618 else
5619 adj = STEP3 + 1;
5620 count3 = (adj + N32 - N31) / STEP3;
5621 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5622 if (cond2 is <)
5623 adj = STEP2 - 1;
5624 else
5625 adj = STEP2 + 1;
5626 count2 = (adj + N22 - N21) / STEP2;
5627 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5628 if (cond1 is <)
5629 adj = STEP1 - 1;
5630 else
5631 adj = STEP1 + 1;
5632 count1 = (adj + N12 - N11) / STEP1;
5633 count = count1 * count2 * count3;
5634 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5635 count = 0;
5636 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5637 of the combined loop constructs, just initialize COUNTS array
5638 from the _looptemp_ clauses. */
5640 /* NOTE: It *could* be better to moosh all of the BBs together,
5641 creating one larger BB with all the computation and the unexpected
5642 jump at the end. I.e.
5644 bool zero3, zero2, zero1, zero;
5646 zero3 = N32 c3 N31;
5647 count3 = (N32 - N31) /[cl] STEP3;
5648 zero2 = N22 c2 N21;
5649 count2 = (N22 - N21) /[cl] STEP2;
5650 zero1 = N12 c1 N11;
5651 count1 = (N12 - N11) /[cl] STEP1;
5652 zero = zero3 || zero2 || zero1;
5653 count = count1 * count2 * count3;
5654 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5656 After all, we expect the zero=false, and thus we expect to have to
5657 evaluate all of the comparison expressions, so short-circuiting
5658 oughtn't be a win. Since the condition isn't protecting a
5659 denominator, we're not concerned about divide-by-zero, so we can
5660 fully evaluate count even if a numerator turned out to be wrong.
5662 It seems like putting this all together would create much better
5663 scheduling opportunities, and less pressure on the chip's branch
5664 predictor. */
5666 static void
5667 expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
5668 basic_block &entry_bb, tree *counts,
5669 basic_block &zero_iter_bb, int &first_zero_iter,
5670 basic_block &l2_dom_bb)
5672 tree t, type = TREE_TYPE (fd->loop.v);
5673 edge e, ne;
5674 int i;
5676 /* Collapsed loops need work for expansion into SSA form. */
5677 gcc_assert (!gimple_in_ssa_p (cfun));
5679 if (gimple_omp_for_combined_into_p (fd->for_stmt)
5680 && TREE_CODE (fd->loop.n2) != INTEGER_CST)
5682 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5683 isn't supposed to be handled, as the inner loop doesn't
5684 use it. */
5685 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
5686 OMP_CLAUSE__LOOPTEMP_);
5687 gcc_assert (innerc);
5688 for (i = 0; i < fd->collapse; i++)
5690 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5691 OMP_CLAUSE__LOOPTEMP_);
5692 gcc_assert (innerc);
5693 if (i)
5694 counts[i] = OMP_CLAUSE_DECL (innerc);
5695 else
5696 counts[0] = NULL_TREE;
5698 return;
5701 for (i = 0; i < fd->collapse; i++)
5703 tree itype = TREE_TYPE (fd->loops[i].v);
5705 if (SSA_VAR_P (fd->loop.n2)
5706 && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
5707 fold_convert (itype, fd->loops[i].n1),
5708 fold_convert (itype, fd->loops[i].n2)))
5709 == NULL_TREE || !integer_onep (t)))
5711 gcond *cond_stmt;
5712 tree n1, n2;
5713 n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1));
5714 n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE,
5715 true, GSI_SAME_STMT);
5716 n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
5717 n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE,
5718 true, GSI_SAME_STMT);
5719 cond_stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
5720 NULL_TREE, NULL_TREE);
5721 gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT);
5722 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
5723 expand_omp_regimplify_p, NULL, NULL)
5724 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
5725 expand_omp_regimplify_p, NULL, NULL))
5727 *gsi = gsi_for_stmt (cond_stmt);
5728 gimple_regimplify_operands (cond_stmt, gsi);
5730 e = split_block (entry_bb, cond_stmt);
5731 if (zero_iter_bb == NULL)
5733 gassign *assign_stmt;
5734 first_zero_iter = i;
5735 zero_iter_bb = create_empty_bb (entry_bb);
5736 add_bb_to_loop (zero_iter_bb, entry_bb->loop_father);
5737 *gsi = gsi_after_labels (zero_iter_bb);
5738 assign_stmt = gimple_build_assign (fd->loop.n2,
5739 build_zero_cst (type));
5740 gsi_insert_before (gsi, assign_stmt, GSI_SAME_STMT);
5741 set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb,
5742 entry_bb);
5744 ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
5745 ne->probability = REG_BR_PROB_BASE / 2000 - 1;
5746 e->flags = EDGE_TRUE_VALUE;
5747 e->probability = REG_BR_PROB_BASE - ne->probability;
5748 if (l2_dom_bb == NULL)
5749 l2_dom_bb = entry_bb;
5750 entry_bb = e->dest;
5751 *gsi = gsi_last_bb (entry_bb);
5754 if (POINTER_TYPE_P (itype))
5755 itype = signed_type_for (itype);
5756 t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
5757 ? -1 : 1));
5758 t = fold_build2 (PLUS_EXPR, itype,
5759 fold_convert (itype, fd->loops[i].step), t);
5760 t = fold_build2 (PLUS_EXPR, itype, t,
5761 fold_convert (itype, fd->loops[i].n2));
5762 t = fold_build2 (MINUS_EXPR, itype, t,
5763 fold_convert (itype, fd->loops[i].n1));
5764 /* ?? We could probably use CEIL_DIV_EXPR instead of
5765 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5766 generate the same code in the end because generically we
5767 don't know that the values involved must be negative for
5768 GT?? */
5769 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
5770 t = fold_build2 (TRUNC_DIV_EXPR, itype,
5771 fold_build1 (NEGATE_EXPR, itype, t),
5772 fold_build1 (NEGATE_EXPR, itype,
5773 fold_convert (itype,
5774 fd->loops[i].step)));
5775 else
5776 t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
5777 fold_convert (itype, fd->loops[i].step));
5778 t = fold_convert (type, t);
5779 if (TREE_CODE (t) == INTEGER_CST)
5780 counts[i] = t;
5781 else
5783 counts[i] = create_tmp_reg (type, ".count");
5784 expand_omp_build_assign (gsi, counts[i], t);
5786 if (SSA_VAR_P (fd->loop.n2))
5788 if (i == 0)
5789 t = counts[0];
5790 else
5791 t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]);
5792 expand_omp_build_assign (gsi, fd->loop.n2, t);
5798 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5799 T = V;
5800 V3 = N31 + (T % count3) * STEP3;
5801 T = T / count3;
5802 V2 = N21 + (T % count2) * STEP2;
5803 T = T / count2;
5804 V1 = N11 + T * STEP1;
5805 if this loop doesn't have an inner loop construct combined with it.
5806 If it does have an inner loop construct combined with it and the
5807 iteration count isn't known constant, store values from counts array
5808 into its _looptemp_ temporaries instead. */
5810 static void
5811 expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
5812 tree *counts, gimple inner_stmt, tree startvar)
5814 int i;
5815 if (gimple_omp_for_combined_p (fd->for_stmt))
5817 /* If fd->loop.n2 is constant, then no propagation of the counts
5818 is needed, they are constant. */
5819 if (TREE_CODE (fd->loop.n2) == INTEGER_CST)
5820 return;
5822 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
5823 ? gimple_omp_parallel_clauses (inner_stmt)
5824 : gimple_omp_for_clauses (inner_stmt);
5825 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5826 isn't supposed to be handled, as the inner loop doesn't
5827 use it. */
5828 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
5829 gcc_assert (innerc);
5830 for (i = 0; i < fd->collapse; i++)
5832 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5833 OMP_CLAUSE__LOOPTEMP_);
5834 gcc_assert (innerc);
5835 if (i)
5837 tree tem = OMP_CLAUSE_DECL (innerc);
5838 tree t = fold_convert (TREE_TYPE (tem), counts[i]);
5839 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
5840 false, GSI_CONTINUE_LINKING);
5841 gassign *stmt = gimple_build_assign (tem, t);
5842 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5845 return;
5848 tree type = TREE_TYPE (fd->loop.v);
5849 tree tem = create_tmp_reg (type, ".tem");
5850 gassign *stmt = gimple_build_assign (tem, startvar);
5851 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5853 for (i = fd->collapse - 1; i >= 0; i--)
5855 tree vtype = TREE_TYPE (fd->loops[i].v), itype, t;
5856 itype = vtype;
5857 if (POINTER_TYPE_P (vtype))
5858 itype = signed_type_for (vtype);
5859 if (i != 0)
5860 t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]);
5861 else
5862 t = tem;
5863 t = fold_convert (itype, t);
5864 t = fold_build2 (MULT_EXPR, itype, t,
5865 fold_convert (itype, fd->loops[i].step));
5866 if (POINTER_TYPE_P (vtype))
5867 t = fold_build_pointer_plus (fd->loops[i].n1, t);
5868 else
5869 t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
5870 t = force_gimple_operand_gsi (gsi, t,
5871 DECL_P (fd->loops[i].v)
5872 && TREE_ADDRESSABLE (fd->loops[i].v),
5873 NULL_TREE, false,
5874 GSI_CONTINUE_LINKING);
5875 stmt = gimple_build_assign (fd->loops[i].v, t);
5876 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5877 if (i != 0)
5879 t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]);
5880 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
5881 false, GSI_CONTINUE_LINKING);
5882 stmt = gimple_build_assign (tem, t);
5883 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5889 /* Helper function for expand_omp_for_*. Generate code like:
5890 L10:
5891 V3 += STEP3;
5892 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5893 L11:
5894 V3 = N31;
5895 V2 += STEP2;
5896 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5897 L12:
5898 V2 = N21;
5899 V1 += STEP1;
5900 goto BODY_BB; */
5902 static basic_block
5903 extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb,
5904 basic_block body_bb)
5906 basic_block last_bb, bb, collapse_bb = NULL;
5907 int i;
5908 gimple_stmt_iterator gsi;
5909 edge e;
5910 tree t;
5911 gimple stmt;
5913 last_bb = cont_bb;
5914 for (i = fd->collapse - 1; i >= 0; i--)
5916 tree vtype = TREE_TYPE (fd->loops[i].v);
5918 bb = create_empty_bb (last_bb);
5919 add_bb_to_loop (bb, last_bb->loop_father);
5920 gsi = gsi_start_bb (bb);
5922 if (i < fd->collapse - 1)
5924 e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
5925 e->probability = REG_BR_PROB_BASE / 8;
5927 t = fd->loops[i + 1].n1;
5928 t = force_gimple_operand_gsi (&gsi, t,
5929 DECL_P (fd->loops[i + 1].v)
5930 && TREE_ADDRESSABLE (fd->loops[i
5931 + 1].v),
5932 NULL_TREE, false,
5933 GSI_CONTINUE_LINKING);
5934 stmt = gimple_build_assign (fd->loops[i + 1].v, t);
5935 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5937 else
5938 collapse_bb = bb;
5940 set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
5942 if (POINTER_TYPE_P (vtype))
5943 t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
5944 else
5945 t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step);
5946 t = force_gimple_operand_gsi (&gsi, t,
5947 DECL_P (fd->loops[i].v)
5948 && TREE_ADDRESSABLE (fd->loops[i].v),
5949 NULL_TREE, false, GSI_CONTINUE_LINKING);
5950 stmt = gimple_build_assign (fd->loops[i].v, t);
5951 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5953 if (i > 0)
5955 t = fd->loops[i].n2;
5956 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5957 false, GSI_CONTINUE_LINKING);
5958 tree v = fd->loops[i].v;
5959 if (DECL_P (v) && TREE_ADDRESSABLE (v))
5960 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
5961 false, GSI_CONTINUE_LINKING);
5962 t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
5963 stmt = gimple_build_cond_empty (t);
5964 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5965 e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
5966 e->probability = REG_BR_PROB_BASE * 7 / 8;
5968 else
5969 make_edge (bb, body_bb, EDGE_FALLTHRU);
5970 last_bb = bb;
5973 return collapse_bb;
5977 /* A subroutine of expand_omp_for. Generate code for a parallel
5978 loop with any schedule. Given parameters:
5980 for (V = N1; V cond N2; V += STEP) BODY;
5982 where COND is "<" or ">", we generate pseudocode
5984 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5985 if (more) goto L0; else goto L3;
5987 V = istart0;
5988 iend = iend0;
5990 BODY;
5991 V += STEP;
5992 if (V cond iend) goto L1; else goto L2;
5994 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5997 If this is a combined omp parallel loop, instead of the call to
5998 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5999 If this is gimple_omp_for_combined_p loop, then instead of assigning
6000 V and iend in L0 we assign the first two _looptemp_ clause decls of the
6001 inner GIMPLE_OMP_FOR and V += STEP; and
6002 if (V cond iend) goto L1; else goto L2; are removed.
6004 For collapsed loops, given parameters:
6005 collapse(3)
6006 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6007 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6008 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6009 BODY;
6011 we generate pseudocode
6013 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6014 if (cond3 is <)
6015 adj = STEP3 - 1;
6016 else
6017 adj = STEP3 + 1;
6018 count3 = (adj + N32 - N31) / STEP3;
6019 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6020 if (cond2 is <)
6021 adj = STEP2 - 1;
6022 else
6023 adj = STEP2 + 1;
6024 count2 = (adj + N22 - N21) / STEP2;
6025 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6026 if (cond1 is <)
6027 adj = STEP1 - 1;
6028 else
6029 adj = STEP1 + 1;
6030 count1 = (adj + N12 - N11) / STEP1;
6031 count = count1 * count2 * count3;
6032 goto Z1;
6034 count = 0;
6036 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6037 if (more) goto L0; else goto L3;
6039 V = istart0;
6040 T = V;
6041 V3 = N31 + (T % count3) * STEP3;
6042 T = T / count3;
6043 V2 = N21 + (T % count2) * STEP2;
6044 T = T / count2;
6045 V1 = N11 + T * STEP1;
6046 iend = iend0;
6048 BODY;
6049 V += 1;
6050 if (V < iend) goto L10; else goto L2;
6051 L10:
6052 V3 += STEP3;
6053 if (V3 cond3 N32) goto L1; else goto L11;
6054 L11:
6055 V3 = N31;
6056 V2 += STEP2;
6057 if (V2 cond2 N22) goto L1; else goto L12;
6058 L12:
6059 V2 = N21;
6060 V1 += STEP1;
6061 goto L1;
6063 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6068 static void
6069 expand_omp_for_generic (struct omp_region *region,
6070 struct omp_for_data *fd,
6071 enum built_in_function start_fn,
6072 enum built_in_function next_fn,
6073 gimple inner_stmt)
6075 tree type, istart0, iend0, iend;
6076 tree t, vmain, vback, bias = NULL_TREE;
6077 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
6078 basic_block l2_bb = NULL, l3_bb = NULL;
6079 gimple_stmt_iterator gsi;
6080 gassign *assign_stmt;
6081 bool in_combined_parallel = is_combined_parallel (region);
6082 bool broken_loop = region->cont == NULL;
6083 edge e, ne;
6084 tree *counts = NULL;
6085 int i;
6087 gcc_assert (!broken_loop || !in_combined_parallel);
6088 gcc_assert (fd->iter_type == long_integer_type_node
6089 || !in_combined_parallel);
6091 type = TREE_TYPE (fd->loop.v);
6092 istart0 = create_tmp_var (fd->iter_type, ".istart0");
6093 iend0 = create_tmp_var (fd->iter_type, ".iend0");
6094 TREE_ADDRESSABLE (istart0) = 1;
6095 TREE_ADDRESSABLE (iend0) = 1;
6097 /* See if we need to bias by LLONG_MIN. */
6098 if (fd->iter_type == long_long_unsigned_type_node
6099 && TREE_CODE (type) == INTEGER_TYPE
6100 && !TYPE_UNSIGNED (type))
6102 tree n1, n2;
6104 if (fd->loop.cond_code == LT_EXPR)
6106 n1 = fd->loop.n1;
6107 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
6109 else
6111 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
6112 n2 = fd->loop.n1;
6114 if (TREE_CODE (n1) != INTEGER_CST
6115 || TREE_CODE (n2) != INTEGER_CST
6116 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
6117 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
6120 entry_bb = region->entry;
6121 cont_bb = region->cont;
6122 collapse_bb = NULL;
6123 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6124 gcc_assert (broken_loop
6125 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
6126 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
6127 l1_bb = single_succ (l0_bb);
6128 if (!broken_loop)
6130 l2_bb = create_empty_bb (cont_bb);
6131 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb);
6132 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6134 else
6135 l2_bb = NULL;
6136 l3_bb = BRANCH_EDGE (entry_bb)->dest;
6137 exit_bb = region->exit;
6139 gsi = gsi_last_bb (entry_bb);
6141 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6142 if (fd->collapse > 1)
6144 int first_zero_iter = -1;
6145 basic_block zero_iter_bb = NULL, l2_dom_bb = NULL;
6147 counts = XALLOCAVEC (tree, fd->collapse);
6148 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6149 zero_iter_bb, first_zero_iter,
6150 l2_dom_bb);
6152 if (zero_iter_bb)
6154 /* Some counts[i] vars might be uninitialized if
6155 some loop has zero iterations. But the body shouldn't
6156 be executed in that case, so just avoid uninit warnings. */
6157 for (i = first_zero_iter; i < fd->collapse; i++)
6158 if (SSA_VAR_P (counts[i]))
6159 TREE_NO_WARNING (counts[i]) = 1;
6160 gsi_prev (&gsi);
6161 e = split_block (entry_bb, gsi_stmt (gsi));
6162 entry_bb = e->dest;
6163 make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU);
6164 gsi = gsi_last_bb (entry_bb);
6165 set_immediate_dominator (CDI_DOMINATORS, entry_bb,
6166 get_immediate_dominator (CDI_DOMINATORS,
6167 zero_iter_bb));
6170 if (in_combined_parallel)
6172 /* In a combined parallel loop, emit a call to
6173 GOMP_loop_foo_next. */
6174 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
6175 build_fold_addr_expr (istart0),
6176 build_fold_addr_expr (iend0));
6178 else
6180 tree t0, t1, t2, t3, t4;
6181 /* If this is not a combined parallel loop, emit a call to
6182 GOMP_loop_foo_start in ENTRY_BB. */
6183 t4 = build_fold_addr_expr (iend0);
6184 t3 = build_fold_addr_expr (istart0);
6185 t2 = fold_convert (fd->iter_type, fd->loop.step);
6186 t1 = fd->loop.n2;
6187 t0 = fd->loop.n1;
6188 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6190 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6191 OMP_CLAUSE__LOOPTEMP_);
6192 gcc_assert (innerc);
6193 t0 = OMP_CLAUSE_DECL (innerc);
6194 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6195 OMP_CLAUSE__LOOPTEMP_);
6196 gcc_assert (innerc);
6197 t1 = OMP_CLAUSE_DECL (innerc);
6199 if (POINTER_TYPE_P (TREE_TYPE (t0))
6200 && TYPE_PRECISION (TREE_TYPE (t0))
6201 != TYPE_PRECISION (fd->iter_type))
6203 /* Avoid casting pointers to integer of a different size. */
6204 tree itype = signed_type_for (type);
6205 t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
6206 t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
6208 else
6210 t1 = fold_convert (fd->iter_type, t1);
6211 t0 = fold_convert (fd->iter_type, t0);
6213 if (bias)
6215 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
6216 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
6218 if (fd->iter_type == long_integer_type_node)
6220 if (fd->chunk_size)
6222 t = fold_convert (fd->iter_type, fd->chunk_size);
6223 t = build_call_expr (builtin_decl_explicit (start_fn),
6224 6, t0, t1, t2, t, t3, t4);
6226 else
6227 t = build_call_expr (builtin_decl_explicit (start_fn),
6228 5, t0, t1, t2, t3, t4);
6230 else
6232 tree t5;
6233 tree c_bool_type;
6234 tree bfn_decl;
6236 /* The GOMP_loop_ull_*start functions have additional boolean
6237 argument, true for < loops and false for > loops.
6238 In Fortran, the C bool type can be different from
6239 boolean_type_node. */
6240 bfn_decl = builtin_decl_explicit (start_fn);
6241 c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl));
6242 t5 = build_int_cst (c_bool_type,
6243 fd->loop.cond_code == LT_EXPR ? 1 : 0);
6244 if (fd->chunk_size)
6246 tree bfn_decl = builtin_decl_explicit (start_fn);
6247 t = fold_convert (fd->iter_type, fd->chunk_size);
6248 t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
6250 else
6251 t = build_call_expr (builtin_decl_explicit (start_fn),
6252 6, t5, t0, t1, t2, t3, t4);
6255 if (TREE_TYPE (t) != boolean_type_node)
6256 t = fold_build2 (NE_EXPR, boolean_type_node,
6257 t, build_int_cst (TREE_TYPE (t), 0));
6258 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6259 true, GSI_SAME_STMT);
6260 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6262 /* Remove the GIMPLE_OMP_FOR statement. */
6263 gsi_remove (&gsi, true);
6265 /* Iteration setup for sequential loop goes in L0_BB. */
6266 tree startvar = fd->loop.v;
6267 tree endvar = NULL_TREE;
6269 if (gimple_omp_for_combined_p (fd->for_stmt))
6271 gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR
6272 && gimple_omp_for_kind (inner_stmt)
6273 == GF_OMP_FOR_KIND_SIMD);
6274 tree innerc = find_omp_clause (gimple_omp_for_clauses (inner_stmt),
6275 OMP_CLAUSE__LOOPTEMP_);
6276 gcc_assert (innerc);
6277 startvar = OMP_CLAUSE_DECL (innerc);
6278 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6279 OMP_CLAUSE__LOOPTEMP_);
6280 gcc_assert (innerc);
6281 endvar = OMP_CLAUSE_DECL (innerc);
6284 gsi = gsi_start_bb (l0_bb);
6285 t = istart0;
6286 if (bias)
6287 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
6288 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
6289 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
6290 t = fold_convert (TREE_TYPE (startvar), t);
6291 t = force_gimple_operand_gsi (&gsi, t,
6292 DECL_P (startvar)
6293 && TREE_ADDRESSABLE (startvar),
6294 NULL_TREE, false, GSI_CONTINUE_LINKING);
6295 assign_stmt = gimple_build_assign (startvar, t);
6296 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6298 t = iend0;
6299 if (bias)
6300 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
6301 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
6302 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
6303 t = fold_convert (TREE_TYPE (startvar), t);
6304 iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6305 false, GSI_CONTINUE_LINKING);
6306 if (endvar)
6308 assign_stmt = gimple_build_assign (endvar, iend);
6309 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6310 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (iend)))
6311 assign_stmt = gimple_build_assign (fd->loop.v, iend);
6312 else
6313 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, iend);
6314 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6316 if (fd->collapse > 1)
6317 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
6319 if (!broken_loop)
6321 /* Code to control the increment and predicate for the sequential
6322 loop goes in the CONT_BB. */
6323 gsi = gsi_last_bb (cont_bb);
6324 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
6325 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
6326 vmain = gimple_omp_continue_control_use (cont_stmt);
6327 vback = gimple_omp_continue_control_def (cont_stmt);
6329 if (!gimple_omp_for_combined_p (fd->for_stmt))
6331 if (POINTER_TYPE_P (type))
6332 t = fold_build_pointer_plus (vmain, fd->loop.step);
6333 else
6334 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
6335 t = force_gimple_operand_gsi (&gsi, t,
6336 DECL_P (vback)
6337 && TREE_ADDRESSABLE (vback),
6338 NULL_TREE, true, GSI_SAME_STMT);
6339 assign_stmt = gimple_build_assign (vback, t);
6340 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6342 t = build2 (fd->loop.cond_code, boolean_type_node,
6343 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
6344 iend);
6345 gcond *cond_stmt = gimple_build_cond_empty (t);
6346 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
6349 /* Remove GIMPLE_OMP_CONTINUE. */
6350 gsi_remove (&gsi, true);
6352 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6353 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, l1_bb);
6355 /* Emit code to get the next parallel iteration in L2_BB. */
6356 gsi = gsi_start_bb (l2_bb);
6358 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
6359 build_fold_addr_expr (istart0),
6360 build_fold_addr_expr (iend0));
6361 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6362 false, GSI_CONTINUE_LINKING);
6363 if (TREE_TYPE (t) != boolean_type_node)
6364 t = fold_build2 (NE_EXPR, boolean_type_node,
6365 t, build_int_cst (TREE_TYPE (t), 0));
6366 gcond *cond_stmt = gimple_build_cond_empty (t);
6367 gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
6370 /* Add the loop cleanup function. */
6371 gsi = gsi_last_bb (exit_bb);
6372 if (gimple_omp_return_nowait_p (gsi_stmt (gsi)))
6373 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
6374 else if (gimple_omp_return_lhs (gsi_stmt (gsi)))
6375 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
6376 else
6377 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
6378 gcall *call_stmt = gimple_build_call (t, 0);
6379 if (gimple_omp_return_lhs (gsi_stmt (gsi)))
6380 gimple_call_set_lhs (call_stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
6381 gsi_insert_after (&gsi, call_stmt, GSI_SAME_STMT);
6382 gsi_remove (&gsi, true);
6384 /* Connect the new blocks. */
6385 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
6386 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
6388 if (!broken_loop)
6390 gimple_seq phis;
6392 e = find_edge (cont_bb, l3_bb);
6393 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
6395 phis = phi_nodes (l3_bb);
6396 for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi))
6398 gimple phi = gsi_stmt (gsi);
6399 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
6400 PHI_ARG_DEF_FROM_EDGE (phi, e));
6402 remove_edge (e);
6404 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
6405 add_bb_to_loop (l2_bb, cont_bb->loop_father);
6406 e = find_edge (cont_bb, l1_bb);
6407 if (gimple_omp_for_combined_p (fd->for_stmt))
6409 remove_edge (e);
6410 e = NULL;
6412 else if (fd->collapse > 1)
6414 remove_edge (e);
6415 e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6417 else
6418 e->flags = EDGE_TRUE_VALUE;
6419 if (e)
6421 e->probability = REG_BR_PROB_BASE * 7 / 8;
6422 find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8;
6424 else
6426 e = find_edge (cont_bb, l2_bb);
6427 e->flags = EDGE_FALLTHRU;
6429 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
6431 set_immediate_dominator (CDI_DOMINATORS, l2_bb,
6432 recompute_dominator (CDI_DOMINATORS, l2_bb));
6433 set_immediate_dominator (CDI_DOMINATORS, l3_bb,
6434 recompute_dominator (CDI_DOMINATORS, l3_bb));
6435 set_immediate_dominator (CDI_DOMINATORS, l0_bb,
6436 recompute_dominator (CDI_DOMINATORS, l0_bb));
6437 set_immediate_dominator (CDI_DOMINATORS, l1_bb,
6438 recompute_dominator (CDI_DOMINATORS, l1_bb));
6440 struct loop *outer_loop = alloc_loop ();
6441 outer_loop->header = l0_bb;
6442 outer_loop->latch = l2_bb;
6443 add_loop (outer_loop, l0_bb->loop_father);
6445 if (!gimple_omp_for_combined_p (fd->for_stmt))
6447 struct loop *loop = alloc_loop ();
6448 loop->header = l1_bb;
6449 /* The loop may have multiple latches. */
6450 add_loop (loop, outer_loop);
6456 /* A subroutine of expand_omp_for. Generate code for a parallel
6457 loop with static schedule and no specified chunk size. Given
6458 parameters:
6460 for (V = N1; V cond N2; V += STEP) BODY;
6462 where COND is "<" or ">", we generate pseudocode
6464 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6465 if (cond is <)
6466 adj = STEP - 1;
6467 else
6468 adj = STEP + 1;
6469 if ((__typeof (V)) -1 > 0 && cond is >)
6470 n = -(adj + N2 - N1) / -STEP;
6471 else
6472 n = (adj + N2 - N1) / STEP;
6473 q = n / nthreads;
6474 tt = n % nthreads;
6475 if (threadid < tt) goto L3; else goto L4;
6477 tt = 0;
6478 q = q + 1;
6480 s0 = q * threadid + tt;
6481 e0 = s0 + q;
6482 V = s0 * STEP + N1;
6483 if (s0 >= e0) goto L2; else goto L0;
6485 e = e0 * STEP + N1;
6487 BODY;
6488 V += STEP;
6489 if (V cond e) goto L1;
6493 static void
6494 expand_omp_for_static_nochunk (struct omp_region *region,
6495 struct omp_for_data *fd,
6496 gimple inner_stmt)
6498 tree n, q, s0, e0, e, t, tt, nthreads, threadid;
6499 tree type, itype, vmain, vback;
6500 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
6501 basic_block body_bb, cont_bb, collapse_bb = NULL;
6502 basic_block fin_bb;
6503 gimple_stmt_iterator gsi;
6504 edge ep;
6505 bool broken_loop = region->cont == NULL;
6506 tree *counts = NULL;
6507 tree n1, n2, step;
6509 gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt)
6510 != GF_OMP_FOR_KIND_OACC_LOOP)
6511 || !inner_stmt);
6513 itype = type = TREE_TYPE (fd->loop.v);
6514 if (POINTER_TYPE_P (type))
6515 itype = signed_type_for (type);
6517 entry_bb = region->entry;
6518 cont_bb = region->cont;
6519 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6520 fin_bb = BRANCH_EDGE (entry_bb)->dest;
6521 gcc_assert (broken_loop
6522 || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
6523 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
6524 body_bb = single_succ (seq_start_bb);
6525 if (!broken_loop)
6527 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
6528 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6530 exit_bb = region->exit;
6532 /* Iteration space partitioning goes in ENTRY_BB. */
6533 gsi = gsi_last_bb (entry_bb);
6534 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6536 if (fd->collapse > 1)
6538 int first_zero_iter = -1;
6539 basic_block l2_dom_bb = NULL;
6541 counts = XALLOCAVEC (tree, fd->collapse);
6542 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6543 fin_bb, first_zero_iter,
6544 l2_dom_bb);
6545 t = NULL_TREE;
6547 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
6548 t = integer_one_node;
6549 else
6550 t = fold_binary (fd->loop.cond_code, boolean_type_node,
6551 fold_convert (type, fd->loop.n1),
6552 fold_convert (type, fd->loop.n2));
6553 if (fd->collapse == 1
6554 && TYPE_UNSIGNED (type)
6555 && (t == NULL_TREE || !integer_onep (t)))
6557 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
6558 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
6559 true, GSI_SAME_STMT);
6560 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
6561 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
6562 true, GSI_SAME_STMT);
6563 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
6564 NULL_TREE, NULL_TREE);
6565 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
6566 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
6567 expand_omp_regimplify_p, NULL, NULL)
6568 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
6569 expand_omp_regimplify_p, NULL, NULL))
6571 gsi = gsi_for_stmt (cond_stmt);
6572 gimple_regimplify_operands (cond_stmt, &gsi);
6574 ep = split_block (entry_bb, cond_stmt);
6575 ep->flags = EDGE_TRUE_VALUE;
6576 entry_bb = ep->dest;
6577 ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
6578 ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
6579 ep->probability = REG_BR_PROB_BASE / 2000 - 1;
6580 if (gimple_in_ssa_p (cfun))
6582 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
6583 for (gphi_iterator gpi = gsi_start_phis (fin_bb);
6584 !gsi_end_p (gpi); gsi_next (&gpi))
6586 gphi *phi = gpi.phi ();
6587 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
6588 ep, UNKNOWN_LOCATION);
6591 gsi = gsi_last_bb (entry_bb);
6594 switch (gimple_omp_for_kind (fd->for_stmt))
6596 case GF_OMP_FOR_KIND_FOR:
6597 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
6598 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
6599 break;
6600 case GF_OMP_FOR_KIND_DISTRIBUTE:
6601 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS);
6602 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
6603 break;
6604 case GF_OMP_FOR_KIND_OACC_LOOP:
6605 nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS);
6606 threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
6607 break;
6608 default:
6609 gcc_unreachable ();
6611 nthreads = build_call_expr (nthreads, 0);
6612 nthreads = fold_convert (itype, nthreads);
6613 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE,
6614 true, GSI_SAME_STMT);
6615 threadid = build_call_expr (threadid, 0);
6616 threadid = fold_convert (itype, threadid);
6617 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
6618 true, GSI_SAME_STMT);
6620 n1 = fd->loop.n1;
6621 n2 = fd->loop.n2;
6622 step = fd->loop.step;
6623 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6625 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6626 OMP_CLAUSE__LOOPTEMP_);
6627 gcc_assert (innerc);
6628 n1 = OMP_CLAUSE_DECL (innerc);
6629 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6630 OMP_CLAUSE__LOOPTEMP_);
6631 gcc_assert (innerc);
6632 n2 = OMP_CLAUSE_DECL (innerc);
6634 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
6635 true, NULL_TREE, true, GSI_SAME_STMT);
6636 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
6637 true, NULL_TREE, true, GSI_SAME_STMT);
6638 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
6639 true, NULL_TREE, true, GSI_SAME_STMT);
6641 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
6642 t = fold_build2 (PLUS_EXPR, itype, step, t);
6643 t = fold_build2 (PLUS_EXPR, itype, t, n2);
6644 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
6645 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
6646 t = fold_build2 (TRUNC_DIV_EXPR, itype,
6647 fold_build1 (NEGATE_EXPR, itype, t),
6648 fold_build1 (NEGATE_EXPR, itype, step));
6649 else
6650 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
6651 t = fold_convert (itype, t);
6652 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
6654 q = create_tmp_reg (itype, "q");
6655 t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads);
6656 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
6657 gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT);
6659 tt = create_tmp_reg (itype, "tt");
6660 t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads);
6661 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
6662 gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT);
6664 t = build2 (LT_EXPR, boolean_type_node, threadid, tt);
6665 gcond *cond_stmt = gimple_build_cond_empty (t);
6666 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
6668 second_bb = split_block (entry_bb, cond_stmt)->dest;
6669 gsi = gsi_last_bb (second_bb);
6670 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6672 gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)),
6673 GSI_SAME_STMT);
6674 gassign *assign_stmt
6675 = gimple_build_assign (q, PLUS_EXPR, q, build_int_cst (itype, 1));
6676 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6678 third_bb = split_block (second_bb, assign_stmt)->dest;
6679 gsi = gsi_last_bb (third_bb);
6680 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6682 t = build2 (MULT_EXPR, itype, q, threadid);
6683 t = build2 (PLUS_EXPR, itype, t, tt);
6684 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
6686 t = fold_build2 (PLUS_EXPR, itype, s0, q);
6687 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
6689 t = build2 (GE_EXPR, boolean_type_node, s0, e0);
6690 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6692 /* Remove the GIMPLE_OMP_FOR statement. */
6693 gsi_remove (&gsi, true);
6695 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6696 gsi = gsi_start_bb (seq_start_bb);
6698 tree startvar = fd->loop.v;
6699 tree endvar = NULL_TREE;
6701 if (gimple_omp_for_combined_p (fd->for_stmt))
6703 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
6704 ? gimple_omp_parallel_clauses (inner_stmt)
6705 : gimple_omp_for_clauses (inner_stmt);
6706 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
6707 gcc_assert (innerc);
6708 startvar = OMP_CLAUSE_DECL (innerc);
6709 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6710 OMP_CLAUSE__LOOPTEMP_);
6711 gcc_assert (innerc);
6712 endvar = OMP_CLAUSE_DECL (innerc);
6714 t = fold_convert (itype, s0);
6715 t = fold_build2 (MULT_EXPR, itype, t, step);
6716 if (POINTER_TYPE_P (type))
6717 t = fold_build_pointer_plus (n1, t);
6718 else
6719 t = fold_build2 (PLUS_EXPR, type, t, n1);
6720 t = fold_convert (TREE_TYPE (startvar), t);
6721 t = force_gimple_operand_gsi (&gsi, t,
6722 DECL_P (startvar)
6723 && TREE_ADDRESSABLE (startvar),
6724 NULL_TREE, false, GSI_CONTINUE_LINKING);
6725 assign_stmt = gimple_build_assign (startvar, t);
6726 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6728 t = fold_convert (itype, e0);
6729 t = fold_build2 (MULT_EXPR, itype, t, step);
6730 if (POINTER_TYPE_P (type))
6731 t = fold_build_pointer_plus (n1, t);
6732 else
6733 t = fold_build2 (PLUS_EXPR, type, t, n1);
6734 t = fold_convert (TREE_TYPE (startvar), t);
6735 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6736 false, GSI_CONTINUE_LINKING);
6737 if (endvar)
6739 assign_stmt = gimple_build_assign (endvar, e);
6740 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6741 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
6742 assign_stmt = gimple_build_assign (fd->loop.v, e);
6743 else
6744 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
6745 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
6747 if (fd->collapse > 1)
6748 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
6750 if (!broken_loop)
6752 /* The code controlling the sequential loop replaces the
6753 GIMPLE_OMP_CONTINUE. */
6754 gsi = gsi_last_bb (cont_bb);
6755 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
6756 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE);
6757 vmain = gimple_omp_continue_control_use (cont_stmt);
6758 vback = gimple_omp_continue_control_def (cont_stmt);
6760 if (!gimple_omp_for_combined_p (fd->for_stmt))
6762 if (POINTER_TYPE_P (type))
6763 t = fold_build_pointer_plus (vmain, step);
6764 else
6765 t = fold_build2 (PLUS_EXPR, type, vmain, step);
6766 t = force_gimple_operand_gsi (&gsi, t,
6767 DECL_P (vback)
6768 && TREE_ADDRESSABLE (vback),
6769 NULL_TREE, true, GSI_SAME_STMT);
6770 assign_stmt = gimple_build_assign (vback, t);
6771 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
6773 t = build2 (fd->loop.cond_code, boolean_type_node,
6774 DECL_P (vback) && TREE_ADDRESSABLE (vback)
6775 ? t : vback, e);
6776 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6779 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6780 gsi_remove (&gsi, true);
6782 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6783 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
6786 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6787 gsi = gsi_last_bb (exit_bb);
6788 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
6790 t = gimple_omp_return_lhs (gsi_stmt (gsi));
6791 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
6792 gcc_checking_assert (t == NULL_TREE);
6793 else
6794 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
6796 gsi_remove (&gsi, true);
6798 /* Connect all the blocks. */
6799 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
6800 ep->probability = REG_BR_PROB_BASE / 4 * 3;
6801 ep = find_edge (entry_bb, second_bb);
6802 ep->flags = EDGE_TRUE_VALUE;
6803 ep->probability = REG_BR_PROB_BASE / 4;
6804 find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
6805 find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
6807 if (!broken_loop)
6809 ep = find_edge (cont_bb, body_bb);
6810 if (gimple_omp_for_combined_p (fd->for_stmt))
6812 remove_edge (ep);
6813 ep = NULL;
6815 else if (fd->collapse > 1)
6817 remove_edge (ep);
6818 ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6820 else
6821 ep->flags = EDGE_TRUE_VALUE;
6822 find_edge (cont_bb, fin_bb)->flags
6823 = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
6826 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
6827 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
6828 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb);
6830 set_immediate_dominator (CDI_DOMINATORS, body_bb,
6831 recompute_dominator (CDI_DOMINATORS, body_bb));
6832 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
6833 recompute_dominator (CDI_DOMINATORS, fin_bb));
6835 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
6837 struct loop *loop = alloc_loop ();
6838 loop->header = body_bb;
6839 if (collapse_bb == NULL)
6840 loop->latch = cont_bb;
6841 add_loop (loop, body_bb->loop_father);
6846 /* A subroutine of expand_omp_for. Generate code for a parallel
6847 loop with static schedule and a specified chunk size. Given
6848 parameters:
6850 for (V = N1; V cond N2; V += STEP) BODY;
6852 where COND is "<" or ">", we generate pseudocode
6854 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6855 if (cond is <)
6856 adj = STEP - 1;
6857 else
6858 adj = STEP + 1;
6859 if ((__typeof (V)) -1 > 0 && cond is >)
6860 n = -(adj + N2 - N1) / -STEP;
6861 else
6862 n = (adj + N2 - N1) / STEP;
6863 trip = 0;
6864 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6865 here so that V is defined
6866 if the loop is not entered
6868 s0 = (trip * nthreads + threadid) * CHUNK;
6869 e0 = min(s0 + CHUNK, n);
6870 if (s0 < n) goto L1; else goto L4;
6872 V = s0 * STEP + N1;
6873 e = e0 * STEP + N1;
6875 BODY;
6876 V += STEP;
6877 if (V cond e) goto L2; else goto L3;
6879 trip += 1;
6880 goto L0;
6884 static void
6885 expand_omp_for_static_chunk (struct omp_region *region,
6886 struct omp_for_data *fd, gimple inner_stmt)
6888 tree n, s0, e0, e, t;
6889 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
6890 tree type, itype, vmain, vback, vextra;
6891 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
6892 basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb;
6893 gimple_stmt_iterator gsi;
6894 edge se;
6895 bool broken_loop = region->cont == NULL;
6896 tree *counts = NULL;
6897 tree n1, n2, step;
6899 gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt)
6900 != GF_OMP_FOR_KIND_OACC_LOOP)
6901 || !inner_stmt);
6903 itype = type = TREE_TYPE (fd->loop.v);
6904 if (POINTER_TYPE_P (type))
6905 itype = signed_type_for (type);
6907 entry_bb = region->entry;
6908 se = split_block (entry_bb, last_stmt (entry_bb));
6909 entry_bb = se->src;
6910 iter_part_bb = se->dest;
6911 cont_bb = region->cont;
6912 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
6913 fin_bb = BRANCH_EDGE (iter_part_bb)->dest;
6914 gcc_assert (broken_loop
6915 || fin_bb == FALLTHRU_EDGE (cont_bb)->dest);
6916 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
6917 body_bb = single_succ (seq_start_bb);
6918 if (!broken_loop)
6920 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
6921 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6922 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
6924 exit_bb = region->exit;
6926 /* Trip and adjustment setup goes in ENTRY_BB. */
6927 gsi = gsi_last_bb (entry_bb);
6928 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6930 if (fd->collapse > 1)
6932 int first_zero_iter = -1;
6933 basic_block l2_dom_bb = NULL;
6935 counts = XALLOCAVEC (tree, fd->collapse);
6936 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6937 fin_bb, first_zero_iter,
6938 l2_dom_bb);
6939 t = NULL_TREE;
6941 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
6942 t = integer_one_node;
6943 else
6944 t = fold_binary (fd->loop.cond_code, boolean_type_node,
6945 fold_convert (type, fd->loop.n1),
6946 fold_convert (type, fd->loop.n2));
6947 if (fd->collapse == 1
6948 && TYPE_UNSIGNED (type)
6949 && (t == NULL_TREE || !integer_onep (t)))
6951 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
6952 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
6953 true, GSI_SAME_STMT);
6954 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
6955 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
6956 true, GSI_SAME_STMT);
6957 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
6958 NULL_TREE, NULL_TREE);
6959 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT);
6960 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
6961 expand_omp_regimplify_p, NULL, NULL)
6962 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
6963 expand_omp_regimplify_p, NULL, NULL))
6965 gsi = gsi_for_stmt (cond_stmt);
6966 gimple_regimplify_operands (cond_stmt, &gsi);
6968 se = split_block (entry_bb, cond_stmt);
6969 se->flags = EDGE_TRUE_VALUE;
6970 entry_bb = se->dest;
6971 se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
6972 se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
6973 se->probability = REG_BR_PROB_BASE / 2000 - 1;
6974 if (gimple_in_ssa_p (cfun))
6976 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
6977 for (gphi_iterator gpi = gsi_start_phis (fin_bb);
6978 !gsi_end_p (gpi); gsi_next (&gpi))
6980 gphi *phi = gpi.phi ();
6981 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
6982 se, UNKNOWN_LOCATION);
6985 gsi = gsi_last_bb (entry_bb);
6988 switch (gimple_omp_for_kind (fd->for_stmt))
6990 case GF_OMP_FOR_KIND_FOR:
6991 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
6992 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
6993 break;
6994 case GF_OMP_FOR_KIND_DISTRIBUTE:
6995 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS);
6996 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
6997 break;
6998 case GF_OMP_FOR_KIND_OACC_LOOP:
6999 nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS);
7000 threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
7001 break;
7002 default:
7003 gcc_unreachable ();
7005 nthreads = build_call_expr (nthreads, 0);
7006 nthreads = fold_convert (itype, nthreads);
7007 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE,
7008 true, GSI_SAME_STMT);
7009 threadid = build_call_expr (threadid, 0);
7010 threadid = fold_convert (itype, threadid);
7011 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
7012 true, GSI_SAME_STMT);
7014 n1 = fd->loop.n1;
7015 n2 = fd->loop.n2;
7016 step = fd->loop.step;
7017 if (gimple_omp_for_combined_into_p (fd->for_stmt))
7019 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7020 OMP_CLAUSE__LOOPTEMP_);
7021 gcc_assert (innerc);
7022 n1 = OMP_CLAUSE_DECL (innerc);
7023 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7024 OMP_CLAUSE__LOOPTEMP_);
7025 gcc_assert (innerc);
7026 n2 = OMP_CLAUSE_DECL (innerc);
7028 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
7029 true, NULL_TREE, true, GSI_SAME_STMT);
7030 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
7031 true, NULL_TREE, true, GSI_SAME_STMT);
7032 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
7033 true, NULL_TREE, true, GSI_SAME_STMT);
7034 fd->chunk_size
7035 = force_gimple_operand_gsi (&gsi, fold_convert (itype, fd->chunk_size),
7036 true, NULL_TREE, true, GSI_SAME_STMT);
7038 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
7039 t = fold_build2 (PLUS_EXPR, itype, step, t);
7040 t = fold_build2 (PLUS_EXPR, itype, t, n2);
7041 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
7042 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
7043 t = fold_build2 (TRUNC_DIV_EXPR, itype,
7044 fold_build1 (NEGATE_EXPR, itype, t),
7045 fold_build1 (NEGATE_EXPR, itype, step));
7046 else
7047 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
7048 t = fold_convert (itype, t);
7049 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7050 true, GSI_SAME_STMT);
7052 trip_var = create_tmp_reg (itype, ".trip");
7053 if (gimple_in_ssa_p (cfun))
7055 trip_init = make_ssa_name (trip_var);
7056 trip_main = make_ssa_name (trip_var);
7057 trip_back = make_ssa_name (trip_var);
7059 else
7061 trip_init = trip_var;
7062 trip_main = trip_var;
7063 trip_back = trip_var;
7066 gassign *assign_stmt
7067 = gimple_build_assign (trip_init, build_int_cst (itype, 0));
7068 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
7070 t = fold_build2 (MULT_EXPR, itype, threadid, fd->chunk_size);
7071 t = fold_build2 (MULT_EXPR, itype, t, step);
7072 if (POINTER_TYPE_P (type))
7073 t = fold_build_pointer_plus (n1, t);
7074 else
7075 t = fold_build2 (PLUS_EXPR, type, t, n1);
7076 vextra = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7077 true, GSI_SAME_STMT);
7079 /* Remove the GIMPLE_OMP_FOR. */
7080 gsi_remove (&gsi, true);
7082 /* Iteration space partitioning goes in ITER_PART_BB. */
7083 gsi = gsi_last_bb (iter_part_bb);
7085 t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads);
7086 t = fold_build2 (PLUS_EXPR, itype, t, threadid);
7087 t = fold_build2 (MULT_EXPR, itype, t, fd->chunk_size);
7088 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7089 false, GSI_CONTINUE_LINKING);
7091 t = fold_build2 (PLUS_EXPR, itype, s0, fd->chunk_size);
7092 t = fold_build2 (MIN_EXPR, itype, t, n);
7093 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7094 false, GSI_CONTINUE_LINKING);
7096 t = build2 (LT_EXPR, boolean_type_node, s0, n);
7097 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING);
7099 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7100 gsi = gsi_start_bb (seq_start_bb);
7102 tree startvar = fd->loop.v;
7103 tree endvar = NULL_TREE;
7105 if (gimple_omp_for_combined_p (fd->for_stmt))
7107 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
7108 ? gimple_omp_parallel_clauses (inner_stmt)
7109 : gimple_omp_for_clauses (inner_stmt);
7110 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
7111 gcc_assert (innerc);
7112 startvar = OMP_CLAUSE_DECL (innerc);
7113 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7114 OMP_CLAUSE__LOOPTEMP_);
7115 gcc_assert (innerc);
7116 endvar = OMP_CLAUSE_DECL (innerc);
7119 t = fold_convert (itype, s0);
7120 t = fold_build2 (MULT_EXPR, itype, t, step);
7121 if (POINTER_TYPE_P (type))
7122 t = fold_build_pointer_plus (n1, t);
7123 else
7124 t = fold_build2 (PLUS_EXPR, type, t, n1);
7125 t = fold_convert (TREE_TYPE (startvar), t);
7126 t = force_gimple_operand_gsi (&gsi, t,
7127 DECL_P (startvar)
7128 && TREE_ADDRESSABLE (startvar),
7129 NULL_TREE, false, GSI_CONTINUE_LINKING);
7130 assign_stmt = gimple_build_assign (startvar, t);
7131 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7133 t = fold_convert (itype, e0);
7134 t = fold_build2 (MULT_EXPR, itype, t, step);
7135 if (POINTER_TYPE_P (type))
7136 t = fold_build_pointer_plus (n1, t);
7137 else
7138 t = fold_build2 (PLUS_EXPR, type, t, n1);
7139 t = fold_convert (TREE_TYPE (startvar), t);
7140 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7141 false, GSI_CONTINUE_LINKING);
7142 if (endvar)
7144 assign_stmt = gimple_build_assign (endvar, e);
7145 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7146 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
7147 assign_stmt = gimple_build_assign (fd->loop.v, e);
7148 else
7149 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e);
7150 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7152 if (fd->collapse > 1)
7153 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
7155 if (!broken_loop)
7157 /* The code controlling the sequential loop goes in CONT_BB,
7158 replacing the GIMPLE_OMP_CONTINUE. */
7159 gsi = gsi_last_bb (cont_bb);
7160 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi));
7161 vmain = gimple_omp_continue_control_use (cont_stmt);
7162 vback = gimple_omp_continue_control_def (cont_stmt);
7164 if (!gimple_omp_for_combined_p (fd->for_stmt))
7166 if (POINTER_TYPE_P (type))
7167 t = fold_build_pointer_plus (vmain, step);
7168 else
7169 t = fold_build2 (PLUS_EXPR, type, vmain, step);
7170 if (DECL_P (vback) && TREE_ADDRESSABLE (vback))
7171 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7172 true, GSI_SAME_STMT);
7173 assign_stmt = gimple_build_assign (vback, t);
7174 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT);
7176 t = build2 (fd->loop.cond_code, boolean_type_node,
7177 DECL_P (vback) && TREE_ADDRESSABLE (vback)
7178 ? t : vback, e);
7179 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
7182 /* Remove GIMPLE_OMP_CONTINUE. */
7183 gsi_remove (&gsi, true);
7185 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
7186 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
7188 /* Trip update code goes into TRIP_UPDATE_BB. */
7189 gsi = gsi_start_bb (trip_update_bb);
7191 t = build_int_cst (itype, 1);
7192 t = build2 (PLUS_EXPR, itype, trip_main, t);
7193 assign_stmt = gimple_build_assign (trip_back, t);
7194 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING);
7197 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7198 gsi = gsi_last_bb (exit_bb);
7199 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
7201 t = gimple_omp_return_lhs (gsi_stmt (gsi));
7202 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
7203 gcc_checking_assert (t == NULL_TREE);
7204 else
7205 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
7207 gsi_remove (&gsi, true);
7209 /* Connect the new blocks. */
7210 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
7211 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
7213 if (!broken_loop)
7215 se = find_edge (cont_bb, body_bb);
7216 if (gimple_omp_for_combined_p (fd->for_stmt))
7218 remove_edge (se);
7219 se = NULL;
7221 else if (fd->collapse > 1)
7223 remove_edge (se);
7224 se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
7226 else
7227 se->flags = EDGE_TRUE_VALUE;
7228 find_edge (cont_bb, trip_update_bb)->flags
7229 = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
7231 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb);
7234 if (gimple_in_ssa_p (cfun))
7236 gphi_iterator psi;
7237 gphi *phi;
7238 edge re, ene;
7239 edge_var_map *vm;
7240 size_t i;
7242 gcc_assert (fd->collapse == 1 && !broken_loop);
7244 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7245 remove arguments of the phi nodes in fin_bb. We need to create
7246 appropriate phi nodes in iter_part_bb instead. */
7247 se = single_pred_edge (fin_bb);
7248 re = single_succ_edge (trip_update_bb);
7249 vec<edge_var_map> *head = redirect_edge_var_map_vector (re);
7250 ene = single_succ_edge (entry_bb);
7252 psi = gsi_start_phis (fin_bb);
7253 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
7254 gsi_next (&psi), ++i)
7256 gphi *nphi;
7257 source_location locus;
7259 phi = psi.phi ();
7260 t = gimple_phi_result (phi);
7261 gcc_assert (t == redirect_edge_var_map_result (vm));
7262 nphi = create_phi_node (t, iter_part_bb);
7264 t = PHI_ARG_DEF_FROM_EDGE (phi, se);
7265 locus = gimple_phi_arg_location_from_edge (phi, se);
7267 /* A special case -- fd->loop.v is not yet computed in
7268 iter_part_bb, we need to use vextra instead. */
7269 if (t == fd->loop.v)
7270 t = vextra;
7271 add_phi_arg (nphi, t, ene, locus);
7272 locus = redirect_edge_var_map_location (vm);
7273 add_phi_arg (nphi, redirect_edge_var_map_def (vm), re, locus);
7275 gcc_assert (gsi_end_p (psi) && i == head->length ());
7276 redirect_edge_var_map_clear (re);
7277 while (1)
7279 psi = gsi_start_phis (fin_bb);
7280 if (gsi_end_p (psi))
7281 break;
7282 remove_phi_node (&psi, false);
7285 /* Make phi node for trip. */
7286 phi = create_phi_node (trip_main, iter_part_bb);
7287 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb),
7288 UNKNOWN_LOCATION);
7289 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb),
7290 UNKNOWN_LOCATION);
7293 if (!broken_loop)
7294 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
7295 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
7296 recompute_dominator (CDI_DOMINATORS, iter_part_bb));
7297 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
7298 recompute_dominator (CDI_DOMINATORS, fin_bb));
7299 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
7300 recompute_dominator (CDI_DOMINATORS, seq_start_bb));
7301 set_immediate_dominator (CDI_DOMINATORS, body_bb,
7302 recompute_dominator (CDI_DOMINATORS, body_bb));
7304 if (!broken_loop)
7306 struct loop *trip_loop = alloc_loop ();
7307 trip_loop->header = iter_part_bb;
7308 trip_loop->latch = trip_update_bb;
7309 add_loop (trip_loop, iter_part_bb->loop_father);
7311 if (!gimple_omp_for_combined_p (fd->for_stmt))
7313 struct loop *loop = alloc_loop ();
7314 loop->header = body_bb;
7315 if (collapse_bb == NULL)
7316 loop->latch = cont_bb;
7317 add_loop (loop, trip_loop);
7322 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7323 Given parameters:
7324 for (V = N1; V cond N2; V += STEP) BODY;
7326 where COND is "<" or ">" or "!=", we generate pseudocode
7328 for (ind_var = low; ind_var < high; ind_var++)
7330 V = n1 + (ind_var * STEP)
7332 <BODY>
7335 In the above pseudocode, low and high are function parameters of the
7336 child function. In the function below, we are inserting a temp.
7337 variable that will be making a call to two OMP functions that will not be
7338 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7339 with _Cilk_for). These functions are replaced with low and high
7340 by the function that handles taskreg. */
7343 static void
7344 expand_cilk_for (struct omp_region *region, struct omp_for_data *fd)
7346 bool broken_loop = region->cont == NULL;
7347 basic_block entry_bb = region->entry;
7348 basic_block cont_bb = region->cont;
7350 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7351 gcc_assert (broken_loop
7352 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
7353 basic_block l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
7354 basic_block l1_bb, l2_bb;
7356 if (!broken_loop)
7358 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
7359 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
7360 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
7361 l2_bb = BRANCH_EDGE (entry_bb)->dest;
7363 else
7365 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
7366 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
7367 l2_bb = single_succ (l1_bb);
7369 basic_block exit_bb = region->exit;
7370 basic_block l2_dom_bb = NULL;
7372 gimple_stmt_iterator gsi = gsi_last_bb (entry_bb);
7374 /* Below statements until the "tree high_val = ..." are pseudo statements
7375 used to pass information to be used by expand_omp_taskreg.
7376 low_val and high_val will be replaced by the __low and __high
7377 parameter from the child function.
7379 The call_exprs part is a place-holder, it is mainly used
7380 to distinctly identify to the top-level part that this is
7381 where we should put low and high (reasoning given in header
7382 comment). */
7384 tree child_fndecl
7385 = gimple_omp_parallel_child_fn (
7386 as_a <gomp_parallel *> (last_stmt (region->outer->entry)));
7387 tree t, low_val = NULL_TREE, high_val = NULL_TREE;
7388 for (t = DECL_ARGUMENTS (child_fndecl); t; t = TREE_CHAIN (t))
7390 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__high"))
7391 high_val = t;
7392 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__low"))
7393 low_val = t;
7395 gcc_assert (low_val && high_val);
7397 tree type = TREE_TYPE (low_val);
7398 tree ind_var = create_tmp_reg (type, "__cilk_ind_var");
7399 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
7401 /* Not needed in SSA form right now. */
7402 gcc_assert (!gimple_in_ssa_p (cfun));
7403 if (l2_dom_bb == NULL)
7404 l2_dom_bb = l1_bb;
7406 tree n1 = low_val;
7407 tree n2 = high_val;
7409 gimple stmt = gimple_build_assign (ind_var, n1);
7411 /* Replace the GIMPLE_OMP_FOR statement. */
7412 gsi_replace (&gsi, stmt, true);
7414 if (!broken_loop)
7416 /* Code to control the increment goes in the CONT_BB. */
7417 gsi = gsi_last_bb (cont_bb);
7418 stmt = gsi_stmt (gsi);
7419 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
7420 stmt = gimple_build_assign (ind_var, PLUS_EXPR, ind_var,
7421 build_one_cst (type));
7423 /* Replace GIMPLE_OMP_CONTINUE. */
7424 gsi_replace (&gsi, stmt, true);
7427 /* Emit the condition in L1_BB. */
7428 gsi = gsi_after_labels (l1_bb);
7429 t = fold_build2 (MULT_EXPR, TREE_TYPE (fd->loop.step),
7430 fold_convert (TREE_TYPE (fd->loop.step), ind_var),
7431 fd->loop.step);
7432 if (POINTER_TYPE_P (TREE_TYPE (fd->loop.n1)))
7433 t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (fd->loop.n1),
7434 fd->loop.n1, fold_convert (sizetype, t));
7435 else
7436 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loop.n1),
7437 fd->loop.n1, fold_convert (TREE_TYPE (fd->loop.n1), t));
7438 t = fold_convert (TREE_TYPE (fd->loop.v), t);
7439 expand_omp_build_assign (&gsi, fd->loop.v, t);
7441 /* The condition is always '<' since the runtime will fill in the low
7442 and high values. */
7443 stmt = gimple_build_cond (LT_EXPR, ind_var, n2, NULL_TREE, NULL_TREE);
7444 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
7446 /* Remove GIMPLE_OMP_RETURN. */
7447 gsi = gsi_last_bb (exit_bb);
7448 gsi_remove (&gsi, true);
7450 /* Connect the new blocks. */
7451 remove_edge (FALLTHRU_EDGE (entry_bb));
7453 edge e, ne;
7454 if (!broken_loop)
7456 remove_edge (BRANCH_EDGE (entry_bb));
7457 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
7459 e = BRANCH_EDGE (l1_bb);
7460 ne = FALLTHRU_EDGE (l1_bb);
7461 e->flags = EDGE_TRUE_VALUE;
7463 else
7465 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7467 ne = single_succ_edge (l1_bb);
7468 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
7471 ne->flags = EDGE_FALSE_VALUE;
7472 e->probability = REG_BR_PROB_BASE * 7 / 8;
7473 ne->probability = REG_BR_PROB_BASE / 8;
7475 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
7476 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
7477 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
7479 if (!broken_loop)
7481 struct loop *loop = alloc_loop ();
7482 loop->header = l1_bb;
7483 loop->latch = cont_bb;
7484 add_loop (loop, l1_bb->loop_father);
7485 loop->safelen = INT_MAX;
7488 /* Pick the correct library function based on the precision of the
7489 induction variable type. */
7490 tree lib_fun = NULL_TREE;
7491 if (TYPE_PRECISION (type) == 32)
7492 lib_fun = cilk_for_32_fndecl;
7493 else if (TYPE_PRECISION (type) == 64)
7494 lib_fun = cilk_for_64_fndecl;
7495 else
7496 gcc_unreachable ();
7498 gcc_assert (fd->sched_kind == OMP_CLAUSE_SCHEDULE_CILKFOR);
7500 /* WS_ARGS contains the library function flavor to call:
7501 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7502 user-defined grain value. If the user does not define one, then zero
7503 is passed in by the parser. */
7504 vec_alloc (region->ws_args, 2);
7505 region->ws_args->quick_push (lib_fun);
7506 region->ws_args->quick_push (fd->chunk_size);
7509 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7510 loop. Given parameters:
7512 for (V = N1; V cond N2; V += STEP) BODY;
7514 where COND is "<" or ">", we generate pseudocode
7516 V = N1;
7517 goto L1;
7519 BODY;
7520 V += STEP;
7522 if (V cond N2) goto L0; else goto L2;
7525 For collapsed loops, given parameters:
7526 collapse(3)
7527 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7528 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7529 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7530 BODY;
7532 we generate pseudocode
7534 if (cond3 is <)
7535 adj = STEP3 - 1;
7536 else
7537 adj = STEP3 + 1;
7538 count3 = (adj + N32 - N31) / STEP3;
7539 if (cond2 is <)
7540 adj = STEP2 - 1;
7541 else
7542 adj = STEP2 + 1;
7543 count2 = (adj + N22 - N21) / STEP2;
7544 if (cond1 is <)
7545 adj = STEP1 - 1;
7546 else
7547 adj = STEP1 + 1;
7548 count1 = (adj + N12 - N11) / STEP1;
7549 count = count1 * count2 * count3;
7550 V = 0;
7551 V1 = N11;
7552 V2 = N21;
7553 V3 = N31;
7554 goto L1;
7556 BODY;
7557 V += 1;
7558 V3 += STEP3;
7559 V2 += (V3 cond3 N32) ? 0 : STEP2;
7560 V3 = (V3 cond3 N32) ? V3 : N31;
7561 V1 += (V2 cond2 N22) ? 0 : STEP1;
7562 V2 = (V2 cond2 N22) ? V2 : N21;
7564 if (V < count) goto L0; else goto L2;
7569 static void
7570 expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
7572 tree type, t;
7573 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb;
7574 gimple_stmt_iterator gsi;
7575 gimple stmt;
7576 gcond *cond_stmt;
7577 bool broken_loop = region->cont == NULL;
7578 edge e, ne;
7579 tree *counts = NULL;
7580 int i;
7581 tree safelen = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7582 OMP_CLAUSE_SAFELEN);
7583 tree simduid = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7584 OMP_CLAUSE__SIMDUID_);
7585 tree n1, n2;
7587 type = TREE_TYPE (fd->loop.v);
7588 entry_bb = region->entry;
7589 cont_bb = region->cont;
7590 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7591 gcc_assert (broken_loop
7592 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
7593 l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
7594 if (!broken_loop)
7596 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
7597 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
7598 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
7599 l2_bb = BRANCH_EDGE (entry_bb)->dest;
7601 else
7603 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
7604 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
7605 l2_bb = single_succ (l1_bb);
7607 exit_bb = region->exit;
7608 l2_dom_bb = NULL;
7610 gsi = gsi_last_bb (entry_bb);
7612 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
7613 /* Not needed in SSA form right now. */
7614 gcc_assert (!gimple_in_ssa_p (cfun));
7615 if (fd->collapse > 1)
7617 int first_zero_iter = -1;
7618 basic_block zero_iter_bb = l2_bb;
7620 counts = XALLOCAVEC (tree, fd->collapse);
7621 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
7622 zero_iter_bb, first_zero_iter,
7623 l2_dom_bb);
7625 if (l2_dom_bb == NULL)
7626 l2_dom_bb = l1_bb;
7628 n1 = fd->loop.n1;
7629 n2 = fd->loop.n2;
7630 if (gimple_omp_for_combined_into_p (fd->for_stmt))
7632 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7633 OMP_CLAUSE__LOOPTEMP_);
7634 gcc_assert (innerc);
7635 n1 = OMP_CLAUSE_DECL (innerc);
7636 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7637 OMP_CLAUSE__LOOPTEMP_);
7638 gcc_assert (innerc);
7639 n2 = OMP_CLAUSE_DECL (innerc);
7640 expand_omp_build_assign (&gsi, fd->loop.v,
7641 fold_convert (type, n1));
7642 if (fd->collapse > 1)
7644 gsi_prev (&gsi);
7645 expand_omp_for_init_vars (fd, &gsi, counts, NULL, n1);
7646 gsi_next (&gsi);
7649 else
7651 expand_omp_build_assign (&gsi, fd->loop.v,
7652 fold_convert (type, fd->loop.n1));
7653 if (fd->collapse > 1)
7654 for (i = 0; i < fd->collapse; i++)
7656 tree itype = TREE_TYPE (fd->loops[i].v);
7657 if (POINTER_TYPE_P (itype))
7658 itype = signed_type_for (itype);
7659 t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1);
7660 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
7664 /* Remove the GIMPLE_OMP_FOR statement. */
7665 gsi_remove (&gsi, true);
7667 if (!broken_loop)
7669 /* Code to control the increment goes in the CONT_BB. */
7670 gsi = gsi_last_bb (cont_bb);
7671 stmt = gsi_stmt (gsi);
7672 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
7674 if (POINTER_TYPE_P (type))
7675 t = fold_build_pointer_plus (fd->loop.v, fd->loop.step);
7676 else
7677 t = fold_build2 (PLUS_EXPR, type, fd->loop.v, fd->loop.step);
7678 expand_omp_build_assign (&gsi, fd->loop.v, t);
7680 if (fd->collapse > 1)
7682 i = fd->collapse - 1;
7683 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
7685 t = fold_convert (sizetype, fd->loops[i].step);
7686 t = fold_build_pointer_plus (fd->loops[i].v, t);
7688 else
7690 t = fold_convert (TREE_TYPE (fd->loops[i].v),
7691 fd->loops[i].step);
7692 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
7693 fd->loops[i].v, t);
7695 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
7697 for (i = fd->collapse - 1; i > 0; i--)
7699 tree itype = TREE_TYPE (fd->loops[i].v);
7700 tree itype2 = TREE_TYPE (fd->loops[i - 1].v);
7701 if (POINTER_TYPE_P (itype2))
7702 itype2 = signed_type_for (itype2);
7703 t = build3 (COND_EXPR, itype2,
7704 build2 (fd->loops[i].cond_code, boolean_type_node,
7705 fd->loops[i].v,
7706 fold_convert (itype, fd->loops[i].n2)),
7707 build_int_cst (itype2, 0),
7708 fold_convert (itype2, fd->loops[i - 1].step));
7709 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i - 1].v)))
7710 t = fold_build_pointer_plus (fd->loops[i - 1].v, t);
7711 else
7712 t = fold_build2 (PLUS_EXPR, itype2, fd->loops[i - 1].v, t);
7713 expand_omp_build_assign (&gsi, fd->loops[i - 1].v, t);
7715 t = build3 (COND_EXPR, itype,
7716 build2 (fd->loops[i].cond_code, boolean_type_node,
7717 fd->loops[i].v,
7718 fold_convert (itype, fd->loops[i].n2)),
7719 fd->loops[i].v,
7720 fold_convert (itype, fd->loops[i].n1));
7721 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
7725 /* Remove GIMPLE_OMP_CONTINUE. */
7726 gsi_remove (&gsi, true);
7729 /* Emit the condition in L1_BB. */
7730 gsi = gsi_start_bb (l1_bb);
7732 t = fold_convert (type, n2);
7733 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7734 false, GSI_CONTINUE_LINKING);
7735 t = build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t);
7736 cond_stmt = gimple_build_cond_empty (t);
7737 gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING);
7738 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p,
7739 NULL, NULL)
7740 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p,
7741 NULL, NULL))
7743 gsi = gsi_for_stmt (cond_stmt);
7744 gimple_regimplify_operands (cond_stmt, &gsi);
7747 /* Remove GIMPLE_OMP_RETURN. */
7748 gsi = gsi_last_bb (exit_bb);
7749 gsi_remove (&gsi, true);
7751 /* Connect the new blocks. */
7752 remove_edge (FALLTHRU_EDGE (entry_bb));
7754 if (!broken_loop)
7756 remove_edge (BRANCH_EDGE (entry_bb));
7757 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
7759 e = BRANCH_EDGE (l1_bb);
7760 ne = FALLTHRU_EDGE (l1_bb);
7761 e->flags = EDGE_TRUE_VALUE;
7763 else
7765 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7767 ne = single_succ_edge (l1_bb);
7768 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
7771 ne->flags = EDGE_FALSE_VALUE;
7772 e->probability = REG_BR_PROB_BASE * 7 / 8;
7773 ne->probability = REG_BR_PROB_BASE / 8;
7775 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
7776 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
7777 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
7779 if (!broken_loop)
7781 struct loop *loop = alloc_loop ();
7782 loop->header = l1_bb;
7783 loop->latch = cont_bb;
7784 add_loop (loop, l1_bb->loop_father);
7785 if (safelen == NULL_TREE)
7786 loop->safelen = INT_MAX;
7787 else
7789 safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen);
7790 if (TREE_CODE (safelen) != INTEGER_CST)
7791 loop->safelen = 0;
7792 else if (!tree_fits_uhwi_p (safelen)
7793 || tree_to_uhwi (safelen) > INT_MAX)
7794 loop->safelen = INT_MAX;
7795 else
7796 loop->safelen = tree_to_uhwi (safelen);
7797 if (loop->safelen == 1)
7798 loop->safelen = 0;
7800 if (simduid)
7802 loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
7803 cfun->has_simduid_loops = true;
7805 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7806 the loop. */
7807 if ((flag_tree_loop_vectorize
7808 || (!global_options_set.x_flag_tree_loop_vectorize
7809 && !global_options_set.x_flag_tree_vectorize))
7810 && flag_tree_loop_optimize
7811 && loop->safelen > 1)
7813 loop->force_vectorize = true;
7814 cfun->has_force_vectorize_loops = true;
7820 /* Expand the OMP loop defined by REGION. */
7822 static void
7823 expand_omp_for (struct omp_region *region, gimple inner_stmt)
7825 struct omp_for_data fd;
7826 struct omp_for_data_loop *loops;
7828 loops
7829 = (struct omp_for_data_loop *)
7830 alloca (gimple_omp_for_collapse (last_stmt (region->entry))
7831 * sizeof (struct omp_for_data_loop));
7832 extract_omp_for_data (as_a <gomp_for *> (last_stmt (region->entry)),
7833 &fd, loops);
7834 region->sched_kind = fd.sched_kind;
7836 gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
7837 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
7838 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
7839 if (region->cont)
7841 gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
7842 BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
7843 FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
7845 else
7846 /* If there isn't a continue then this is a degerate case where
7847 the introduction of abnormal edges during lowering will prevent
7848 original loops from being detected. Fix that up. */
7849 loops_state_set (LOOPS_NEED_FIXUP);
7851 if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD)
7852 expand_omp_simd (region, &fd);
7853 else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_CILKFOR)
7854 expand_cilk_for (region, &fd);
7855 else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
7856 && !fd.have_ordered)
7858 if (fd.chunk_size == NULL)
7859 expand_omp_for_static_nochunk (region, &fd, inner_stmt);
7860 else
7861 expand_omp_for_static_chunk (region, &fd, inner_stmt);
7863 else
7865 int fn_index, start_ix, next_ix;
7867 gcc_assert (gimple_omp_for_kind (fd.for_stmt)
7868 == GF_OMP_FOR_KIND_FOR);
7869 if (fd.chunk_size == NULL
7870 && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
7871 fd.chunk_size = integer_zero_node;
7872 gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
7873 fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
7874 ? 3 : fd.sched_kind;
7875 fn_index += fd.have_ordered * 4;
7876 start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
7877 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
7878 if (fd.iter_type == long_long_unsigned_type_node)
7880 start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7881 - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
7882 next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7883 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
7885 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
7886 (enum built_in_function) next_ix, inner_stmt);
7889 if (gimple_in_ssa_p (cfun))
7890 update_ssa (TODO_update_ssa_only_virtuals);
7894 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7896 v = GOMP_sections_start (n);
7898 switch (v)
7900 case 0:
7901 goto L2;
7902 case 1:
7903 section 1;
7904 goto L1;
7905 case 2:
7907 case n:
7909 default:
7910 abort ();
7913 v = GOMP_sections_next ();
7914 goto L0;
7916 reduction;
7918 If this is a combined parallel sections, replace the call to
7919 GOMP_sections_start with call to GOMP_sections_next. */
7921 static void
7922 expand_omp_sections (struct omp_region *region)
7924 tree t, u, vin = NULL, vmain, vnext, l2;
7925 unsigned len;
7926 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
7927 gimple_stmt_iterator si, switch_si;
7928 gomp_sections *sections_stmt;
7929 gimple stmt;
7930 gomp_continue *cont;
7931 edge_iterator ei;
7932 edge e;
7933 struct omp_region *inner;
7934 unsigned i, casei;
7935 bool exit_reachable = region->cont != NULL;
7937 gcc_assert (region->exit != NULL);
7938 entry_bb = region->entry;
7939 l0_bb = single_succ (entry_bb);
7940 l1_bb = region->cont;
7941 l2_bb = region->exit;
7942 if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb)
7943 l2 = gimple_block_label (l2_bb);
7944 else
7946 /* This can happen if there are reductions. */
7947 len = EDGE_COUNT (l0_bb->succs);
7948 gcc_assert (len > 0);
7949 e = EDGE_SUCC (l0_bb, len - 1);
7950 si = gsi_last_bb (e->dest);
7951 l2 = NULL_TREE;
7952 if (gsi_end_p (si)
7953 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
7954 l2 = gimple_block_label (e->dest);
7955 else
7956 FOR_EACH_EDGE (e, ei, l0_bb->succs)
7958 si = gsi_last_bb (e->dest);
7959 if (gsi_end_p (si)
7960 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
7962 l2 = gimple_block_label (e->dest);
7963 break;
7967 if (exit_reachable)
7968 default_bb = create_empty_bb (l1_bb->prev_bb);
7969 else
7970 default_bb = create_empty_bb (l0_bb);
7972 /* We will build a switch() with enough cases for all the
7973 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7974 and a default case to abort if something goes wrong. */
7975 len = EDGE_COUNT (l0_bb->succs);
7977 /* Use vec::quick_push on label_vec throughout, since we know the size
7978 in advance. */
7979 auto_vec<tree> label_vec (len);
7981 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7982 GIMPLE_OMP_SECTIONS statement. */
7983 si = gsi_last_bb (entry_bb);
7984 sections_stmt = as_a <gomp_sections *> (gsi_stmt (si));
7985 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
7986 vin = gimple_omp_sections_control (sections_stmt);
7987 if (!is_combined_parallel (region))
7989 /* If we are not inside a combined parallel+sections region,
7990 call GOMP_sections_start. */
7991 t = build_int_cst (unsigned_type_node, len - 1);
7992 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
7993 stmt = gimple_build_call (u, 1, t);
7995 else
7997 /* Otherwise, call GOMP_sections_next. */
7998 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
7999 stmt = gimple_build_call (u, 0);
8001 gimple_call_set_lhs (stmt, vin);
8002 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8003 gsi_remove (&si, true);
8005 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8006 L0_BB. */
8007 switch_si = gsi_last_bb (l0_bb);
8008 gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH);
8009 if (exit_reachable)
8011 cont = as_a <gomp_continue *> (last_stmt (l1_bb));
8012 gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE);
8013 vmain = gimple_omp_continue_control_use (cont);
8014 vnext = gimple_omp_continue_control_def (cont);
8016 else
8018 vmain = vin;
8019 vnext = NULL_TREE;
8022 t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2);
8023 label_vec.quick_push (t);
8024 i = 1;
8026 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8027 for (inner = region->inner, casei = 1;
8028 inner;
8029 inner = inner->next, i++, casei++)
8031 basic_block s_entry_bb, s_exit_bb;
8033 /* Skip optional reduction region. */
8034 if (inner->type == GIMPLE_OMP_ATOMIC_LOAD)
8036 --i;
8037 --casei;
8038 continue;
8041 s_entry_bb = inner->entry;
8042 s_exit_bb = inner->exit;
8044 t = gimple_block_label (s_entry_bb);
8045 u = build_int_cst (unsigned_type_node, casei);
8046 u = build_case_label (u, NULL, t);
8047 label_vec.quick_push (u);
8049 si = gsi_last_bb (s_entry_bb);
8050 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION);
8051 gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si)));
8052 gsi_remove (&si, true);
8053 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
8055 if (s_exit_bb == NULL)
8056 continue;
8058 si = gsi_last_bb (s_exit_bb);
8059 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
8060 gsi_remove (&si, true);
8062 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
8065 /* Error handling code goes in DEFAULT_BB. */
8066 t = gimple_block_label (default_bb);
8067 u = build_case_label (NULL, NULL, t);
8068 make_edge (l0_bb, default_bb, 0);
8069 add_bb_to_loop (default_bb, current_loops->tree_root);
8071 stmt = gimple_build_switch (vmain, u, label_vec);
8072 gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT);
8073 gsi_remove (&switch_si, true);
8075 si = gsi_start_bb (default_bb);
8076 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
8077 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
8079 if (exit_reachable)
8081 tree bfn_decl;
8083 /* Code to get the next section goes in L1_BB. */
8084 si = gsi_last_bb (l1_bb);
8085 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
8087 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
8088 stmt = gimple_build_call (bfn_decl, 0);
8089 gimple_call_set_lhs (stmt, vnext);
8090 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8091 gsi_remove (&si, true);
8093 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
8096 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8097 si = gsi_last_bb (l2_bb);
8098 if (gimple_omp_return_nowait_p (gsi_stmt (si)))
8099 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT);
8100 else if (gimple_omp_return_lhs (gsi_stmt (si)))
8101 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL);
8102 else
8103 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END);
8104 stmt = gimple_build_call (t, 0);
8105 if (gimple_omp_return_lhs (gsi_stmt (si)))
8106 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si)));
8107 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8108 gsi_remove (&si, true);
8110 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
8114 /* Expand code for an OpenMP single directive. We've already expanded
8115 much of the code, here we simply place the GOMP_barrier call. */
8117 static void
8118 expand_omp_single (struct omp_region *region)
8120 basic_block entry_bb, exit_bb;
8121 gimple_stmt_iterator si;
8123 entry_bb = region->entry;
8124 exit_bb = region->exit;
8126 si = gsi_last_bb (entry_bb);
8127 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE);
8128 gsi_remove (&si, true);
8129 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8131 si = gsi_last_bb (exit_bb);
8132 if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
8134 tree t = gimple_omp_return_lhs (gsi_stmt (si));
8135 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT);
8137 gsi_remove (&si, true);
8138 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
8142 /* Generic expansion for OpenMP synchronization directives: master,
8143 ordered and critical. All we need to do here is remove the entry
8144 and exit markers for REGION. */
8146 static void
8147 expand_omp_synch (struct omp_region *region)
8149 basic_block entry_bb, exit_bb;
8150 gimple_stmt_iterator si;
8152 entry_bb = region->entry;
8153 exit_bb = region->exit;
8155 si = gsi_last_bb (entry_bb);
8156 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
8157 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
8158 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
8159 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
8160 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
8161 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS);
8162 gsi_remove (&si, true);
8163 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8165 if (exit_bb)
8167 si = gsi_last_bb (exit_bb);
8168 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
8169 gsi_remove (&si, true);
8170 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
8174 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8175 operation as a normal volatile load. */
8177 static bool
8178 expand_omp_atomic_load (basic_block load_bb, tree addr,
8179 tree loaded_val, int index)
8181 enum built_in_function tmpbase;
8182 gimple_stmt_iterator gsi;
8183 basic_block store_bb;
8184 location_t loc;
8185 gimple stmt;
8186 tree decl, call, type, itype;
8188 gsi = gsi_last_bb (load_bb);
8189 stmt = gsi_stmt (gsi);
8190 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
8191 loc = gimple_location (stmt);
8193 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8194 is smaller than word size, then expand_atomic_load assumes that the load
8195 is atomic. We could avoid the builtin entirely in this case. */
8197 tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
8198 decl = builtin_decl_explicit (tmpbase);
8199 if (decl == NULL_TREE)
8200 return false;
8202 type = TREE_TYPE (loaded_val);
8203 itype = TREE_TYPE (TREE_TYPE (decl));
8205 call = build_call_expr_loc (loc, decl, 2, addr,
8206 build_int_cst (NULL,
8207 gimple_omp_atomic_seq_cst_p (stmt)
8208 ? MEMMODEL_SEQ_CST
8209 : MEMMODEL_RELAXED));
8210 if (!useless_type_conversion_p (type, itype))
8211 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
8212 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
8214 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
8215 gsi_remove (&gsi, true);
8217 store_bb = single_succ (load_bb);
8218 gsi = gsi_last_bb (store_bb);
8219 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
8220 gsi_remove (&gsi, true);
8222 if (gimple_in_ssa_p (cfun))
8223 update_ssa (TODO_update_ssa_no_phi);
8225 return true;
8228 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8229 operation as a normal volatile store. */
8231 static bool
8232 expand_omp_atomic_store (basic_block load_bb, tree addr,
8233 tree loaded_val, tree stored_val, int index)
8235 enum built_in_function tmpbase;
8236 gimple_stmt_iterator gsi;
8237 basic_block store_bb = single_succ (load_bb);
8238 location_t loc;
8239 gimple stmt;
8240 tree decl, call, type, itype;
8241 machine_mode imode;
8242 bool exchange;
8244 gsi = gsi_last_bb (load_bb);
8245 stmt = gsi_stmt (gsi);
8246 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
8248 /* If the load value is needed, then this isn't a store but an exchange. */
8249 exchange = gimple_omp_atomic_need_value_p (stmt);
8251 gsi = gsi_last_bb (store_bb);
8252 stmt = gsi_stmt (gsi);
8253 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE);
8254 loc = gimple_location (stmt);
8256 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8257 is smaller than word size, then expand_atomic_store assumes that the store
8258 is atomic. We could avoid the builtin entirely in this case. */
8260 tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N);
8261 tmpbase = (enum built_in_function) ((int) tmpbase + index + 1);
8262 decl = builtin_decl_explicit (tmpbase);
8263 if (decl == NULL_TREE)
8264 return false;
8266 type = TREE_TYPE (stored_val);
8268 /* Dig out the type of the function's second argument. */
8269 itype = TREE_TYPE (decl);
8270 itype = TYPE_ARG_TYPES (itype);
8271 itype = TREE_CHAIN (itype);
8272 itype = TREE_VALUE (itype);
8273 imode = TYPE_MODE (itype);
8275 if (exchange && !can_atomic_exchange_p (imode, true))
8276 return false;
8278 if (!useless_type_conversion_p (itype, type))
8279 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
8280 call = build_call_expr_loc (loc, decl, 3, addr, stored_val,
8281 build_int_cst (NULL,
8282 gimple_omp_atomic_seq_cst_p (stmt)
8283 ? MEMMODEL_SEQ_CST
8284 : MEMMODEL_RELAXED));
8285 if (exchange)
8287 if (!useless_type_conversion_p (type, itype))
8288 call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
8289 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
8292 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
8293 gsi_remove (&gsi, true);
8295 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8296 gsi = gsi_last_bb (load_bb);
8297 gsi_remove (&gsi, true);
8299 if (gimple_in_ssa_p (cfun))
8300 update_ssa (TODO_update_ssa_no_phi);
8302 return true;
8305 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8306 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8307 size of the data type, and thus usable to find the index of the builtin
8308 decl. Returns false if the expression is not of the proper form. */
8310 static bool
8311 expand_omp_atomic_fetch_op (basic_block load_bb,
8312 tree addr, tree loaded_val,
8313 tree stored_val, int index)
8315 enum built_in_function oldbase, newbase, tmpbase;
8316 tree decl, itype, call;
8317 tree lhs, rhs;
8318 basic_block store_bb = single_succ (load_bb);
8319 gimple_stmt_iterator gsi;
8320 gimple stmt;
8321 location_t loc;
8322 enum tree_code code;
8323 bool need_old, need_new;
8324 machine_mode imode;
8325 bool seq_cst;
8327 /* We expect to find the following sequences:
8329 load_bb:
8330 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8332 store_bb:
8333 val = tmp OP something; (or: something OP tmp)
8334 GIMPLE_OMP_STORE (val)
8336 ???FIXME: Allow a more flexible sequence.
8337 Perhaps use data flow to pick the statements.
8341 gsi = gsi_after_labels (store_bb);
8342 stmt = gsi_stmt (gsi);
8343 loc = gimple_location (stmt);
8344 if (!is_gimple_assign (stmt))
8345 return false;
8346 gsi_next (&gsi);
8347 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
8348 return false;
8349 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
8350 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb));
8351 seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb));
8352 gcc_checking_assert (!need_old || !need_new);
8354 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
8355 return false;
8357 /* Check for one of the supported fetch-op operations. */
8358 code = gimple_assign_rhs_code (stmt);
8359 switch (code)
8361 case PLUS_EXPR:
8362 case POINTER_PLUS_EXPR:
8363 oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N;
8364 newbase = BUILT_IN_ATOMIC_ADD_FETCH_N;
8365 break;
8366 case MINUS_EXPR:
8367 oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N;
8368 newbase = BUILT_IN_ATOMIC_SUB_FETCH_N;
8369 break;
8370 case BIT_AND_EXPR:
8371 oldbase = BUILT_IN_ATOMIC_FETCH_AND_N;
8372 newbase = BUILT_IN_ATOMIC_AND_FETCH_N;
8373 break;
8374 case BIT_IOR_EXPR:
8375 oldbase = BUILT_IN_ATOMIC_FETCH_OR_N;
8376 newbase = BUILT_IN_ATOMIC_OR_FETCH_N;
8377 break;
8378 case BIT_XOR_EXPR:
8379 oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N;
8380 newbase = BUILT_IN_ATOMIC_XOR_FETCH_N;
8381 break;
8382 default:
8383 return false;
8386 /* Make sure the expression is of the proper form. */
8387 if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
8388 rhs = gimple_assign_rhs2 (stmt);
8389 else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
8390 && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
8391 rhs = gimple_assign_rhs1 (stmt);
8392 else
8393 return false;
8395 tmpbase = ((enum built_in_function)
8396 ((need_new ? newbase : oldbase) + index + 1));
8397 decl = builtin_decl_explicit (tmpbase);
8398 if (decl == NULL_TREE)
8399 return false;
8400 itype = TREE_TYPE (TREE_TYPE (decl));
8401 imode = TYPE_MODE (itype);
8403 /* We could test all of the various optabs involved, but the fact of the
8404 matter is that (with the exception of i486 vs i586 and xadd) all targets
8405 that support any atomic operaton optab also implements compare-and-swap.
8406 Let optabs.c take care of expanding any compare-and-swap loop. */
8407 if (!can_compare_and_swap_p (imode, true))
8408 return false;
8410 gsi = gsi_last_bb (load_bb);
8411 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
8413 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8414 It only requires that the operation happen atomically. Thus we can
8415 use the RELAXED memory model. */
8416 call = build_call_expr_loc (loc, decl, 3, addr,
8417 fold_convert_loc (loc, itype, rhs),
8418 build_int_cst (NULL,
8419 seq_cst ? MEMMODEL_SEQ_CST
8420 : MEMMODEL_RELAXED));
8422 if (need_old || need_new)
8424 lhs = need_old ? loaded_val : stored_val;
8425 call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
8426 call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call);
8428 else
8429 call = fold_convert_loc (loc, void_type_node, call);
8430 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
8431 gsi_remove (&gsi, true);
8433 gsi = gsi_last_bb (store_bb);
8434 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
8435 gsi_remove (&gsi, true);
8436 gsi = gsi_last_bb (store_bb);
8437 gsi_remove (&gsi, true);
8439 if (gimple_in_ssa_p (cfun))
8440 update_ssa (TODO_update_ssa_no_phi);
8442 return true;
8445 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8447 oldval = *addr;
8448 repeat:
8449 newval = rhs; // with oldval replacing *addr in rhs
8450 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8451 if (oldval != newval)
8452 goto repeat;
8454 INDEX is log2 of the size of the data type, and thus usable to find the
8455 index of the builtin decl. */
8457 static bool
8458 expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
8459 tree addr, tree loaded_val, tree stored_val,
8460 int index)
8462 tree loadedi, storedi, initial, new_storedi, old_vali;
8463 tree type, itype, cmpxchg, iaddr;
8464 gimple_stmt_iterator si;
8465 basic_block loop_header = single_succ (load_bb);
8466 gimple phi, stmt;
8467 edge e;
8468 enum built_in_function fncode;
8470 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8471 order to use the RELAXED memory model effectively. */
8472 fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8473 + index + 1);
8474 cmpxchg = builtin_decl_explicit (fncode);
8475 if (cmpxchg == NULL_TREE)
8476 return false;
8477 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
8478 itype = TREE_TYPE (TREE_TYPE (cmpxchg));
8480 if (!can_compare_and_swap_p (TYPE_MODE (itype), true))
8481 return false;
8483 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8484 si = gsi_last_bb (load_bb);
8485 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
8487 /* For floating-point values, we'll need to view-convert them to integers
8488 so that we can perform the atomic compare and swap. Simplify the
8489 following code by always setting up the "i"ntegral variables. */
8490 if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
8492 tree iaddr_val;
8494 iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode,
8495 true));
8496 iaddr_val
8497 = force_gimple_operand_gsi (&si,
8498 fold_convert (TREE_TYPE (iaddr), addr),
8499 false, NULL_TREE, true, GSI_SAME_STMT);
8500 stmt = gimple_build_assign (iaddr, iaddr_val);
8501 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8502 loadedi = create_tmp_var (itype);
8503 if (gimple_in_ssa_p (cfun))
8504 loadedi = make_ssa_name (loadedi);
8506 else
8508 iaddr = addr;
8509 loadedi = loaded_val;
8512 fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
8513 tree loaddecl = builtin_decl_explicit (fncode);
8514 if (loaddecl)
8515 initial
8516 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr)),
8517 build_call_expr (loaddecl, 2, iaddr,
8518 build_int_cst (NULL_TREE,
8519 MEMMODEL_RELAXED)));
8520 else
8521 initial = build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)), iaddr,
8522 build_int_cst (TREE_TYPE (iaddr), 0));
8524 initial
8525 = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true,
8526 GSI_SAME_STMT);
8528 /* Move the value to the LOADEDI temporary. */
8529 if (gimple_in_ssa_p (cfun))
8531 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
8532 phi = create_phi_node (loadedi, loop_header);
8533 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
8534 initial);
8536 else
8537 gsi_insert_before (&si,
8538 gimple_build_assign (loadedi, initial),
8539 GSI_SAME_STMT);
8540 if (loadedi != loaded_val)
8542 gimple_stmt_iterator gsi2;
8543 tree x;
8545 x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
8546 gsi2 = gsi_start_bb (loop_header);
8547 if (gimple_in_ssa_p (cfun))
8549 gassign *stmt;
8550 x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
8551 true, GSI_SAME_STMT);
8552 stmt = gimple_build_assign (loaded_val, x);
8553 gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
8555 else
8557 x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
8558 force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
8559 true, GSI_SAME_STMT);
8562 gsi_remove (&si, true);
8564 si = gsi_last_bb (store_bb);
8565 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
8567 if (iaddr == addr)
8568 storedi = stored_val;
8569 else
8570 storedi =
8571 force_gimple_operand_gsi (&si,
8572 build1 (VIEW_CONVERT_EXPR, itype,
8573 stored_val), true, NULL_TREE, true,
8574 GSI_SAME_STMT);
8576 /* Build the compare&swap statement. */
8577 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
8578 new_storedi = force_gimple_operand_gsi (&si,
8579 fold_convert (TREE_TYPE (loadedi),
8580 new_storedi),
8581 true, NULL_TREE,
8582 true, GSI_SAME_STMT);
8584 if (gimple_in_ssa_p (cfun))
8585 old_vali = loadedi;
8586 else
8588 old_vali = create_tmp_var (TREE_TYPE (loadedi));
8589 stmt = gimple_build_assign (old_vali, loadedi);
8590 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8592 stmt = gimple_build_assign (loadedi, new_storedi);
8593 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8596 /* Note that we always perform the comparison as an integer, even for
8597 floating point. This allows the atomic operation to properly
8598 succeed even with NaNs and -0.0. */
8599 stmt = gimple_build_cond_empty
8600 (build2 (NE_EXPR, boolean_type_node,
8601 new_storedi, old_vali));
8602 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8604 /* Update cfg. */
8605 e = single_succ_edge (store_bb);
8606 e->flags &= ~EDGE_FALLTHRU;
8607 e->flags |= EDGE_FALSE_VALUE;
8609 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
8611 /* Copy the new value to loadedi (we already did that before the condition
8612 if we are not in SSA). */
8613 if (gimple_in_ssa_p (cfun))
8615 phi = gimple_seq_first_stmt (phi_nodes (loop_header));
8616 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
8619 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8620 gsi_remove (&si, true);
8622 struct loop *loop = alloc_loop ();
8623 loop->header = loop_header;
8624 loop->latch = store_bb;
8625 add_loop (loop, loop_header->loop_father);
8627 if (gimple_in_ssa_p (cfun))
8628 update_ssa (TODO_update_ssa_no_phi);
8630 return true;
8633 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8635 GOMP_atomic_start ();
8636 *addr = rhs;
8637 GOMP_atomic_end ();
8639 The result is not globally atomic, but works so long as all parallel
8640 references are within #pragma omp atomic directives. According to
8641 responses received from omp@openmp.org, appears to be within spec.
8642 Which makes sense, since that's how several other compilers handle
8643 this situation as well.
8644 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8645 expanding. STORED_VAL is the operand of the matching
8646 GIMPLE_OMP_ATOMIC_STORE.
8648 We replace
8649 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8650 loaded_val = *addr;
8652 and replace
8653 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8654 *addr = stored_val;
8657 static bool
8658 expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
8659 tree addr, tree loaded_val, tree stored_val)
8661 gimple_stmt_iterator si;
8662 gassign *stmt;
8663 tree t;
8665 si = gsi_last_bb (load_bb);
8666 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
8668 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
8669 t = build_call_expr (t, 0);
8670 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
8672 stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr));
8673 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8674 gsi_remove (&si, true);
8676 si = gsi_last_bb (store_bb);
8677 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
8679 stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)),
8680 stored_val);
8681 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8683 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
8684 t = build_call_expr (t, 0);
8685 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
8686 gsi_remove (&si, true);
8688 if (gimple_in_ssa_p (cfun))
8689 update_ssa (TODO_update_ssa_no_phi);
8690 return true;
8693 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8694 using expand_omp_atomic_fetch_op. If it failed, we try to
8695 call expand_omp_atomic_pipeline, and if it fails too, the
8696 ultimate fallback is wrapping the operation in a mutex
8697 (expand_omp_atomic_mutex). REGION is the atomic region built
8698 by build_omp_regions_1(). */
8700 static void
8701 expand_omp_atomic (struct omp_region *region)
8703 basic_block load_bb = region->entry, store_bb = region->exit;
8704 gomp_atomic_load *load = as_a <gomp_atomic_load *> (last_stmt (load_bb));
8705 gomp_atomic_store *store = as_a <gomp_atomic_store *> (last_stmt (store_bb));
8706 tree loaded_val = gimple_omp_atomic_load_lhs (load);
8707 tree addr = gimple_omp_atomic_load_rhs (load);
8708 tree stored_val = gimple_omp_atomic_store_val (store);
8709 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
8710 HOST_WIDE_INT index;
8712 /* Make sure the type is one of the supported sizes. */
8713 index = tree_to_uhwi (TYPE_SIZE_UNIT (type));
8714 index = exact_log2 (index);
8715 if (index >= 0 && index <= 4)
8717 unsigned int align = TYPE_ALIGN_UNIT (type);
8719 /* __sync builtins require strict data alignment. */
8720 if (exact_log2 (align) >= index)
8722 /* Atomic load. */
8723 if (loaded_val == stored_val
8724 && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
8725 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
8726 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
8727 && expand_omp_atomic_load (load_bb, addr, loaded_val, index))
8728 return;
8730 /* Atomic store. */
8731 if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
8732 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
8733 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
8734 && store_bb == single_succ (load_bb)
8735 && first_stmt (store_bb) == store
8736 && expand_omp_atomic_store (load_bb, addr, loaded_val,
8737 stored_val, index))
8738 return;
8740 /* When possible, use specialized atomic update functions. */
8741 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
8742 && store_bb == single_succ (load_bb)
8743 && expand_omp_atomic_fetch_op (load_bb, addr,
8744 loaded_val, stored_val, index))
8745 return;
8747 /* If we don't have specialized __sync builtins, try and implement
8748 as a compare and swap loop. */
8749 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
8750 loaded_val, stored_val, index))
8751 return;
8755 /* The ultimate fallback is wrapping the operation in a mutex. */
8756 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
8760 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8762 static void
8763 expand_omp_target (struct omp_region *region)
8765 basic_block entry_bb, exit_bb, new_bb;
8766 struct function *child_cfun;
8767 tree child_fn, block, t;
8768 gimple_stmt_iterator gsi;
8769 gomp_target *entry_stmt;
8770 gimple stmt;
8771 edge e;
8772 bool offloaded, data_region;
8774 entry_stmt = as_a <gomp_target *> (last_stmt (region->entry));
8775 new_bb = region->entry;
8777 offloaded = is_gimple_omp_offloaded (entry_stmt);
8778 switch (gimple_omp_target_kind (entry_stmt))
8780 case GF_OMP_TARGET_KIND_REGION:
8781 case GF_OMP_TARGET_KIND_UPDATE:
8782 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
8783 case GF_OMP_TARGET_KIND_OACC_KERNELS:
8784 case GF_OMP_TARGET_KIND_OACC_UPDATE:
8785 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
8786 data_region = false;
8787 break;
8788 case GF_OMP_TARGET_KIND_DATA:
8789 case GF_OMP_TARGET_KIND_OACC_DATA:
8790 data_region = true;
8791 break;
8792 default:
8793 gcc_unreachable ();
8796 child_fn = NULL_TREE;
8797 child_cfun = NULL;
8798 if (offloaded)
8800 child_fn = gimple_omp_target_child_fn (entry_stmt);
8801 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
8804 /* Supported by expand_omp_taskreg, but not here. */
8805 if (child_cfun != NULL)
8806 gcc_checking_assert (!child_cfun->cfg);
8807 gcc_checking_assert (!gimple_in_ssa_p (cfun));
8809 entry_bb = region->entry;
8810 exit_bb = region->exit;
8812 if (offloaded)
8814 unsigned srcidx, dstidx, num;
8816 /* If the offloading region needs data sent from the parent
8817 function, then the very first statement (except possible
8818 tree profile counter updates) of the offloading body
8819 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8820 &.OMP_DATA_O is passed as an argument to the child function,
8821 we need to replace it with the argument as seen by the child
8822 function.
8824 In most cases, this will end up being the identity assignment
8825 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8826 a function call that has been inlined, the original PARM_DECL
8827 .OMP_DATA_I may have been converted into a different local
8828 variable. In which case, we need to keep the assignment. */
8829 tree data_arg = gimple_omp_target_data_arg (entry_stmt);
8830 if (data_arg)
8832 basic_block entry_succ_bb = single_succ (entry_bb);
8833 gimple_stmt_iterator gsi;
8834 tree arg;
8835 gimple tgtcopy_stmt = NULL;
8836 tree sender = TREE_VEC_ELT (data_arg, 0);
8838 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
8840 gcc_assert (!gsi_end_p (gsi));
8841 stmt = gsi_stmt (gsi);
8842 if (gimple_code (stmt) != GIMPLE_ASSIGN)
8843 continue;
8845 if (gimple_num_ops (stmt) == 2)
8847 tree arg = gimple_assign_rhs1 (stmt);
8849 /* We're ignoring the subcode because we're
8850 effectively doing a STRIP_NOPS. */
8852 if (TREE_CODE (arg) == ADDR_EXPR
8853 && TREE_OPERAND (arg, 0) == sender)
8855 tgtcopy_stmt = stmt;
8856 break;
8861 gcc_assert (tgtcopy_stmt != NULL);
8862 arg = DECL_ARGUMENTS (child_fn);
8864 gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg);
8865 gsi_remove (&gsi, true);
8868 /* Declare local variables needed in CHILD_CFUN. */
8869 block = DECL_INITIAL (child_fn);
8870 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
8871 /* The gimplifier could record temporaries in the offloading block
8872 rather than in containing function's local_decls chain,
8873 which would mean cgraph missed finalizing them. Do it now. */
8874 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
8875 if (TREE_CODE (t) == VAR_DECL
8876 && TREE_STATIC (t)
8877 && !DECL_EXTERNAL (t))
8878 varpool_node::finalize_decl (t);
8879 DECL_SAVED_TREE (child_fn) = NULL;
8880 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8881 gimple_set_body (child_fn, NULL);
8882 TREE_USED (block) = 1;
8884 /* Reset DECL_CONTEXT on function arguments. */
8885 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
8886 DECL_CONTEXT (t) = child_fn;
8888 /* Split ENTRY_BB at GIMPLE_*,
8889 so that it can be moved to the child function. */
8890 gsi = gsi_last_bb (entry_bb);
8891 stmt = gsi_stmt (gsi);
8892 gcc_assert (stmt
8893 && gimple_code (stmt) == gimple_code (entry_stmt));
8894 e = split_block (entry_bb, stmt);
8895 gsi_remove (&gsi, true);
8896 entry_bb = e->dest;
8897 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8899 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8900 if (exit_bb)
8902 gsi = gsi_last_bb (exit_bb);
8903 gcc_assert (!gsi_end_p (gsi)
8904 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
8905 stmt = gimple_build_return (NULL);
8906 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
8907 gsi_remove (&gsi, true);
8910 /* Move the offloading region into CHILD_CFUN. */
8912 block = gimple_block (entry_stmt);
8914 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
8915 if (exit_bb)
8916 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
8917 /* When the OMP expansion process cannot guarantee an up-to-date
8918 loop tree arrange for the child function to fixup loops. */
8919 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
8920 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
8922 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8923 num = vec_safe_length (child_cfun->local_decls);
8924 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
8926 t = (*child_cfun->local_decls)[srcidx];
8927 if (DECL_CONTEXT (t) == cfun->decl)
8928 continue;
8929 if (srcidx != dstidx)
8930 (*child_cfun->local_decls)[dstidx] = t;
8931 dstidx++;
8933 if (dstidx != num)
8934 vec_safe_truncate (child_cfun->local_decls, dstidx);
8936 /* Inform the callgraph about the new function. */
8937 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
8938 cgraph_node::add_new_function (child_fn, true);
8940 #ifdef ENABLE_OFFLOADING
8941 /* Add the new function to the offload table. */
8942 vec_safe_push (offload_funcs, child_fn);
8943 #endif
8945 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8946 fixed in a following pass. */
8947 push_cfun (child_cfun);
8948 cgraph_edge::rebuild_edges ();
8950 #ifdef ENABLE_OFFLOADING
8951 /* Prevent IPA from removing child_fn as unreachable, since there are no
8952 refs from the parent function to child_fn in offload LTO mode. */
8953 struct cgraph_node *node = cgraph_node::get (child_fn);
8954 node->mark_force_output ();
8955 #endif
8957 /* Some EH regions might become dead, see PR34608. If
8958 pass_cleanup_cfg isn't the first pass to happen with the
8959 new child, these dead EH edges might cause problems.
8960 Clean them up now. */
8961 if (flag_exceptions)
8963 basic_block bb;
8964 bool changed = false;
8966 FOR_EACH_BB_FN (bb, cfun)
8967 changed |= gimple_purge_dead_eh_edges (bb);
8968 if (changed)
8969 cleanup_tree_cfg ();
8971 pop_cfun ();
8974 /* Emit a library call to launch the offloading region, or do data
8975 transfers. */
8976 tree t1, t2, t3, t4, device, cond, c, clauses;
8977 enum built_in_function start_ix;
8978 location_t clause_loc;
8980 switch (gimple_omp_target_kind (entry_stmt))
8982 case GF_OMP_TARGET_KIND_REGION:
8983 start_ix = BUILT_IN_GOMP_TARGET;
8984 break;
8985 case GF_OMP_TARGET_KIND_DATA:
8986 start_ix = BUILT_IN_GOMP_TARGET_DATA;
8987 break;
8988 case GF_OMP_TARGET_KIND_UPDATE:
8989 start_ix = BUILT_IN_GOMP_TARGET_UPDATE;
8990 break;
8991 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
8992 case GF_OMP_TARGET_KIND_OACC_KERNELS:
8993 start_ix = BUILT_IN_GOACC_PARALLEL;
8994 break;
8995 case GF_OMP_TARGET_KIND_OACC_DATA:
8996 start_ix = BUILT_IN_GOACC_DATA_START;
8997 break;
8998 case GF_OMP_TARGET_KIND_OACC_UPDATE:
8999 start_ix = BUILT_IN_GOACC_UPDATE;
9000 break;
9001 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
9002 start_ix = BUILT_IN_GOACC_ENTER_EXIT_DATA;
9003 break;
9004 default:
9005 gcc_unreachable ();
9008 clauses = gimple_omp_target_clauses (entry_stmt);
9010 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
9011 library choose) and there is no conditional. */
9012 cond = NULL_TREE;
9013 device = build_int_cst (integer_type_node, GOMP_DEVICE_ICV);
9015 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
9016 if (c)
9017 cond = OMP_CLAUSE_IF_EXPR (c);
9019 c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE);
9020 if (c)
9022 /* Even if we pass it to all library function calls, it is currently only
9023 defined/used for the OpenMP target ones. */
9024 gcc_checking_assert (start_ix == BUILT_IN_GOMP_TARGET
9025 || start_ix == BUILT_IN_GOMP_TARGET_DATA
9026 || start_ix == BUILT_IN_GOMP_TARGET_UPDATE);
9028 device = OMP_CLAUSE_DEVICE_ID (c);
9029 clause_loc = OMP_CLAUSE_LOCATION (c);
9031 else
9032 clause_loc = gimple_location (entry_stmt);
9034 /* Ensure 'device' is of the correct type. */
9035 device = fold_convert_loc (clause_loc, integer_type_node, device);
9037 /* If we found the clause 'if (cond)', build
9038 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
9039 if (cond)
9041 cond = gimple_boolify (cond);
9043 basic_block cond_bb, then_bb, else_bb;
9044 edge e;
9045 tree tmp_var;
9047 tmp_var = create_tmp_var (TREE_TYPE (device));
9048 if (offloaded)
9049 e = split_block (new_bb, NULL);
9050 else
9052 gsi = gsi_last_bb (new_bb);
9053 gsi_prev (&gsi);
9054 e = split_block (new_bb, gsi_stmt (gsi));
9056 cond_bb = e->src;
9057 new_bb = e->dest;
9058 remove_edge (e);
9060 then_bb = create_empty_bb (cond_bb);
9061 else_bb = create_empty_bb (then_bb);
9062 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
9063 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
9065 stmt = gimple_build_cond_empty (cond);
9066 gsi = gsi_last_bb (cond_bb);
9067 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
9069 gsi = gsi_start_bb (then_bb);
9070 stmt = gimple_build_assign (tmp_var, device);
9071 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
9073 gsi = gsi_start_bb (else_bb);
9074 stmt = gimple_build_assign (tmp_var,
9075 build_int_cst (integer_type_node,
9076 GOMP_DEVICE_HOST_FALLBACK));
9077 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
9079 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
9080 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
9081 add_bb_to_loop (then_bb, cond_bb->loop_father);
9082 add_bb_to_loop (else_bb, cond_bb->loop_father);
9083 make_edge (then_bb, new_bb, EDGE_FALLTHRU);
9084 make_edge (else_bb, new_bb, EDGE_FALLTHRU);
9086 device = tmp_var;
9089 gsi = gsi_last_bb (new_bb);
9090 t = gimple_omp_target_data_arg (entry_stmt);
9091 if (t == NULL)
9093 t1 = size_zero_node;
9094 t2 = build_zero_cst (ptr_type_node);
9095 t3 = t2;
9096 t4 = t2;
9098 else
9100 t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1))));
9101 t1 = size_binop (PLUS_EXPR, t1, size_int (1));
9102 t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0));
9103 t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1));
9104 t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2));
9107 gimple g;
9108 /* The maximum number used by any start_ix, without varargs. */
9109 auto_vec<tree, 11> args;
9110 args.quick_push (device);
9111 if (offloaded)
9112 args.quick_push (build_fold_addr_expr (child_fn));
9113 switch (start_ix)
9115 case BUILT_IN_GOMP_TARGET:
9116 case BUILT_IN_GOMP_TARGET_DATA:
9117 case BUILT_IN_GOMP_TARGET_UPDATE:
9118 /* This const void * is part of the current ABI, but we're not actually
9119 using it. */
9120 args.quick_push (build_zero_cst (ptr_type_node));
9121 break;
9122 case BUILT_IN_GOACC_DATA_START:
9123 case BUILT_IN_GOACC_ENTER_EXIT_DATA:
9124 case BUILT_IN_GOACC_PARALLEL:
9125 case BUILT_IN_GOACC_UPDATE:
9126 break;
9127 default:
9128 gcc_unreachable ();
9130 args.quick_push (t1);
9131 args.quick_push (t2);
9132 args.quick_push (t3);
9133 args.quick_push (t4);
9134 switch (start_ix)
9136 case BUILT_IN_GOACC_DATA_START:
9137 case BUILT_IN_GOMP_TARGET:
9138 case BUILT_IN_GOMP_TARGET_DATA:
9139 case BUILT_IN_GOMP_TARGET_UPDATE:
9140 break;
9141 case BUILT_IN_GOACC_PARALLEL:
9143 tree t_num_gangs, t_num_workers, t_vector_length;
9145 /* Default values for num_gangs, num_workers, and vector_length. */
9146 t_num_gangs = t_num_workers = t_vector_length
9147 = fold_convert_loc (gimple_location (entry_stmt),
9148 integer_type_node, integer_one_node);
9149 /* ..., but if present, use the value specified by the respective
9150 clause, making sure that are of the correct type. */
9151 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS);
9152 if (c)
9153 t_num_gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
9154 integer_type_node,
9155 OMP_CLAUSE_NUM_GANGS_EXPR (c));
9156 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_WORKERS);
9157 if (c)
9158 t_num_workers = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
9159 integer_type_node,
9160 OMP_CLAUSE_NUM_WORKERS_EXPR (c));
9161 c = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH);
9162 if (c)
9163 t_vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
9164 integer_type_node,
9165 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c));
9166 args.quick_push (t_num_gangs);
9167 args.quick_push (t_num_workers);
9168 args.quick_push (t_vector_length);
9170 /* FALLTHRU */
9171 case BUILT_IN_GOACC_ENTER_EXIT_DATA:
9172 case BUILT_IN_GOACC_UPDATE:
9174 tree t_async;
9175 int t_wait_idx;
9177 /* Default values for t_async. */
9178 t_async = fold_convert_loc (gimple_location (entry_stmt),
9179 integer_type_node,
9180 build_int_cst (integer_type_node,
9181 GOMP_ASYNC_SYNC));
9182 /* ..., but if present, use the value specified by the respective
9183 clause, making sure that is of the correct type. */
9184 c = find_omp_clause (clauses, OMP_CLAUSE_ASYNC);
9185 if (c)
9186 t_async = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
9187 integer_type_node,
9188 OMP_CLAUSE_ASYNC_EXPR (c));
9190 args.quick_push (t_async);
9191 /* Save the index, and... */
9192 t_wait_idx = args.length ();
9193 /* ... push a default value. */
9194 args.quick_push (fold_convert_loc (gimple_location (entry_stmt),
9195 integer_type_node,
9196 integer_zero_node));
9197 c = find_omp_clause (clauses, OMP_CLAUSE_WAIT);
9198 if (c)
9200 int n = 0;
9202 for (; c; c = OMP_CLAUSE_CHAIN (c))
9204 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WAIT)
9206 args.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c),
9207 integer_type_node,
9208 OMP_CLAUSE_WAIT_EXPR (c)));
9209 n++;
9213 /* Now that we know the number, replace the default value. */
9214 args.ordered_remove (t_wait_idx);
9215 args.quick_insert (t_wait_idx,
9216 fold_convert_loc (gimple_location (entry_stmt),
9217 integer_type_node,
9218 build_int_cst (integer_type_node, n)));
9221 break;
9222 default:
9223 gcc_unreachable ();
9226 g = gimple_build_call_vec (builtin_decl_explicit (start_ix), args);
9227 gimple_set_location (g, gimple_location (entry_stmt));
9228 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9229 if (!offloaded)
9231 g = gsi_stmt (gsi);
9232 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET);
9233 gsi_remove (&gsi, true);
9235 if (data_region
9236 && region->exit)
9238 gsi = gsi_last_bb (region->exit);
9239 g = gsi_stmt (gsi);
9240 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN);
9241 gsi_remove (&gsi, true);
9246 /* Expand the parallel region tree rooted at REGION. Expansion
9247 proceeds in depth-first order. Innermost regions are expanded
9248 first. This way, parallel regions that require a new function to
9249 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9250 internal dependencies in their body. */
9252 static void
9253 expand_omp (struct omp_region *region)
9255 while (region)
9257 location_t saved_location;
9258 gimple inner_stmt = NULL;
9260 /* First, determine whether this is a combined parallel+workshare
9261 region. */
9262 if (region->type == GIMPLE_OMP_PARALLEL)
9263 determine_parallel_type (region);
9265 if (region->type == GIMPLE_OMP_FOR
9266 && gimple_omp_for_combined_p (last_stmt (region->entry)))
9267 inner_stmt = last_stmt (region->inner->entry);
9269 if (region->inner)
9270 expand_omp (region->inner);
9272 saved_location = input_location;
9273 if (gimple_has_location (last_stmt (region->entry)))
9274 input_location = gimple_location (last_stmt (region->entry));
9276 switch (region->type)
9278 case GIMPLE_OMP_PARALLEL:
9279 case GIMPLE_OMP_TASK:
9280 expand_omp_taskreg (region);
9281 break;
9283 case GIMPLE_OMP_FOR:
9284 expand_omp_for (region, inner_stmt);
9285 break;
9287 case GIMPLE_OMP_SECTIONS:
9288 expand_omp_sections (region);
9289 break;
9291 case GIMPLE_OMP_SECTION:
9292 /* Individual omp sections are handled together with their
9293 parent GIMPLE_OMP_SECTIONS region. */
9294 break;
9296 case GIMPLE_OMP_SINGLE:
9297 expand_omp_single (region);
9298 break;
9300 case GIMPLE_OMP_MASTER:
9301 case GIMPLE_OMP_TASKGROUP:
9302 case GIMPLE_OMP_ORDERED:
9303 case GIMPLE_OMP_CRITICAL:
9304 case GIMPLE_OMP_TEAMS:
9305 expand_omp_synch (region);
9306 break;
9308 case GIMPLE_OMP_ATOMIC_LOAD:
9309 expand_omp_atomic (region);
9310 break;
9312 case GIMPLE_OMP_TARGET:
9313 expand_omp_target (region);
9314 break;
9316 default:
9317 gcc_unreachable ();
9320 input_location = saved_location;
9321 region = region->next;
9326 /* Helper for build_omp_regions. Scan the dominator tree starting at
9327 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9328 true, the function ends once a single tree is built (otherwise, whole
9329 forest of OMP constructs may be built). */
9331 static void
9332 build_omp_regions_1 (basic_block bb, struct omp_region *parent,
9333 bool single_tree)
9335 gimple_stmt_iterator gsi;
9336 gimple stmt;
9337 basic_block son;
9339 gsi = gsi_last_bb (bb);
9340 if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi)))
9342 struct omp_region *region;
9343 enum gimple_code code;
9345 stmt = gsi_stmt (gsi);
9346 code = gimple_code (stmt);
9347 if (code == GIMPLE_OMP_RETURN)
9349 /* STMT is the return point out of region PARENT. Mark it
9350 as the exit point and make PARENT the immediately
9351 enclosing region. */
9352 gcc_assert (parent);
9353 region = parent;
9354 region->exit = bb;
9355 parent = parent->outer;
9357 else if (code == GIMPLE_OMP_ATOMIC_STORE)
9359 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9360 GIMPLE_OMP_RETURN, but matches with
9361 GIMPLE_OMP_ATOMIC_LOAD. */
9362 gcc_assert (parent);
9363 gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD);
9364 region = parent;
9365 region->exit = bb;
9366 parent = parent->outer;
9368 else if (code == GIMPLE_OMP_CONTINUE)
9370 gcc_assert (parent);
9371 parent->cont = bb;
9373 else if (code == GIMPLE_OMP_SECTIONS_SWITCH)
9375 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9376 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9378 else
9380 region = new_omp_region (bb, code, parent);
9381 /* Otherwise... */
9382 if (code == GIMPLE_OMP_TARGET)
9384 switch (gimple_omp_target_kind (stmt))
9386 case GF_OMP_TARGET_KIND_REGION:
9387 case GF_OMP_TARGET_KIND_DATA:
9388 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
9389 case GF_OMP_TARGET_KIND_OACC_KERNELS:
9390 case GF_OMP_TARGET_KIND_OACC_DATA:
9391 break;
9392 case GF_OMP_TARGET_KIND_UPDATE:
9393 case GF_OMP_TARGET_KIND_OACC_UPDATE:
9394 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
9395 /* ..., other than for those stand-alone directives... */
9396 region = NULL;
9397 break;
9398 default:
9399 gcc_unreachable ();
9402 /* ..., this directive becomes the parent for a new region. */
9403 if (region)
9404 parent = region;
9408 if (single_tree && !parent)
9409 return;
9411 for (son = first_dom_son (CDI_DOMINATORS, bb);
9412 son;
9413 son = next_dom_son (CDI_DOMINATORS, son))
9414 build_omp_regions_1 (son, parent, single_tree);
9417 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9418 root_omp_region. */
9420 static void
9421 build_omp_regions_root (basic_block root)
9423 gcc_assert (root_omp_region == NULL);
9424 build_omp_regions_1 (root, NULL, true);
9425 gcc_assert (root_omp_region != NULL);
9428 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9430 void
9431 omp_expand_local (basic_block head)
9433 build_omp_regions_root (head);
9434 if (dump_file && (dump_flags & TDF_DETAILS))
9436 fprintf (dump_file, "\nOMP region tree\n\n");
9437 dump_omp_region (dump_file, root_omp_region, 0);
9438 fprintf (dump_file, "\n");
9441 remove_exit_barriers (root_omp_region);
9442 expand_omp (root_omp_region);
9444 free_omp_regions ();
9447 /* Scan the CFG and build a tree of OMP regions. Return the root of
9448 the OMP region tree. */
9450 static void
9451 build_omp_regions (void)
9453 gcc_assert (root_omp_region == NULL);
9454 calculate_dominance_info (CDI_DOMINATORS);
9455 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false);
9458 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9460 static unsigned int
9461 execute_expand_omp (void)
9463 build_omp_regions ();
9465 if (!root_omp_region)
9466 return 0;
9468 if (dump_file)
9470 fprintf (dump_file, "\nOMP region tree\n\n");
9471 dump_omp_region (dump_file, root_omp_region, 0);
9472 fprintf (dump_file, "\n");
9475 remove_exit_barriers (root_omp_region);
9477 expand_omp (root_omp_region);
9479 cleanup_tree_cfg ();
9481 free_omp_regions ();
9483 return 0;
9486 /* OMP expansion -- the default pass, run before creation of SSA form. */
9488 namespace {
9490 const pass_data pass_data_expand_omp =
9492 GIMPLE_PASS, /* type */
9493 "ompexp", /* name */
9494 OPTGROUP_NONE, /* optinfo_flags */
9495 TV_NONE, /* tv_id */
9496 PROP_gimple_any, /* properties_required */
9497 PROP_gimple_eomp, /* properties_provided */
9498 0, /* properties_destroyed */
9499 0, /* todo_flags_start */
9500 0, /* todo_flags_finish */
9503 class pass_expand_omp : public gimple_opt_pass
9505 public:
9506 pass_expand_omp (gcc::context *ctxt)
9507 : gimple_opt_pass (pass_data_expand_omp, ctxt)
9510 /* opt_pass methods: */
9511 virtual unsigned int execute (function *)
9513 bool gate = ((flag_cilkplus != 0 || flag_openacc != 0 || flag_openmp != 0
9514 || flag_openmp_simd != 0)
9515 && !seen_error ());
9517 /* This pass always runs, to provide PROP_gimple_eomp.
9518 But often, there is nothing to do. */
9519 if (!gate)
9520 return 0;
9522 return execute_expand_omp ();
9525 }; // class pass_expand_omp
9527 } // anon namespace
9529 gimple_opt_pass *
9530 make_pass_expand_omp (gcc::context *ctxt)
9532 return new pass_expand_omp (ctxt);
9535 namespace {
9537 const pass_data pass_data_expand_omp_ssa =
9539 GIMPLE_PASS, /* type */
9540 "ompexpssa", /* name */
9541 OPTGROUP_NONE, /* optinfo_flags */
9542 TV_NONE, /* tv_id */
9543 PROP_cfg | PROP_ssa, /* properties_required */
9544 PROP_gimple_eomp, /* properties_provided */
9545 0, /* properties_destroyed */
9546 0, /* todo_flags_start */
9547 TODO_cleanup_cfg | TODO_rebuild_alias, /* todo_flags_finish */
9550 class pass_expand_omp_ssa : public gimple_opt_pass
9552 public:
9553 pass_expand_omp_ssa (gcc::context *ctxt)
9554 : gimple_opt_pass (pass_data_expand_omp_ssa, ctxt)
9557 /* opt_pass methods: */
9558 virtual bool gate (function *fun)
9560 return !(fun->curr_properties & PROP_gimple_eomp);
9562 virtual unsigned int execute (function *) { return execute_expand_omp (); }
9564 }; // class pass_expand_omp_ssa
9566 } // anon namespace
9568 gimple_opt_pass *
9569 make_pass_expand_omp_ssa (gcc::context *ctxt)
9571 return new pass_expand_omp_ssa (ctxt);
9574 /* Routines to lower OMP directives into OMP-GIMPLE. */
9576 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9577 convert it to gimple. */
9578 static void
9579 oacc_gimple_assign (tree dest, tree_code op, tree src, gimple_seq *seq)
9581 gimple stmt;
9583 if (TREE_CODE (TREE_TYPE (dest)) != COMPLEX_TYPE)
9585 stmt = gimple_build_assign (dest, op, dest, src);
9586 gimple_seq_add_stmt (seq, stmt);
9587 return;
9590 tree t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9591 tree rdest = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest);
9592 gimplify_assign (t, rdest, seq);
9593 rdest = t;
9595 t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9596 tree idest = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest);
9597 gimplify_assign (t, idest, seq);
9598 idest = t;
9600 t = create_tmp_var (TREE_TYPE (TREE_TYPE (src)));
9601 tree rsrc = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src);
9602 gimplify_assign (t, rsrc, seq);
9603 rsrc = t;
9605 t = create_tmp_var (TREE_TYPE (TREE_TYPE (src)));
9606 tree isrc = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src);
9607 gimplify_assign (t, isrc, seq);
9608 isrc = t;
9610 tree r = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9611 tree i = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9612 tree result;
9614 if (op == PLUS_EXPR)
9616 stmt = gimple_build_assign (r, op, rdest, rsrc);
9617 gimple_seq_add_stmt (seq, stmt);
9619 stmt = gimple_build_assign (i, op, idest, isrc);
9620 gimple_seq_add_stmt (seq, stmt);
9622 else if (op == MULT_EXPR)
9624 /* Let x = a + ib = dest, y = c + id = src.
9625 x * y = (ac - bd) + i(ad + bc) */
9626 tree ac = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9627 tree bd = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9628 tree ad = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9629 tree bc = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)));
9631 stmt = gimple_build_assign (ac, MULT_EXPR, rdest, rsrc);
9632 gimple_seq_add_stmt (seq, stmt);
9634 stmt = gimple_build_assign (bd, MULT_EXPR, idest, isrc);
9635 gimple_seq_add_stmt (seq, stmt);
9637 stmt = gimple_build_assign (r, MINUS_EXPR, ac, bd);
9638 gimple_seq_add_stmt (seq, stmt);
9640 stmt = gimple_build_assign (ad, MULT_EXPR, rdest, isrc);
9641 gimple_seq_add_stmt (seq, stmt);
9643 stmt = gimple_build_assign (bd, MULT_EXPR, idest, rsrc);
9644 gimple_seq_add_stmt (seq, stmt);
9646 stmt = gimple_build_assign (i, PLUS_EXPR, ad, bc);
9647 gimple_seq_add_stmt (seq, stmt);
9649 else
9650 gcc_unreachable ();
9652 result = build2 (COMPLEX_EXPR, TREE_TYPE (dest), r, i);
9653 gimplify_assign (dest, result, seq);
9656 /* Helper function to initialize local data for the reduction arrays.
9657 The reduction arrays need to be placed inside the calling function
9658 for accelerators, or else the host won't be able to preform the final
9659 reduction. */
9661 static void
9662 oacc_initialize_reduction_data (tree clauses, tree nthreads,
9663 gimple_seq *stmt_seqp, omp_context *ctx)
9665 tree c, t, oc;
9666 gimple stmt;
9667 omp_context *octx;
9669 /* Find the innermost OpenACC parallel context. */
9670 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
9671 && (gimple_omp_target_kind (ctx->stmt)
9672 == GF_OMP_TARGET_KIND_OACC_PARALLEL))
9673 octx = ctx;
9674 else
9675 octx = ctx->outer;
9676 gcc_checking_assert (gimple_code (octx->stmt) == GIMPLE_OMP_TARGET
9677 && (gimple_omp_target_kind (octx->stmt)
9678 == GF_OMP_TARGET_KIND_OACC_PARALLEL));
9680 /* Extract the clauses. */
9681 oc = gimple_omp_target_clauses (octx->stmt);
9683 /* Find the last outer clause. */
9684 for (; oc && OMP_CLAUSE_CHAIN (oc); oc = OMP_CLAUSE_CHAIN (oc))
9687 /* Allocate arrays for each reduction variable. */
9688 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9690 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
9691 continue;
9693 tree var = OMP_CLAUSE_DECL (c);
9694 tree type = get_base_type (var);
9695 tree array = lookup_oacc_reduction (oacc_get_reduction_array_id (var),
9696 ctx);
9697 tree size, call;
9699 /* Calculate size of the reduction array. */
9700 t = create_tmp_var (TREE_TYPE (nthreads));
9701 stmt = gimple_build_assign (t, MULT_EXPR, nthreads,
9702 fold_convert (TREE_TYPE (nthreads),
9703 TYPE_SIZE_UNIT (type)));
9704 gimple_seq_add_stmt (stmt_seqp, stmt);
9706 size = create_tmp_var (sizetype);
9707 gimplify_assign (size, fold_build1 (NOP_EXPR, sizetype, t), stmt_seqp);
9709 /* Now allocate memory for it. */
9710 call = unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA));
9711 stmt = gimple_build_call (call, 1, size);
9712 gimple_call_set_lhs (stmt, array);
9713 gimple_seq_add_stmt (stmt_seqp, stmt);
9715 /* Map this array into the accelerator. */
9717 /* Add the reduction array to the list of clauses. */
9718 tree x = array;
9719 t = build_omp_clause (gimple_location (ctx->stmt), OMP_CLAUSE_MAP);
9720 OMP_CLAUSE_SET_MAP_KIND (t, GOMP_MAP_FORCE_FROM);
9721 OMP_CLAUSE_DECL (t) = x;
9722 OMP_CLAUSE_CHAIN (t) = NULL;
9723 if (oc)
9724 OMP_CLAUSE_CHAIN (oc) = t;
9725 else
9726 gimple_omp_target_set_clauses (as_a <gomp_target *> (octx->stmt), t);
9727 OMP_CLAUSE_SIZE (t) = size;
9728 oc = t;
9732 /* Helper function to process the array of partial reductions. Nthreads
9733 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9734 cannot be used here, because nthreads on the host may be different than
9735 on the accelerator. */
9737 static void
9738 oacc_finalize_reduction_data (tree clauses, tree nthreads,
9739 gimple_seq *stmt_seqp, omp_context *ctx)
9741 tree c, x, var, array, loop_header, loop_body, loop_exit, type;
9742 gimple stmt;
9744 /* Create for loop.
9746 let var = the original reduction variable
9747 let array = reduction variable array
9749 for (i = 0; i < nthreads; i++)
9750 var op= array[i]
9753 loop_header = create_artificial_label (UNKNOWN_LOCATION);
9754 loop_body = create_artificial_label (UNKNOWN_LOCATION);
9755 loop_exit = create_artificial_label (UNKNOWN_LOCATION);
9757 /* Create and initialize an index variable. */
9758 tree ix = create_tmp_var (sizetype);
9759 gimplify_assign (ix, fold_build1 (NOP_EXPR, sizetype, integer_zero_node),
9760 stmt_seqp);
9762 /* Insert the loop header label here. */
9763 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_header));
9765 /* Exit loop if ix >= nthreads. */
9766 x = create_tmp_var (sizetype);
9767 gimplify_assign (x, fold_build1 (NOP_EXPR, sizetype, nthreads), stmt_seqp);
9768 stmt = gimple_build_cond (GE_EXPR, ix, x, loop_exit, loop_body);
9769 gimple_seq_add_stmt (stmt_seqp, stmt);
9771 /* Insert the loop body label here. */
9772 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_body));
9774 /* Collapse each reduction array, one element at a time. */
9775 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9777 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
9778 continue;
9780 tree_code reduction_code = OMP_CLAUSE_REDUCTION_CODE (c);
9782 /* reduction(-:var) sums up the partial results, so it acts
9783 identically to reduction(+:var). */
9784 if (reduction_code == MINUS_EXPR)
9785 reduction_code = PLUS_EXPR;
9787 /* Set up reduction variable var. */
9788 var = OMP_CLAUSE_DECL (c);
9789 type = get_base_type (var);
9790 array = lookup_oacc_reduction (oacc_get_reduction_array_id
9791 (OMP_CLAUSE_DECL (c)), ctx);
9793 /* Calculate the array offset. */
9794 tree offset = create_tmp_var (sizetype);
9795 gimplify_assign (offset, TYPE_SIZE_UNIT (type), stmt_seqp);
9796 stmt = gimple_build_assign (offset, MULT_EXPR, offset, ix);
9797 gimple_seq_add_stmt (stmt_seqp, stmt);
9799 tree ptr = create_tmp_var (TREE_TYPE (array));
9800 stmt = gimple_build_assign (ptr, POINTER_PLUS_EXPR, array, offset);
9801 gimple_seq_add_stmt (stmt_seqp, stmt);
9803 /* Extract array[ix] into mem. */
9804 tree mem = create_tmp_var (type);
9805 gimplify_assign (mem, build_simple_mem_ref (ptr), stmt_seqp);
9807 /* Find the original reduction variable. */
9808 if (is_reference (var))
9809 var = build_simple_mem_ref (var);
9811 tree t = create_tmp_var (type);
9813 x = lang_hooks.decls.omp_clause_assign_op (c, t, var);
9814 gimplify_and_add (unshare_expr(x), stmt_seqp);
9816 /* var = var op mem */
9817 switch (OMP_CLAUSE_REDUCTION_CODE (c))
9819 case TRUTH_ANDIF_EXPR:
9820 case TRUTH_ORIF_EXPR:
9821 t = fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c), integer_type_node,
9822 t, mem);
9823 gimplify_and_add (t, stmt_seqp);
9824 break;
9825 default:
9826 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9827 oacc_gimple_assign (t, OMP_CLAUSE_REDUCTION_CODE (c), mem,
9828 stmt_seqp);
9831 t = fold_build1 (NOP_EXPR, TREE_TYPE (var), t);
9832 x = lang_hooks.decls.omp_clause_assign_op (c, var, t);
9833 gimplify_and_add (unshare_expr(x), stmt_seqp);
9836 /* Increment the induction variable. */
9837 tree one = fold_build1 (NOP_EXPR, sizetype, integer_one_node);
9838 stmt = gimple_build_assign (ix, PLUS_EXPR, ix, one);
9839 gimple_seq_add_stmt (stmt_seqp, stmt);
9841 /* Go back to the top of the loop. */
9842 gimple_seq_add_stmt (stmt_seqp, gimple_build_goto (loop_header));
9844 /* Place the loop exit label here. */
9845 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_exit));
9848 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9849 scan that for reductions. */
9851 static void
9852 oacc_process_reduction_data (gimple_seq *body, gimple_seq *in_stmt_seqp,
9853 gimple_seq *out_stmt_seqp, omp_context *ctx)
9855 gimple_stmt_iterator gsi;
9856 gimple_seq inner = NULL;
9858 /* A collapse clause may have inserted a new bind block. */
9859 gsi = gsi_start (*body);
9860 while (!gsi_end_p (gsi))
9862 gimple stmt = gsi_stmt (gsi);
9863 if (gbind *bind_stmt = dyn_cast <gbind *> (stmt))
9865 inner = gimple_bind_body (bind_stmt);
9866 body = &inner;
9867 gsi = gsi_start (*body);
9869 else if (dyn_cast <gomp_for *> (stmt))
9870 break;
9871 else
9872 gsi_next (&gsi);
9875 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
9877 tree clauses, nthreads, t, c, acc_device, acc_device_host, call,
9878 enter, exit;
9879 bool reduction_found = false;
9881 gimple stmt = gsi_stmt (gsi);
9883 switch (gimple_code (stmt))
9885 case GIMPLE_OMP_FOR:
9886 clauses = gimple_omp_for_clauses (stmt);
9888 /* Search for a reduction clause. */
9889 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9890 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
9892 reduction_found = true;
9893 break;
9896 if (!reduction_found)
9897 break;
9899 ctx = maybe_lookup_ctx (stmt);
9900 t = NULL_TREE;
9902 /* Extract the number of threads. */
9903 nthreads = create_tmp_var (sizetype);
9904 t = oacc_max_threads (ctx);
9905 gimplify_assign (nthreads, t, in_stmt_seqp);
9907 /* Determine if this is kernel will be executed on the host. */
9908 call = builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE);
9909 acc_device = create_tmp_var (integer_type_node, ".acc_device_type");
9910 stmt = gimple_build_call (call, 0);
9911 gimple_call_set_lhs (stmt, acc_device);
9912 gimple_seq_add_stmt (in_stmt_seqp, stmt);
9914 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9915 acc_device_host = create_tmp_var (integer_type_node,
9916 ".acc_device_host");
9917 gimplify_assign (acc_device_host,
9918 build_int_cst (integer_type_node,
9919 GOMP_DEVICE_HOST),
9920 in_stmt_seqp);
9922 enter = create_artificial_label (UNKNOWN_LOCATION);
9923 exit = create_artificial_label (UNKNOWN_LOCATION);
9925 stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host,
9926 enter, exit);
9927 gimple_seq_add_stmt (in_stmt_seqp, stmt);
9928 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter));
9929 gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype,
9930 integer_one_node),
9931 in_stmt_seqp);
9932 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit));
9934 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9935 gimplify_assign (acc_device_host,
9936 build_int_cst (integer_type_node,
9937 GOMP_DEVICE_HOST_NONSHM),
9938 in_stmt_seqp);
9940 enter = create_artificial_label (UNKNOWN_LOCATION);
9941 exit = create_artificial_label (UNKNOWN_LOCATION);
9943 stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host,
9944 enter, exit);
9945 gimple_seq_add_stmt (in_stmt_seqp, stmt);
9946 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter));
9947 gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype,
9948 integer_one_node),
9949 in_stmt_seqp);
9950 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit));
9952 oacc_initialize_reduction_data (clauses, nthreads, in_stmt_seqp,
9953 ctx);
9954 oacc_finalize_reduction_data (clauses, nthreads, out_stmt_seqp, ctx);
9955 break;
9956 default:
9957 // Scan for other directives which support reduction here.
9958 break;
9963 /* If ctx is a worksharing context inside of a cancellable parallel
9964 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
9965 and conditional branch to parallel's cancel_label to handle
9966 cancellation in the implicit barrier. */
9968 static void
9969 maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body)
9971 gimple omp_return = gimple_seq_last_stmt (*body);
9972 gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
9973 if (gimple_omp_return_nowait_p (omp_return))
9974 return;
9975 if (ctx->outer
9976 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL
9977 && ctx->outer->cancellable)
9979 tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
9980 tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
9981 tree lhs = create_tmp_var (c_bool_type);
9982 gimple_omp_return_set_lhs (omp_return, lhs);
9983 tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
9984 gimple g = gimple_build_cond (NE_EXPR, lhs,
9985 fold_convert (c_bool_type,
9986 boolean_false_node),
9987 ctx->outer->cancel_label, fallthru_label);
9988 gimple_seq_add_stmt (body, g);
9989 gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
9993 /* Lower the OpenMP sections directive in the current statement in GSI_P.
9994 CTX is the enclosing OMP context for the current statement. */
9996 static void
9997 lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9999 tree block, control;
10000 gimple_stmt_iterator tgsi;
10001 gomp_sections *stmt;
10002 gimple t;
10003 gbind *new_stmt, *bind;
10004 gimple_seq ilist, dlist, olist, new_body;
10006 stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p));
10008 push_gimplify_context ();
10010 dlist = NULL;
10011 ilist = NULL;
10012 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
10013 &ilist, &dlist, ctx, NULL);
10015 new_body = gimple_omp_body (stmt);
10016 gimple_omp_set_body (stmt, NULL);
10017 tgsi = gsi_start (new_body);
10018 for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
10020 omp_context *sctx;
10021 gimple sec_start;
10023 sec_start = gsi_stmt (tgsi);
10024 sctx = maybe_lookup_ctx (sec_start);
10025 gcc_assert (sctx);
10027 lower_omp (gimple_omp_body_ptr (sec_start), sctx);
10028 gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
10029 GSI_CONTINUE_LINKING);
10030 gimple_omp_set_body (sec_start, NULL);
10032 if (gsi_one_before_end_p (tgsi))
10034 gimple_seq l = NULL;
10035 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
10036 &l, ctx);
10037 gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
10038 gimple_omp_section_set_last (sec_start);
10041 gsi_insert_after (&tgsi, gimple_build_omp_return (false),
10042 GSI_CONTINUE_LINKING);
10045 block = make_node (BLOCK);
10046 bind = gimple_build_bind (NULL, new_body, block);
10048 olist = NULL;
10049 lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx);
10051 block = make_node (BLOCK);
10052 new_stmt = gimple_build_bind (NULL, NULL, block);
10053 gsi_replace (gsi_p, new_stmt, true);
10055 pop_gimplify_context (new_stmt);
10056 gimple_bind_append_vars (new_stmt, ctx->block_vars);
10057 BLOCK_VARS (block) = gimple_bind_vars (bind);
10058 if (BLOCK_VARS (block))
10059 TREE_USED (block) = 1;
10061 new_body = NULL;
10062 gimple_seq_add_seq (&new_body, ilist);
10063 gimple_seq_add_stmt (&new_body, stmt);
10064 gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
10065 gimple_seq_add_stmt (&new_body, bind);
10067 control = create_tmp_var (unsigned_type_node, ".section");
10068 t = gimple_build_omp_continue (control, control);
10069 gimple_omp_sections_set_control (stmt, control);
10070 gimple_seq_add_stmt (&new_body, t);
10072 gimple_seq_add_seq (&new_body, olist);
10073 if (ctx->cancellable)
10074 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
10075 gimple_seq_add_seq (&new_body, dlist);
10077 new_body = maybe_catch_exception (new_body);
10079 t = gimple_build_omp_return
10080 (!!find_omp_clause (gimple_omp_sections_clauses (stmt),
10081 OMP_CLAUSE_NOWAIT));
10082 gimple_seq_add_stmt (&new_body, t);
10083 maybe_add_implicit_barrier_cancel (ctx, &new_body);
10085 gimple_bind_set_body (new_stmt, new_body);
10089 /* A subroutine of lower_omp_single. Expand the simple form of
10090 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10092 if (GOMP_single_start ())
10093 BODY;
10094 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10096 FIXME. It may be better to delay expanding the logic of this until
10097 pass_expand_omp. The expanded logic may make the job more difficult
10098 to a synchronization analysis pass. */
10100 static void
10101 lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p)
10103 location_t loc = gimple_location (single_stmt);
10104 tree tlabel = create_artificial_label (loc);
10105 tree flabel = create_artificial_label (loc);
10106 gimple call, cond;
10107 tree lhs, decl;
10109 decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START);
10110 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)));
10111 call = gimple_build_call (decl, 0);
10112 gimple_call_set_lhs (call, lhs);
10113 gimple_seq_add_stmt (pre_p, call);
10115 cond = gimple_build_cond (EQ_EXPR, lhs,
10116 fold_convert_loc (loc, TREE_TYPE (lhs),
10117 boolean_true_node),
10118 tlabel, flabel);
10119 gimple_seq_add_stmt (pre_p, cond);
10120 gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
10121 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
10122 gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
10126 /* A subroutine of lower_omp_single. Expand the simple form of
10127 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10129 #pragma omp single copyprivate (a, b, c)
10131 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10134 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10136 BODY;
10137 copyout.a = a;
10138 copyout.b = b;
10139 copyout.c = c;
10140 GOMP_single_copy_end (&copyout);
10142 else
10144 a = copyout_p->a;
10145 b = copyout_p->b;
10146 c = copyout_p->c;
10148 GOMP_barrier ();
10151 FIXME. It may be better to delay expanding the logic of this until
10152 pass_expand_omp. The expanded logic may make the job more difficult
10153 to a synchronization analysis pass. */
10155 static void
10156 lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p,
10157 omp_context *ctx)
10159 tree ptr_type, t, l0, l1, l2, bfn_decl;
10160 gimple_seq copyin_seq;
10161 location_t loc = gimple_location (single_stmt);
10163 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
10165 ptr_type = build_pointer_type (ctx->record_type);
10166 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
10168 l0 = create_artificial_label (loc);
10169 l1 = create_artificial_label (loc);
10170 l2 = create_artificial_label (loc);
10172 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
10173 t = build_call_expr_loc (loc, bfn_decl, 0);
10174 t = fold_convert_loc (loc, ptr_type, t);
10175 gimplify_assign (ctx->receiver_decl, t, pre_p);
10177 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
10178 build_int_cst (ptr_type, 0));
10179 t = build3 (COND_EXPR, void_type_node, t,
10180 build_and_jump (&l0), build_and_jump (&l1));
10181 gimplify_and_add (t, pre_p);
10183 gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
10185 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
10187 copyin_seq = NULL;
10188 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
10189 &copyin_seq, ctx);
10191 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
10192 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
10193 t = build_call_expr_loc (loc, bfn_decl, 1, t);
10194 gimplify_and_add (t, pre_p);
10196 t = build_and_jump (&l2);
10197 gimplify_and_add (t, pre_p);
10199 gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
10201 gimple_seq_add_seq (pre_p, copyin_seq);
10203 gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
10207 /* Expand code for an OpenMP single directive. */
10209 static void
10210 lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10212 tree block;
10213 gimple t;
10214 gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p));
10215 gbind *bind;
10216 gimple_seq bind_body, bind_body_tail = NULL, dlist;
10218 push_gimplify_context ();
10220 block = make_node (BLOCK);
10221 bind = gimple_build_bind (NULL, NULL, block);
10222 gsi_replace (gsi_p, bind, true);
10223 bind_body = NULL;
10224 dlist = NULL;
10225 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
10226 &bind_body, &dlist, ctx, NULL);
10227 lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
10229 gimple_seq_add_stmt (&bind_body, single_stmt);
10231 if (ctx->record_type)
10232 lower_omp_single_copy (single_stmt, &bind_body, ctx);
10233 else
10234 lower_omp_single_simple (single_stmt, &bind_body);
10236 gimple_omp_set_body (single_stmt, NULL);
10238 gimple_seq_add_seq (&bind_body, dlist);
10240 bind_body = maybe_catch_exception (bind_body);
10242 t = gimple_build_omp_return
10243 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt),
10244 OMP_CLAUSE_NOWAIT));
10245 gimple_seq_add_stmt (&bind_body_tail, t);
10246 maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail);
10247 if (ctx->record_type)
10249 gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
10250 tree clobber = build_constructor (ctx->record_type, NULL);
10251 TREE_THIS_VOLATILE (clobber) = 1;
10252 gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
10253 clobber), GSI_SAME_STMT);
10255 gimple_seq_add_seq (&bind_body, bind_body_tail);
10256 gimple_bind_set_body (bind, bind_body);
10258 pop_gimplify_context (bind);
10260 gimple_bind_append_vars (bind, ctx->block_vars);
10261 BLOCK_VARS (block) = ctx->block_vars;
10262 if (BLOCK_VARS (block))
10263 TREE_USED (block) = 1;
10267 /* Expand code for an OpenMP master directive. */
10269 static void
10270 lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10272 tree block, lab = NULL, x, bfn_decl;
10273 gimple stmt = gsi_stmt (*gsi_p);
10274 gbind *bind;
10275 location_t loc = gimple_location (stmt);
10276 gimple_seq tseq;
10278 push_gimplify_context ();
10280 block = make_node (BLOCK);
10281 bind = gimple_build_bind (NULL, NULL, block);
10282 gsi_replace (gsi_p, bind, true);
10283 gimple_bind_add_stmt (bind, stmt);
10285 bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
10286 x = build_call_expr_loc (loc, bfn_decl, 0);
10287 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node);
10288 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
10289 tseq = NULL;
10290 gimplify_and_add (x, &tseq);
10291 gimple_bind_add_seq (bind, tseq);
10293 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10294 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10295 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10296 gimple_omp_set_body (stmt, NULL);
10298 gimple_bind_add_stmt (bind, gimple_build_label (lab));
10300 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10302 pop_gimplify_context (bind);
10304 gimple_bind_append_vars (bind, ctx->block_vars);
10305 BLOCK_VARS (block) = ctx->block_vars;
10309 /* Expand code for an OpenMP taskgroup directive. */
10311 static void
10312 lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10314 gimple stmt = gsi_stmt (*gsi_p);
10315 gcall *x;
10316 gbind *bind;
10317 tree block = make_node (BLOCK);
10319 bind = gimple_build_bind (NULL, NULL, block);
10320 gsi_replace (gsi_p, bind, true);
10321 gimple_bind_add_stmt (bind, stmt);
10323 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
10325 gimple_bind_add_stmt (bind, x);
10327 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10328 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10329 gimple_omp_set_body (stmt, NULL);
10331 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10333 gimple_bind_append_vars (bind, ctx->block_vars);
10334 BLOCK_VARS (block) = ctx->block_vars;
10338 /* Expand code for an OpenMP ordered directive. */
10340 static void
10341 lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10343 tree block;
10344 gimple stmt = gsi_stmt (*gsi_p);
10345 gcall *x;
10346 gbind *bind;
10348 push_gimplify_context ();
10350 block = make_node (BLOCK);
10351 bind = gimple_build_bind (NULL, NULL, block);
10352 gsi_replace (gsi_p, bind, true);
10353 gimple_bind_add_stmt (bind, stmt);
10355 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
10357 gimple_bind_add_stmt (bind, x);
10359 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10360 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10361 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10362 gimple_omp_set_body (stmt, NULL);
10364 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END), 0);
10365 gimple_bind_add_stmt (bind, x);
10367 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10369 pop_gimplify_context (bind);
10371 gimple_bind_append_vars (bind, ctx->block_vars);
10372 BLOCK_VARS (block) = gimple_bind_vars (bind);
10376 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10377 substitution of a couple of function calls. But in the NAMED case,
10378 requires that languages coordinate a symbol name. It is therefore
10379 best put here in common code. */
10381 static GTY(()) hash_map<tree, tree> *critical_name_mutexes;
10383 static void
10384 lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10386 tree block;
10387 tree name, lock, unlock;
10388 gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p));
10389 gbind *bind;
10390 location_t loc = gimple_location (stmt);
10391 gimple_seq tbody;
10393 name = gimple_omp_critical_name (stmt);
10394 if (name)
10396 tree decl;
10398 if (!critical_name_mutexes)
10399 critical_name_mutexes = hash_map<tree, tree>::create_ggc (10);
10401 tree *n = critical_name_mutexes->get (name);
10402 if (n == NULL)
10404 char *new_str;
10406 decl = create_tmp_var_raw (ptr_type_node);
10408 new_str = ACONCAT ((".gomp_critical_user_",
10409 IDENTIFIER_POINTER (name), NULL));
10410 DECL_NAME (decl) = get_identifier (new_str);
10411 TREE_PUBLIC (decl) = 1;
10412 TREE_STATIC (decl) = 1;
10413 DECL_COMMON (decl) = 1;
10414 DECL_ARTIFICIAL (decl) = 1;
10415 DECL_IGNORED_P (decl) = 1;
10417 varpool_node::finalize_decl (decl);
10419 critical_name_mutexes->put (name, decl);
10421 else
10422 decl = *n;
10424 /* If '#pragma omp critical' is inside offloaded region or
10425 inside function marked as offloadable, the symbol must be
10426 marked as offloadable too. */
10427 omp_context *octx;
10428 if (cgraph_node::get (current_function_decl)->offloadable)
10429 varpool_node::get_create (decl)->offloadable = 1;
10430 else
10431 for (octx = ctx->outer; octx; octx = octx->outer)
10432 if (is_gimple_omp_offloaded (octx->stmt))
10434 varpool_node::get_create (decl)->offloadable = 1;
10435 break;
10438 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
10439 lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl));
10441 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
10442 unlock = build_call_expr_loc (loc, unlock, 1,
10443 build_fold_addr_expr_loc (loc, decl));
10445 else
10447 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
10448 lock = build_call_expr_loc (loc, lock, 0);
10450 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
10451 unlock = build_call_expr_loc (loc, unlock, 0);
10454 push_gimplify_context ();
10456 block = make_node (BLOCK);
10457 bind = gimple_build_bind (NULL, NULL, block);
10458 gsi_replace (gsi_p, bind, true);
10459 gimple_bind_add_stmt (bind, stmt);
10461 tbody = gimple_bind_body (bind);
10462 gimplify_and_add (lock, &tbody);
10463 gimple_bind_set_body (bind, tbody);
10465 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10466 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10467 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10468 gimple_omp_set_body (stmt, NULL);
10470 tbody = gimple_bind_body (bind);
10471 gimplify_and_add (unlock, &tbody);
10472 gimple_bind_set_body (bind, tbody);
10474 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10476 pop_gimplify_context (bind);
10477 gimple_bind_append_vars (bind, ctx->block_vars);
10478 BLOCK_VARS (block) = gimple_bind_vars (bind);
10482 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10483 for a lastprivate clause. Given a loop control predicate of (V
10484 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10485 is appended to *DLIST, iterator initialization is appended to
10486 *BODY_P. */
10488 static void
10489 lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
10490 gimple_seq *dlist, struct omp_context *ctx)
10492 tree clauses, cond, vinit;
10493 enum tree_code cond_code;
10494 gimple_seq stmts;
10496 cond_code = fd->loop.cond_code;
10497 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
10499 /* When possible, use a strict equality expression. This can let VRP
10500 type optimizations deduce the value and remove a copy. */
10501 if (tree_fits_shwi_p (fd->loop.step))
10503 HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
10504 if (step == 1 || step == -1)
10505 cond_code = EQ_EXPR;
10508 cond = build2 (cond_code, boolean_type_node, fd->loop.v, fd->loop.n2);
10510 clauses = gimple_omp_for_clauses (fd->for_stmt);
10511 stmts = NULL;
10512 lower_lastprivate_clauses (clauses, cond, &stmts, ctx);
10513 if (!gimple_seq_empty_p (stmts))
10515 gimple_seq_add_seq (&stmts, *dlist);
10516 *dlist = stmts;
10518 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10519 vinit = fd->loop.n1;
10520 if (cond_code == EQ_EXPR
10521 && tree_fits_shwi_p (fd->loop.n2)
10522 && ! integer_zerop (fd->loop.n2))
10523 vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
10524 else
10525 vinit = unshare_expr (vinit);
10527 /* Initialize the iterator variable, so that threads that don't execute
10528 any iterations don't execute the lastprivate clauses by accident. */
10529 gimplify_assign (fd->loop.v, vinit, body_p);
10534 /* Lower code for an OMP loop directive. */
10536 static void
10537 lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10539 tree *rhs_p, block;
10540 struct omp_for_data fd, *fdp = NULL;
10541 gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
10542 gbind *new_stmt;
10543 gimple_seq omp_for_body, body, dlist;
10544 size_t i;
10546 push_gimplify_context ();
10548 lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
10550 block = make_node (BLOCK);
10551 new_stmt = gimple_build_bind (NULL, NULL, block);
10552 /* Replace at gsi right away, so that 'stmt' is no member
10553 of a sequence anymore as we're going to add to to a different
10554 one below. */
10555 gsi_replace (gsi_p, new_stmt, true);
10557 /* Move declaration of temporaries in the loop body before we make
10558 it go away. */
10559 omp_for_body = gimple_omp_body (stmt);
10560 if (!gimple_seq_empty_p (omp_for_body)
10561 && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
10563 gbind *inner_bind
10564 = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body));
10565 tree vars = gimple_bind_vars (inner_bind);
10566 gimple_bind_append_vars (new_stmt, vars);
10567 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10568 keep them on the inner_bind and it's block. */
10569 gimple_bind_set_vars (inner_bind, NULL_TREE);
10570 if (gimple_bind_block (inner_bind))
10571 BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
10574 if (gimple_omp_for_combined_into_p (stmt))
10576 extract_omp_for_data (stmt, &fd, NULL);
10577 fdp = &fd;
10579 /* We need two temporaries with fd.loop.v type (istart/iend)
10580 and then (fd.collapse - 1) temporaries with the same
10581 type for count2 ... countN-1 vars if not constant. */
10582 size_t count = 2;
10583 tree type = fd.iter_type;
10584 if (fd.collapse > 1
10585 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
10586 count += fd.collapse - 1;
10587 bool parallel_for = gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR;
10588 tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
10589 tree clauses = *pc;
10590 if (parallel_for)
10591 outerc
10592 = find_omp_clause (gimple_omp_parallel_clauses (ctx->outer->stmt),
10593 OMP_CLAUSE__LOOPTEMP_);
10594 for (i = 0; i < count; i++)
10596 tree temp;
10597 if (parallel_for)
10599 gcc_assert (outerc);
10600 temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
10601 outerc = find_omp_clause (OMP_CLAUSE_CHAIN (outerc),
10602 OMP_CLAUSE__LOOPTEMP_);
10604 else
10606 temp = create_tmp_var (type);
10607 insert_decl_map (&ctx->outer->cb, temp, temp);
10609 *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
10610 OMP_CLAUSE_DECL (*pc) = temp;
10611 pc = &OMP_CLAUSE_CHAIN (*pc);
10613 *pc = clauses;
10616 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10617 dlist = NULL;
10618 body = NULL;
10619 lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
10620 fdp);
10621 gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt));
10623 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10625 /* Lower the header expressions. At this point, we can assume that
10626 the header is of the form:
10628 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10630 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10631 using the .omp_data_s mapping, if needed. */
10632 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
10634 rhs_p = gimple_omp_for_initial_ptr (stmt, i);
10635 if (!is_gimple_min_invariant (*rhs_p))
10636 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
10638 rhs_p = gimple_omp_for_final_ptr (stmt, i);
10639 if (!is_gimple_min_invariant (*rhs_p))
10640 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
10642 rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
10643 if (!is_gimple_min_invariant (*rhs_p))
10644 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
10647 /* Once lowered, extract the bounds and clauses. */
10648 extract_omp_for_data (stmt, &fd, NULL);
10650 lower_omp_for_lastprivate (&fd, &body, &dlist, ctx);
10652 gimple_seq_add_stmt (&body, stmt);
10653 gimple_seq_add_seq (&body, gimple_omp_body (stmt));
10655 gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
10656 fd.loop.v));
10658 /* After the loop, add exit clauses. */
10659 lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx);
10661 if (ctx->cancellable)
10662 gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
10664 gimple_seq_add_seq (&body, dlist);
10666 body = maybe_catch_exception (body);
10668 /* Region exit marker goes at the end of the loop body. */
10669 gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait));
10670 maybe_add_implicit_barrier_cancel (ctx, &body);
10671 pop_gimplify_context (new_stmt);
10673 gimple_bind_append_vars (new_stmt, ctx->block_vars);
10674 BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
10675 if (BLOCK_VARS (block))
10676 TREE_USED (block) = 1;
10678 gimple_bind_set_body (new_stmt, body);
10679 gimple_omp_set_body (stmt, NULL);
10680 gimple_omp_for_set_pre_body (stmt, NULL);
10683 /* Callback for walk_stmts. Check if the current statement only contains
10684 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10686 static tree
10687 check_combined_parallel (gimple_stmt_iterator *gsi_p,
10688 bool *handled_ops_p,
10689 struct walk_stmt_info *wi)
10691 int *info = (int *) wi->info;
10692 gimple stmt = gsi_stmt (*gsi_p);
10694 *handled_ops_p = true;
10695 switch (gimple_code (stmt))
10697 WALK_SUBSTMTS;
10699 case GIMPLE_OMP_FOR:
10700 case GIMPLE_OMP_SECTIONS:
10701 *info = *info == 0 ? 1 : -1;
10702 break;
10703 default:
10704 *info = -1;
10705 break;
10707 return NULL;
10710 struct omp_taskcopy_context
10712 /* This field must be at the beginning, as we do "inheritance": Some
10713 callback functions for tree-inline.c (e.g., omp_copy_decl)
10714 receive a copy_body_data pointer that is up-casted to an
10715 omp_context pointer. */
10716 copy_body_data cb;
10717 omp_context *ctx;
10720 static tree
10721 task_copyfn_copy_decl (tree var, copy_body_data *cb)
10723 struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
10725 if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
10726 return create_tmp_var (TREE_TYPE (var));
10728 return var;
10731 static tree
10732 task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
10734 tree name, new_fields = NULL, type, f;
10736 type = lang_hooks.types.make_type (RECORD_TYPE);
10737 name = DECL_NAME (TYPE_NAME (orig_type));
10738 name = build_decl (gimple_location (tcctx->ctx->stmt),
10739 TYPE_DECL, name, type);
10740 TYPE_NAME (type) = name;
10742 for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
10744 tree new_f = copy_node (f);
10745 DECL_CONTEXT (new_f) = type;
10746 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
10747 TREE_CHAIN (new_f) = new_fields;
10748 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
10749 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
10750 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
10751 &tcctx->cb, NULL);
10752 new_fields = new_f;
10753 tcctx->cb.decl_map->put (f, new_f);
10755 TYPE_FIELDS (type) = nreverse (new_fields);
10756 layout_type (type);
10757 return type;
10760 /* Create task copyfn. */
10762 static void
10763 create_task_copyfn (gomp_task *task_stmt, omp_context *ctx)
10765 struct function *child_cfun;
10766 tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
10767 tree record_type, srecord_type, bind, list;
10768 bool record_needs_remap = false, srecord_needs_remap = false;
10769 splay_tree_node n;
10770 struct omp_taskcopy_context tcctx;
10771 location_t loc = gimple_location (task_stmt);
10773 child_fn = gimple_omp_task_copy_fn (task_stmt);
10774 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
10775 gcc_assert (child_cfun->cfg == NULL);
10776 DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
10778 /* Reset DECL_CONTEXT on function arguments. */
10779 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
10780 DECL_CONTEXT (t) = child_fn;
10782 /* Populate the function. */
10783 push_gimplify_context ();
10784 push_cfun (child_cfun);
10786 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
10787 TREE_SIDE_EFFECTS (bind) = 1;
10788 list = NULL;
10789 DECL_SAVED_TREE (child_fn) = bind;
10790 DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
10792 /* Remap src and dst argument types if needed. */
10793 record_type = ctx->record_type;
10794 srecord_type = ctx->srecord_type;
10795 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
10796 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
10798 record_needs_remap = true;
10799 break;
10801 for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
10802 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
10804 srecord_needs_remap = true;
10805 break;
10808 if (record_needs_remap || srecord_needs_remap)
10810 memset (&tcctx, '\0', sizeof (tcctx));
10811 tcctx.cb.src_fn = ctx->cb.src_fn;
10812 tcctx.cb.dst_fn = child_fn;
10813 tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn);
10814 gcc_checking_assert (tcctx.cb.src_node);
10815 tcctx.cb.dst_node = tcctx.cb.src_node;
10816 tcctx.cb.src_cfun = ctx->cb.src_cfun;
10817 tcctx.cb.copy_decl = task_copyfn_copy_decl;
10818 tcctx.cb.eh_lp_nr = 0;
10819 tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
10820 tcctx.cb.decl_map = new hash_map<tree, tree>;
10821 tcctx.ctx = ctx;
10823 if (record_needs_remap)
10824 record_type = task_copyfn_remap_type (&tcctx, record_type);
10825 if (srecord_needs_remap)
10826 srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
10828 else
10829 tcctx.cb.decl_map = NULL;
10831 arg = DECL_ARGUMENTS (child_fn);
10832 TREE_TYPE (arg) = build_pointer_type (record_type);
10833 sarg = DECL_CHAIN (arg);
10834 TREE_TYPE (sarg) = build_pointer_type (srecord_type);
10836 /* First pass: initialize temporaries used in record_type and srecord_type
10837 sizes and field offsets. */
10838 if (tcctx.cb.decl_map)
10839 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
10840 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
10842 tree *p;
10844 decl = OMP_CLAUSE_DECL (c);
10845 p = tcctx.cb.decl_map->get (decl);
10846 if (p == NULL)
10847 continue;
10848 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
10849 sf = (tree) n->value;
10850 sf = *tcctx.cb.decl_map->get (sf);
10851 src = build_simple_mem_ref_loc (loc, sarg);
10852 src = omp_build_component_ref (src, sf);
10853 t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
10854 append_to_statement_list (t, &list);
10857 /* Second pass: copy shared var pointers and copy construct non-VLA
10858 firstprivate vars. */
10859 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
10860 switch (OMP_CLAUSE_CODE (c))
10862 case OMP_CLAUSE_SHARED:
10863 decl = OMP_CLAUSE_DECL (c);
10864 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
10865 if (n == NULL)
10866 break;
10867 f = (tree) n->value;
10868 if (tcctx.cb.decl_map)
10869 f = *tcctx.cb.decl_map->get (f);
10870 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
10871 sf = (tree) n->value;
10872 if (tcctx.cb.decl_map)
10873 sf = *tcctx.cb.decl_map->get (sf);
10874 src = build_simple_mem_ref_loc (loc, sarg);
10875 src = omp_build_component_ref (src, sf);
10876 dst = build_simple_mem_ref_loc (loc, arg);
10877 dst = omp_build_component_ref (dst, f);
10878 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
10879 append_to_statement_list (t, &list);
10880 break;
10881 case OMP_CLAUSE_FIRSTPRIVATE:
10882 decl = OMP_CLAUSE_DECL (c);
10883 if (is_variable_sized (decl))
10884 break;
10885 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
10886 if (n == NULL)
10887 break;
10888 f = (tree) n->value;
10889 if (tcctx.cb.decl_map)
10890 f = *tcctx.cb.decl_map->get (f);
10891 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
10892 if (n != NULL)
10894 sf = (tree) n->value;
10895 if (tcctx.cb.decl_map)
10896 sf = *tcctx.cb.decl_map->get (sf);
10897 src = build_simple_mem_ref_loc (loc, sarg);
10898 src = omp_build_component_ref (src, sf);
10899 if (use_pointer_for_field (decl, NULL) || is_reference (decl))
10900 src = build_simple_mem_ref_loc (loc, src);
10902 else
10903 src = decl;
10904 dst = build_simple_mem_ref_loc (loc, arg);
10905 dst = omp_build_component_ref (dst, f);
10906 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
10907 append_to_statement_list (t, &list);
10908 break;
10909 case OMP_CLAUSE_PRIVATE:
10910 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
10911 break;
10912 decl = OMP_CLAUSE_DECL (c);
10913 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
10914 f = (tree) n->value;
10915 if (tcctx.cb.decl_map)
10916 f = *tcctx.cb.decl_map->get (f);
10917 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
10918 if (n != NULL)
10920 sf = (tree) n->value;
10921 if (tcctx.cb.decl_map)
10922 sf = *tcctx.cb.decl_map->get (sf);
10923 src = build_simple_mem_ref_loc (loc, sarg);
10924 src = omp_build_component_ref (src, sf);
10925 if (use_pointer_for_field (decl, NULL))
10926 src = build_simple_mem_ref_loc (loc, src);
10928 else
10929 src = decl;
10930 dst = build_simple_mem_ref_loc (loc, arg);
10931 dst = omp_build_component_ref (dst, f);
10932 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
10933 append_to_statement_list (t, &list);
10934 break;
10935 default:
10936 break;
10939 /* Last pass: handle VLA firstprivates. */
10940 if (tcctx.cb.decl_map)
10941 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
10942 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
10944 tree ind, ptr, df;
10946 decl = OMP_CLAUSE_DECL (c);
10947 if (!is_variable_sized (decl))
10948 continue;
10949 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
10950 if (n == NULL)
10951 continue;
10952 f = (tree) n->value;
10953 f = *tcctx.cb.decl_map->get (f);
10954 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
10955 ind = DECL_VALUE_EXPR (decl);
10956 gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
10957 gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
10958 n = splay_tree_lookup (ctx->sfield_map,
10959 (splay_tree_key) TREE_OPERAND (ind, 0));
10960 sf = (tree) n->value;
10961 sf = *tcctx.cb.decl_map->get (sf);
10962 src = build_simple_mem_ref_loc (loc, sarg);
10963 src = omp_build_component_ref (src, sf);
10964 src = build_simple_mem_ref_loc (loc, src);
10965 dst = build_simple_mem_ref_loc (loc, arg);
10966 dst = omp_build_component_ref (dst, f);
10967 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
10968 append_to_statement_list (t, &list);
10969 n = splay_tree_lookup (ctx->field_map,
10970 (splay_tree_key) TREE_OPERAND (ind, 0));
10971 df = (tree) n->value;
10972 df = *tcctx.cb.decl_map->get (df);
10973 ptr = build_simple_mem_ref_loc (loc, arg);
10974 ptr = omp_build_component_ref (ptr, df);
10975 t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
10976 build_fold_addr_expr_loc (loc, dst));
10977 append_to_statement_list (t, &list);
10980 t = build1 (RETURN_EXPR, void_type_node, NULL);
10981 append_to_statement_list (t, &list);
10983 if (tcctx.cb.decl_map)
10984 delete tcctx.cb.decl_map;
10985 pop_gimplify_context (NULL);
10986 BIND_EXPR_BODY (bind) = list;
10987 pop_cfun ();
10990 static void
10991 lower_depend_clauses (gimple stmt, gimple_seq *iseq, gimple_seq *oseq)
10993 tree c, clauses;
10994 gimple g;
10995 size_t n_in = 0, n_out = 0, idx = 2, i;
10997 clauses = find_omp_clause (gimple_omp_task_clauses (stmt),
10998 OMP_CLAUSE_DEPEND);
10999 gcc_assert (clauses);
11000 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
11001 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
11002 switch (OMP_CLAUSE_DEPEND_KIND (c))
11004 case OMP_CLAUSE_DEPEND_IN:
11005 n_in++;
11006 break;
11007 case OMP_CLAUSE_DEPEND_OUT:
11008 case OMP_CLAUSE_DEPEND_INOUT:
11009 n_out++;
11010 break;
11011 default:
11012 gcc_unreachable ();
11014 tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2);
11015 tree array = create_tmp_var (type);
11016 tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
11017 NULL_TREE);
11018 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out));
11019 gimple_seq_add_stmt (iseq, g);
11020 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
11021 NULL_TREE);
11022 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out));
11023 gimple_seq_add_stmt (iseq, g);
11024 for (i = 0; i < 2; i++)
11026 if ((i ? n_in : n_out) == 0)
11027 continue;
11028 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
11029 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
11030 && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i))
11032 tree t = OMP_CLAUSE_DECL (c);
11033 t = fold_convert (ptr_type_node, t);
11034 gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
11035 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
11036 NULL_TREE, NULL_TREE);
11037 g = gimple_build_assign (r, t);
11038 gimple_seq_add_stmt (iseq, g);
11041 tree *p = gimple_omp_task_clauses_ptr (stmt);
11042 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
11043 OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
11044 OMP_CLAUSE_CHAIN (c) = *p;
11045 *p = c;
11046 tree clobber = build_constructor (type, NULL);
11047 TREE_THIS_VOLATILE (clobber) = 1;
11048 g = gimple_build_assign (array, clobber);
11049 gimple_seq_add_stmt (oseq, g);
11052 /* Lower the OpenMP parallel or task directive in the current statement
11053 in GSI_P. CTX holds context information for the directive. */
11055 static void
11056 lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11058 tree clauses;
11059 tree child_fn, t;
11060 gimple stmt = gsi_stmt (*gsi_p);
11061 gbind *par_bind, *bind, *dep_bind = NULL;
11062 gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body;
11063 location_t loc = gimple_location (stmt);
11065 clauses = gimple_omp_taskreg_clauses (stmt);
11066 par_bind
11067 = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
11068 par_body = gimple_bind_body (par_bind);
11069 child_fn = ctx->cb.dst_fn;
11070 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
11071 && !gimple_omp_parallel_combined_p (stmt))
11073 struct walk_stmt_info wi;
11074 int ws_num = 0;
11076 memset (&wi, 0, sizeof (wi));
11077 wi.info = &ws_num;
11078 wi.val_only = true;
11079 walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
11080 if (ws_num == 1)
11081 gimple_omp_parallel_set_combined_p (stmt, true);
11083 gimple_seq dep_ilist = NULL;
11084 gimple_seq dep_olist = NULL;
11085 if (gimple_code (stmt) == GIMPLE_OMP_TASK
11086 && find_omp_clause (clauses, OMP_CLAUSE_DEPEND))
11088 push_gimplify_context ();
11089 dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
11090 lower_depend_clauses (stmt, &dep_ilist, &dep_olist);
11093 if (ctx->srecord_type)
11094 create_task_copyfn (as_a <gomp_task *> (stmt), ctx);
11096 push_gimplify_context ();
11098 par_olist = NULL;
11099 par_ilist = NULL;
11100 par_rlist = NULL;
11101 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
11102 lower_omp (&par_body, ctx);
11103 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL)
11104 lower_reduction_clauses (clauses, &par_rlist, ctx);
11106 /* Declare all the variables created by mapping and the variables
11107 declared in the scope of the parallel body. */
11108 record_vars_into (ctx->block_vars, child_fn);
11109 record_vars_into (gimple_bind_vars (par_bind), child_fn);
11111 if (ctx->record_type)
11113 ctx->sender_decl
11114 = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
11115 : ctx->record_type, ".omp_data_o");
11116 DECL_NAMELESS (ctx->sender_decl) = 1;
11117 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
11118 gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
11121 olist = NULL;
11122 ilist = NULL;
11123 lower_send_clauses (clauses, &ilist, &olist, ctx);
11124 lower_send_shared_vars (&ilist, &olist, ctx);
11126 if (ctx->record_type)
11128 tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL);
11129 TREE_THIS_VOLATILE (clobber) = 1;
11130 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
11131 clobber));
11134 /* Once all the expansions are done, sequence all the different
11135 fragments inside gimple_omp_body. */
11137 new_body = NULL;
11139 if (ctx->record_type)
11141 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
11142 /* fixup_child_record_type might have changed receiver_decl's type. */
11143 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
11144 gimple_seq_add_stmt (&new_body,
11145 gimple_build_assign (ctx->receiver_decl, t));
11148 gimple_seq_add_seq (&new_body, par_ilist);
11149 gimple_seq_add_seq (&new_body, par_body);
11150 gimple_seq_add_seq (&new_body, par_rlist);
11151 if (ctx->cancellable)
11152 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
11153 gimple_seq_add_seq (&new_body, par_olist);
11154 new_body = maybe_catch_exception (new_body);
11155 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
11156 gimple_omp_set_body (stmt, new_body);
11158 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
11159 gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
11160 gimple_bind_add_seq (bind, ilist);
11161 gimple_bind_add_stmt (bind, stmt);
11162 gimple_bind_add_seq (bind, olist);
11164 pop_gimplify_context (NULL);
11166 if (dep_bind)
11168 gimple_bind_add_seq (dep_bind, dep_ilist);
11169 gimple_bind_add_stmt (dep_bind, bind);
11170 gimple_bind_add_seq (dep_bind, dep_olist);
11171 pop_gimplify_context (dep_bind);
11175 /* Lower the GIMPLE_OMP_TARGET in the current statement
11176 in GSI_P. CTX holds context information for the directive. */
11178 static void
11179 lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11181 tree clauses;
11182 tree child_fn, t, c;
11183 gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p));
11184 gbind *tgt_bind, *bind;
11185 gimple_seq tgt_body, olist, ilist, orlist, irlist, new_body;
11186 location_t loc = gimple_location (stmt);
11187 bool offloaded, data_region;
11188 unsigned int map_cnt = 0;
11190 offloaded = is_gimple_omp_offloaded (stmt);
11191 switch (gimple_omp_target_kind (stmt))
11193 case GF_OMP_TARGET_KIND_REGION:
11194 case GF_OMP_TARGET_KIND_UPDATE:
11195 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
11196 case GF_OMP_TARGET_KIND_OACC_KERNELS:
11197 case GF_OMP_TARGET_KIND_OACC_UPDATE:
11198 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
11199 data_region = false;
11200 break;
11201 case GF_OMP_TARGET_KIND_DATA:
11202 case GF_OMP_TARGET_KIND_OACC_DATA:
11203 data_region = true;
11204 break;
11205 default:
11206 gcc_unreachable ();
11209 clauses = gimple_omp_target_clauses (stmt);
11211 tgt_bind = NULL;
11212 tgt_body = NULL;
11213 if (offloaded)
11215 tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt));
11216 tgt_body = gimple_bind_body (tgt_bind);
11218 else if (data_region)
11219 tgt_body = gimple_omp_body (stmt);
11220 child_fn = ctx->cb.dst_fn;
11222 push_gimplify_context ();
11224 irlist = NULL;
11225 orlist = NULL;
11226 if (offloaded
11227 && is_gimple_omp_oacc (stmt))
11228 oacc_process_reduction_data (&tgt_body, &irlist, &orlist, ctx);
11230 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
11231 switch (OMP_CLAUSE_CODE (c))
11233 tree var, x;
11235 default:
11236 break;
11237 case OMP_CLAUSE_MAP:
11238 #ifdef ENABLE_CHECKING
11239 /* First check what we're prepared to handle in the following. */
11240 switch (OMP_CLAUSE_MAP_KIND (c))
11242 case GOMP_MAP_ALLOC:
11243 case GOMP_MAP_TO:
11244 case GOMP_MAP_FROM:
11245 case GOMP_MAP_TOFROM:
11246 case GOMP_MAP_POINTER:
11247 case GOMP_MAP_TO_PSET:
11248 break;
11249 case GOMP_MAP_FORCE_ALLOC:
11250 case GOMP_MAP_FORCE_TO:
11251 case GOMP_MAP_FORCE_FROM:
11252 case GOMP_MAP_FORCE_TOFROM:
11253 case GOMP_MAP_FORCE_PRESENT:
11254 case GOMP_MAP_FORCE_DEALLOC:
11255 case GOMP_MAP_FORCE_DEVICEPTR:
11256 gcc_assert (is_gimple_omp_oacc (stmt));
11257 break;
11258 default:
11259 gcc_unreachable ();
11261 #endif
11262 /* FALLTHRU */
11263 case OMP_CLAUSE_TO:
11264 case OMP_CLAUSE_FROM:
11265 var = OMP_CLAUSE_DECL (c);
11266 if (!DECL_P (var))
11268 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
11269 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
11270 map_cnt++;
11271 continue;
11274 if (DECL_SIZE (var)
11275 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
11277 tree var2 = DECL_VALUE_EXPR (var);
11278 gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
11279 var2 = TREE_OPERAND (var2, 0);
11280 gcc_assert (DECL_P (var2));
11281 var = var2;
11284 if (!maybe_lookup_field (var, ctx))
11285 continue;
11287 if (offloaded)
11289 x = build_receiver_ref (var, true, ctx);
11290 tree new_var = lookup_decl (var, ctx);
11291 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
11292 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
11293 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
11294 x = build_simple_mem_ref (x);
11295 SET_DECL_VALUE_EXPR (new_var, x);
11296 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
11298 map_cnt++;
11301 if (offloaded)
11303 target_nesting_level++;
11304 lower_omp (&tgt_body, ctx);
11305 target_nesting_level--;
11307 else if (data_region)
11308 lower_omp (&tgt_body, ctx);
11310 if (offloaded)
11312 /* Declare all the variables created by mapping and the variables
11313 declared in the scope of the target body. */
11314 record_vars_into (ctx->block_vars, child_fn);
11315 record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
11318 olist = NULL;
11319 ilist = NULL;
11320 if (ctx->record_type)
11322 ctx->sender_decl
11323 = create_tmp_var (ctx->record_type, ".omp_data_arr");
11324 DECL_NAMELESS (ctx->sender_decl) = 1;
11325 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
11326 t = make_tree_vec (3);
11327 TREE_VEC_ELT (t, 0) = ctx->sender_decl;
11328 TREE_VEC_ELT (t, 1)
11329 = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt),
11330 ".omp_data_sizes");
11331 DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
11332 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
11333 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
11334 tree tkind_type;
11335 int talign_shift;
11336 if (is_gimple_omp_oacc (stmt))
11338 tkind_type = short_unsigned_type_node;
11339 talign_shift = 8;
11341 else
11343 tkind_type = unsigned_char_type_node;
11344 talign_shift = 3;
11346 TREE_VEC_ELT (t, 2)
11347 = create_tmp_var (build_array_type_nelts (tkind_type, map_cnt),
11348 ".omp_data_kinds");
11349 DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
11350 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
11351 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
11352 gimple_omp_target_set_data_arg (stmt, t);
11354 vec<constructor_elt, va_gc> *vsize;
11355 vec<constructor_elt, va_gc> *vkind;
11356 vec_alloc (vsize, map_cnt);
11357 vec_alloc (vkind, map_cnt);
11358 unsigned int map_idx = 0;
11360 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
11361 switch (OMP_CLAUSE_CODE (c))
11363 tree ovar, nc;
11365 default:
11366 break;
11367 case OMP_CLAUSE_MAP:
11368 case OMP_CLAUSE_TO:
11369 case OMP_CLAUSE_FROM:
11370 nc = c;
11371 ovar = OMP_CLAUSE_DECL (c);
11372 if (!DECL_P (ovar))
11374 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
11375 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
11377 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c))
11378 == get_base_address (ovar));
11379 nc = OMP_CLAUSE_CHAIN (c);
11380 ovar = OMP_CLAUSE_DECL (nc);
11382 else
11384 tree x = build_sender_ref (ovar, ctx);
11385 tree v
11386 = build_fold_addr_expr_with_type (ovar, ptr_type_node);
11387 gimplify_assign (x, v, &ilist);
11388 nc = NULL_TREE;
11391 else
11393 if (DECL_SIZE (ovar)
11394 && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST)
11396 tree ovar2 = DECL_VALUE_EXPR (ovar);
11397 gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
11398 ovar2 = TREE_OPERAND (ovar2, 0);
11399 gcc_assert (DECL_P (ovar2));
11400 ovar = ovar2;
11402 if (!maybe_lookup_field (ovar, ctx))
11403 continue;
11406 unsigned int talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
11407 if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
11408 talign = DECL_ALIGN_UNIT (ovar);
11409 if (nc)
11411 tree var = lookup_decl_in_outer_ctx (ovar, ctx);
11412 tree x = build_sender_ref (ovar, ctx);
11413 if (maybe_lookup_oacc_reduction (var, ctx))
11415 gcc_checking_assert (offloaded
11416 && is_gimple_omp_oacc (stmt));
11417 gimplify_assign (x, var, &ilist);
11419 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
11420 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
11421 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
11422 && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
11424 gcc_assert (offloaded);
11425 tree avar
11426 = create_tmp_var (TREE_TYPE (TREE_TYPE (x)));
11427 mark_addressable (avar);
11428 gimplify_assign (avar, build_fold_addr_expr (var), &ilist);
11429 talign = DECL_ALIGN_UNIT (avar);
11430 avar = build_fold_addr_expr (avar);
11431 gimplify_assign (x, avar, &ilist);
11433 else if (is_gimple_reg (var))
11435 gcc_assert (offloaded);
11436 tree avar = create_tmp_var (TREE_TYPE (var));
11437 mark_addressable (avar);
11438 enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c);
11439 if (GOMP_MAP_COPY_TO_P (map_kind)
11440 || map_kind == GOMP_MAP_POINTER
11441 || map_kind == GOMP_MAP_TO_PSET
11442 || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
11443 gimplify_assign (avar, var, &ilist);
11444 avar = build_fold_addr_expr (avar);
11445 gimplify_assign (x, avar, &ilist);
11446 if ((GOMP_MAP_COPY_FROM_P (map_kind)
11447 || map_kind == GOMP_MAP_FORCE_DEVICEPTR)
11448 && !TYPE_READONLY (TREE_TYPE (var)))
11450 x = build_sender_ref (ovar, ctx);
11451 x = build_simple_mem_ref (x);
11452 gimplify_assign (var, x, &olist);
11455 else
11457 var = build_fold_addr_expr (var);
11458 gimplify_assign (x, var, &ilist);
11461 tree s = OMP_CLAUSE_SIZE (c);
11462 if (s == NULL_TREE)
11463 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
11464 s = fold_convert (size_type_node, s);
11465 tree purpose = size_int (map_idx++);
11466 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
11467 if (TREE_CODE (s) != INTEGER_CST)
11468 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
11470 unsigned HOST_WIDE_INT tkind;
11471 switch (OMP_CLAUSE_CODE (c))
11473 case OMP_CLAUSE_MAP:
11474 tkind = OMP_CLAUSE_MAP_KIND (c);
11475 break;
11476 case OMP_CLAUSE_TO:
11477 tkind = GOMP_MAP_TO;
11478 break;
11479 case OMP_CLAUSE_FROM:
11480 tkind = GOMP_MAP_FROM;
11481 break;
11482 default:
11483 gcc_unreachable ();
11485 gcc_checking_assert (tkind
11486 < (HOST_WIDE_INT_C (1U) << talign_shift));
11487 talign = ceil_log2 (talign);
11488 tkind |= talign << talign_shift;
11489 gcc_checking_assert (tkind
11490 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
11491 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
11492 build_int_cstu (tkind_type, tkind));
11493 if (nc && nc != c)
11494 c = nc;
11497 gcc_assert (map_idx == map_cnt);
11499 DECL_INITIAL (TREE_VEC_ELT (t, 1))
11500 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
11501 DECL_INITIAL (TREE_VEC_ELT (t, 2))
11502 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
11503 if (!TREE_STATIC (TREE_VEC_ELT (t, 1)))
11505 gimple_seq initlist = NULL;
11506 force_gimple_operand (build1 (DECL_EXPR, void_type_node,
11507 TREE_VEC_ELT (t, 1)),
11508 &initlist, true, NULL_TREE);
11509 gimple_seq_add_seq (&ilist, initlist);
11511 tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)),
11512 NULL);
11513 TREE_THIS_VOLATILE (clobber) = 1;
11514 gimple_seq_add_stmt (&olist,
11515 gimple_build_assign (TREE_VEC_ELT (t, 1),
11516 clobber));
11519 tree clobber = build_constructor (ctx->record_type, NULL);
11520 TREE_THIS_VOLATILE (clobber) = 1;
11521 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
11522 clobber));
11525 /* Once all the expansions are done, sequence all the different
11526 fragments inside gimple_omp_body. */
11528 new_body = NULL;
11530 if (offloaded
11531 && ctx->record_type)
11533 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
11534 /* fixup_child_record_type might have changed receiver_decl's type. */
11535 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
11536 gimple_seq_add_stmt (&new_body,
11537 gimple_build_assign (ctx->receiver_decl, t));
11540 if (offloaded)
11542 gimple_seq_add_seq (&new_body, tgt_body);
11543 new_body = maybe_catch_exception (new_body);
11545 else if (data_region)
11546 new_body = tgt_body;
11547 if (offloaded || data_region)
11549 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
11550 gimple_omp_set_body (stmt, new_body);
11553 bind = gimple_build_bind (NULL, NULL,
11554 tgt_bind ? gimple_bind_block (tgt_bind)
11555 : NULL_TREE);
11556 gsi_replace (gsi_p, bind, true);
11557 gimple_bind_add_seq (bind, irlist);
11558 gimple_bind_add_seq (bind, ilist);
11559 gimple_bind_add_stmt (bind, stmt);
11560 gimple_bind_add_seq (bind, olist);
11561 gimple_bind_add_seq (bind, orlist);
11563 pop_gimplify_context (NULL);
11566 /* Expand code for an OpenMP teams directive. */
11568 static void
11569 lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11571 gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p));
11572 push_gimplify_context ();
11574 tree block = make_node (BLOCK);
11575 gbind *bind = gimple_build_bind (NULL, NULL, block);
11576 gsi_replace (gsi_p, bind, true);
11577 gimple_seq bind_body = NULL;
11578 gimple_seq dlist = NULL;
11579 gimple_seq olist = NULL;
11581 tree num_teams = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
11582 OMP_CLAUSE_NUM_TEAMS);
11583 if (num_teams == NULL_TREE)
11584 num_teams = build_int_cst (unsigned_type_node, 0);
11585 else
11587 num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams);
11588 num_teams = fold_convert (unsigned_type_node, num_teams);
11589 gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
11591 tree thread_limit = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
11592 OMP_CLAUSE_THREAD_LIMIT);
11593 if (thread_limit == NULL_TREE)
11594 thread_limit = build_int_cst (unsigned_type_node, 0);
11595 else
11597 thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
11598 thread_limit = fold_convert (unsigned_type_node, thread_limit);
11599 gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
11600 fb_rvalue);
11603 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
11604 &bind_body, &dlist, ctx, NULL);
11605 lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
11606 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx);
11607 gimple_seq_add_stmt (&bind_body, teams_stmt);
11609 location_t loc = gimple_location (teams_stmt);
11610 tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS);
11611 gimple call = gimple_build_call (decl, 2, num_teams, thread_limit);
11612 gimple_set_location (call, loc);
11613 gimple_seq_add_stmt (&bind_body, call);
11615 gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
11616 gimple_omp_set_body (teams_stmt, NULL);
11617 gimple_seq_add_seq (&bind_body, olist);
11618 gimple_seq_add_seq (&bind_body, dlist);
11619 gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
11620 gimple_bind_set_body (bind, bind_body);
11622 pop_gimplify_context (bind);
11624 gimple_bind_append_vars (bind, ctx->block_vars);
11625 BLOCK_VARS (block) = ctx->block_vars;
11626 if (BLOCK_VARS (block))
11627 TREE_USED (block) = 1;
11631 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11632 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11633 of OMP context, but with task_shared_vars set. */
11635 static tree
11636 lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
11637 void *data)
11639 tree t = *tp;
11641 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11642 if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t))
11643 return t;
11645 if (task_shared_vars
11646 && DECL_P (t)
11647 && bitmap_bit_p (task_shared_vars, DECL_UID (t)))
11648 return t;
11650 /* If a global variable has been privatized, TREE_CONSTANT on
11651 ADDR_EXPR might be wrong. */
11652 if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
11653 recompute_tree_invariant_for_addr_expr (t);
11655 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
11656 return NULL_TREE;
11659 static void
11660 lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11662 gimple stmt = gsi_stmt (*gsi_p);
11663 struct walk_stmt_info wi;
11664 gcall *call_stmt;
11666 if (gimple_has_location (stmt))
11667 input_location = gimple_location (stmt);
11669 if (task_shared_vars)
11670 memset (&wi, '\0', sizeof (wi));
11672 /* If we have issued syntax errors, avoid doing any heavy lifting.
11673 Just replace the OMP directives with a NOP to avoid
11674 confusing RTL expansion. */
11675 if (seen_error () && is_gimple_omp (stmt))
11677 gsi_replace (gsi_p, gimple_build_nop (), true);
11678 return;
11681 switch (gimple_code (stmt))
11683 case GIMPLE_COND:
11685 gcond *cond_stmt = as_a <gcond *> (stmt);
11686 if ((ctx || task_shared_vars)
11687 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt),
11688 lower_omp_regimplify_p,
11689 ctx ? NULL : &wi, NULL)
11690 || walk_tree (gimple_cond_rhs_ptr (cond_stmt),
11691 lower_omp_regimplify_p,
11692 ctx ? NULL : &wi, NULL)))
11693 gimple_regimplify_operands (cond_stmt, gsi_p);
11695 break;
11696 case GIMPLE_CATCH:
11697 lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx);
11698 break;
11699 case GIMPLE_EH_FILTER:
11700 lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
11701 break;
11702 case GIMPLE_TRY:
11703 lower_omp (gimple_try_eval_ptr (stmt), ctx);
11704 lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
11705 break;
11706 case GIMPLE_TRANSACTION:
11707 lower_omp (gimple_transaction_body_ptr (
11708 as_a <gtransaction *> (stmt)),
11709 ctx);
11710 break;
11711 case GIMPLE_BIND:
11712 lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx);
11713 break;
11714 case GIMPLE_OMP_PARALLEL:
11715 case GIMPLE_OMP_TASK:
11716 ctx = maybe_lookup_ctx (stmt);
11717 gcc_assert (ctx);
11718 if (ctx->cancellable)
11719 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
11720 lower_omp_taskreg (gsi_p, ctx);
11721 break;
11722 case GIMPLE_OMP_FOR:
11723 ctx = maybe_lookup_ctx (stmt);
11724 gcc_assert (ctx);
11725 if (ctx->cancellable)
11726 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
11727 lower_omp_for (gsi_p, ctx);
11728 break;
11729 case GIMPLE_OMP_SECTIONS:
11730 ctx = maybe_lookup_ctx (stmt);
11731 gcc_assert (ctx);
11732 if (ctx->cancellable)
11733 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
11734 lower_omp_sections (gsi_p, ctx);
11735 break;
11736 case GIMPLE_OMP_SINGLE:
11737 ctx = maybe_lookup_ctx (stmt);
11738 gcc_assert (ctx);
11739 lower_omp_single (gsi_p, ctx);
11740 break;
11741 case GIMPLE_OMP_MASTER:
11742 ctx = maybe_lookup_ctx (stmt);
11743 gcc_assert (ctx);
11744 lower_omp_master (gsi_p, ctx);
11745 break;
11746 case GIMPLE_OMP_TASKGROUP:
11747 ctx = maybe_lookup_ctx (stmt);
11748 gcc_assert (ctx);
11749 lower_omp_taskgroup (gsi_p, ctx);
11750 break;
11751 case GIMPLE_OMP_ORDERED:
11752 ctx = maybe_lookup_ctx (stmt);
11753 gcc_assert (ctx);
11754 lower_omp_ordered (gsi_p, ctx);
11755 break;
11756 case GIMPLE_OMP_CRITICAL:
11757 ctx = maybe_lookup_ctx (stmt);
11758 gcc_assert (ctx);
11759 lower_omp_critical (gsi_p, ctx);
11760 break;
11761 case GIMPLE_OMP_ATOMIC_LOAD:
11762 if ((ctx || task_shared_vars)
11763 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
11764 as_a <gomp_atomic_load *> (stmt)),
11765 lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
11766 gimple_regimplify_operands (stmt, gsi_p);
11767 break;
11768 case GIMPLE_OMP_TARGET:
11769 ctx = maybe_lookup_ctx (stmt);
11770 gcc_assert (ctx);
11771 lower_omp_target (gsi_p, ctx);
11772 break;
11773 case GIMPLE_OMP_TEAMS:
11774 ctx = maybe_lookup_ctx (stmt);
11775 gcc_assert (ctx);
11776 lower_omp_teams (gsi_p, ctx);
11777 break;
11778 case GIMPLE_CALL:
11779 tree fndecl;
11780 call_stmt = as_a <gcall *> (stmt);
11781 fndecl = gimple_call_fndecl (call_stmt);
11782 if (fndecl
11783 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
11784 switch (DECL_FUNCTION_CODE (fndecl))
11786 case BUILT_IN_GOMP_BARRIER:
11787 if (ctx == NULL)
11788 break;
11789 /* FALLTHRU */
11790 case BUILT_IN_GOMP_CANCEL:
11791 case BUILT_IN_GOMP_CANCELLATION_POINT:
11792 omp_context *cctx;
11793 cctx = ctx;
11794 if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
11795 cctx = cctx->outer;
11796 gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE);
11797 if (!cctx->cancellable)
11799 if (DECL_FUNCTION_CODE (fndecl)
11800 == BUILT_IN_GOMP_CANCELLATION_POINT)
11802 stmt = gimple_build_nop ();
11803 gsi_replace (gsi_p, stmt, false);
11805 break;
11807 if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
11809 fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
11810 gimple_call_set_fndecl (call_stmt, fndecl);
11811 gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl));
11813 tree lhs;
11814 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)));
11815 gimple_call_set_lhs (call_stmt, lhs);
11816 tree fallthru_label;
11817 fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
11818 gimple g;
11819 g = gimple_build_label (fallthru_label);
11820 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
11821 g = gimple_build_cond (NE_EXPR, lhs,
11822 fold_convert (TREE_TYPE (lhs),
11823 boolean_false_node),
11824 cctx->cancel_label, fallthru_label);
11825 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
11826 break;
11827 default:
11828 break;
11830 /* FALLTHRU */
11831 default:
11832 if ((ctx || task_shared_vars)
11833 && walk_gimple_op (stmt, lower_omp_regimplify_p,
11834 ctx ? NULL : &wi))
11836 /* Just remove clobbers, this should happen only if we have
11837 "privatized" local addressable variables in SIMD regions,
11838 the clobber isn't needed in that case and gimplifying address
11839 of the ARRAY_REF into a pointer and creating MEM_REF based
11840 clobber would create worse code than we get with the clobber
11841 dropped. */
11842 if (gimple_clobber_p (stmt))
11844 gsi_replace (gsi_p, gimple_build_nop (), true);
11845 break;
11847 gimple_regimplify_operands (stmt, gsi_p);
11849 break;
11853 static void
11854 lower_omp (gimple_seq *body, omp_context *ctx)
11856 location_t saved_location = input_location;
11857 gimple_stmt_iterator gsi;
11858 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
11859 lower_omp_1 (&gsi, ctx);
11860 /* During gimplification, we haven't folded statments inside offloading
11861 regions (gimplify.c:maybe_fold_stmt); do that now. */
11862 if (target_nesting_level)
11863 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
11864 fold_stmt (&gsi);
11865 input_location = saved_location;
11868 /* Main entry point. */
11870 static unsigned int
11871 execute_lower_omp (void)
11873 gimple_seq body;
11874 int i;
11875 omp_context *ctx;
11877 /* This pass always runs, to provide PROP_gimple_lomp.
11878 But often, there is nothing to do. */
11879 if (flag_cilkplus == 0 && flag_openacc == 0 && flag_openmp == 0
11880 && flag_openmp_simd == 0)
11881 return 0;
11883 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
11884 delete_omp_context);
11886 body = gimple_body (current_function_decl);
11887 scan_omp (&body, NULL);
11888 gcc_assert (taskreg_nesting_level == 0);
11889 FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
11890 finish_taskreg_scan (ctx);
11891 taskreg_contexts.release ();
11893 if (all_contexts->root)
11895 if (task_shared_vars)
11896 push_gimplify_context ();
11897 lower_omp (&body, NULL);
11898 if (task_shared_vars)
11899 pop_gimplify_context (NULL);
11902 if (all_contexts)
11904 splay_tree_delete (all_contexts);
11905 all_contexts = NULL;
11907 BITMAP_FREE (task_shared_vars);
11908 return 0;
11911 namespace {
11913 const pass_data pass_data_lower_omp =
11915 GIMPLE_PASS, /* type */
11916 "omplower", /* name */
11917 OPTGROUP_NONE, /* optinfo_flags */
11918 TV_NONE, /* tv_id */
11919 PROP_gimple_any, /* properties_required */
11920 PROP_gimple_lomp, /* properties_provided */
11921 0, /* properties_destroyed */
11922 0, /* todo_flags_start */
11923 0, /* todo_flags_finish */
11926 class pass_lower_omp : public gimple_opt_pass
11928 public:
11929 pass_lower_omp (gcc::context *ctxt)
11930 : gimple_opt_pass (pass_data_lower_omp, ctxt)
11933 /* opt_pass methods: */
11934 virtual unsigned int execute (function *) { return execute_lower_omp (); }
11936 }; // class pass_lower_omp
11938 } // anon namespace
11940 gimple_opt_pass *
11941 make_pass_lower_omp (gcc::context *ctxt)
11943 return new pass_lower_omp (ctxt);
11946 /* The following is a utility to diagnose structured block violations.
11947 It is not part of the "omplower" pass, as that's invoked too late. It
11948 should be invoked by the respective front ends after gimplification. */
11950 static splay_tree all_labels;
11952 /* Check for mismatched contexts and generate an error if needed. Return
11953 true if an error is detected. */
11955 static bool
11956 diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
11957 gimple branch_ctx, gimple label_ctx)
11959 gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx));
11960 gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx));
11962 if (label_ctx == branch_ctx)
11963 return false;
11965 const char* kind = NULL;
11967 if (flag_cilkplus)
11969 if ((branch_ctx
11970 && gimple_code (branch_ctx) == GIMPLE_OMP_FOR
11971 && gimple_omp_for_kind (branch_ctx) == GF_OMP_FOR_KIND_CILKSIMD)
11972 || (label_ctx
11973 && gimple_code (label_ctx) == GIMPLE_OMP_FOR
11974 && gimple_omp_for_kind (label_ctx) == GF_OMP_FOR_KIND_CILKSIMD))
11975 kind = "Cilk Plus";
11977 if (flag_openacc)
11979 if ((branch_ctx && is_gimple_omp_oacc (branch_ctx))
11980 || (label_ctx && is_gimple_omp_oacc (label_ctx)))
11982 gcc_checking_assert (kind == NULL);
11983 kind = "OpenACC";
11986 if (kind == NULL)
11988 gcc_checking_assert (flag_openmp);
11989 kind = "OpenMP";
11993 Previously we kept track of the label's entire context in diagnose_sb_[12]
11994 so we could traverse it and issue a correct "exit" or "enter" error
11995 message upon a structured block violation.
11997 We built the context by building a list with tree_cons'ing, but there is
11998 no easy counterpart in gimple tuples. It seems like far too much work
11999 for issuing exit/enter error messages. If someone really misses the
12000 distinct error message... patches welcome.
12003 #if 0
12004 /* Try to avoid confusing the user by producing and error message
12005 with correct "exit" or "enter" verbiage. We prefer "exit"
12006 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12007 if (branch_ctx == NULL)
12008 exit_p = false;
12009 else
12011 while (label_ctx)
12013 if (TREE_VALUE (label_ctx) == branch_ctx)
12015 exit_p = false;
12016 break;
12018 label_ctx = TREE_CHAIN (label_ctx);
12022 if (exit_p)
12023 error ("invalid exit from %s structured block", kind);
12024 else
12025 error ("invalid entry to %s structured block", kind);
12026 #endif
12028 /* If it's obvious we have an invalid entry, be specific about the error. */
12029 if (branch_ctx == NULL)
12030 error ("invalid entry to %s structured block", kind);
12031 else
12033 /* Otherwise, be vague and lazy, but efficient. */
12034 error ("invalid branch to/from %s structured block", kind);
12037 gsi_replace (gsi_p, gimple_build_nop (), false);
12038 return true;
12041 /* Pass 1: Create a minimal tree of structured blocks, and record
12042 where each label is found. */
12044 static tree
12045 diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
12046 struct walk_stmt_info *wi)
12048 gimple context = (gimple) wi->info;
12049 gimple inner_context;
12050 gimple stmt = gsi_stmt (*gsi_p);
12052 *handled_ops_p = true;
12054 switch (gimple_code (stmt))
12056 WALK_SUBSTMTS;
12058 case GIMPLE_OMP_PARALLEL:
12059 case GIMPLE_OMP_TASK:
12060 case GIMPLE_OMP_SECTIONS:
12061 case GIMPLE_OMP_SINGLE:
12062 case GIMPLE_OMP_SECTION:
12063 case GIMPLE_OMP_MASTER:
12064 case GIMPLE_OMP_ORDERED:
12065 case GIMPLE_OMP_CRITICAL:
12066 case GIMPLE_OMP_TARGET:
12067 case GIMPLE_OMP_TEAMS:
12068 case GIMPLE_OMP_TASKGROUP:
12069 /* The minimal context here is just the current OMP construct. */
12070 inner_context = stmt;
12071 wi->info = inner_context;
12072 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
12073 wi->info = context;
12074 break;
12076 case GIMPLE_OMP_FOR:
12077 inner_context = stmt;
12078 wi->info = inner_context;
12079 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12080 walk them. */
12081 walk_gimple_seq (gimple_omp_for_pre_body (stmt),
12082 diagnose_sb_1, NULL, wi);
12083 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
12084 wi->info = context;
12085 break;
12087 case GIMPLE_LABEL:
12088 splay_tree_insert (all_labels,
12089 (splay_tree_key) gimple_label_label (
12090 as_a <glabel *> (stmt)),
12091 (splay_tree_value) context);
12092 break;
12094 default:
12095 break;
12098 return NULL_TREE;
12101 /* Pass 2: Check each branch and see if its context differs from that of
12102 the destination label's context. */
12104 static tree
12105 diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
12106 struct walk_stmt_info *wi)
12108 gimple context = (gimple) wi->info;
12109 splay_tree_node n;
12110 gimple stmt = gsi_stmt (*gsi_p);
12112 *handled_ops_p = true;
12114 switch (gimple_code (stmt))
12116 WALK_SUBSTMTS;
12118 case GIMPLE_OMP_PARALLEL:
12119 case GIMPLE_OMP_TASK:
12120 case GIMPLE_OMP_SECTIONS:
12121 case GIMPLE_OMP_SINGLE:
12122 case GIMPLE_OMP_SECTION:
12123 case GIMPLE_OMP_MASTER:
12124 case GIMPLE_OMP_ORDERED:
12125 case GIMPLE_OMP_CRITICAL:
12126 case GIMPLE_OMP_TARGET:
12127 case GIMPLE_OMP_TEAMS:
12128 case GIMPLE_OMP_TASKGROUP:
12129 wi->info = stmt;
12130 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
12131 wi->info = context;
12132 break;
12134 case GIMPLE_OMP_FOR:
12135 wi->info = stmt;
12136 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12137 walk them. */
12138 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
12139 diagnose_sb_2, NULL, wi);
12140 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
12141 wi->info = context;
12142 break;
12144 case GIMPLE_COND:
12146 gcond *cond_stmt = as_a <gcond *> (stmt);
12147 tree lab = gimple_cond_true_label (cond_stmt);
12148 if (lab)
12150 n = splay_tree_lookup (all_labels,
12151 (splay_tree_key) lab);
12152 diagnose_sb_0 (gsi_p, context,
12153 n ? (gimple) n->value : NULL);
12155 lab = gimple_cond_false_label (cond_stmt);
12156 if (lab)
12158 n = splay_tree_lookup (all_labels,
12159 (splay_tree_key) lab);
12160 diagnose_sb_0 (gsi_p, context,
12161 n ? (gimple) n->value : NULL);
12164 break;
12166 case GIMPLE_GOTO:
12168 tree lab = gimple_goto_dest (stmt);
12169 if (TREE_CODE (lab) != LABEL_DECL)
12170 break;
12172 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
12173 diagnose_sb_0 (gsi_p, context, n ? (gimple) n->value : NULL);
12175 break;
12177 case GIMPLE_SWITCH:
12179 gswitch *switch_stmt = as_a <gswitch *> (stmt);
12180 unsigned int i;
12181 for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i)
12183 tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i));
12184 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
12185 if (n && diagnose_sb_0 (gsi_p, context, (gimple) n->value))
12186 break;
12189 break;
12191 case GIMPLE_RETURN:
12192 diagnose_sb_0 (gsi_p, context, NULL);
12193 break;
12195 default:
12196 break;
12199 return NULL_TREE;
12202 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12203 GIMPLE_* codes. */
12204 bool
12205 make_gimple_omp_edges (basic_block bb, struct omp_region **region,
12206 int *region_idx)
12208 gimple last = last_stmt (bb);
12209 enum gimple_code code = gimple_code (last);
12210 struct omp_region *cur_region = *region;
12211 bool fallthru = false;
12213 switch (code)
12215 case GIMPLE_OMP_PARALLEL:
12216 case GIMPLE_OMP_TASK:
12217 case GIMPLE_OMP_FOR:
12218 case GIMPLE_OMP_SINGLE:
12219 case GIMPLE_OMP_TEAMS:
12220 case GIMPLE_OMP_MASTER:
12221 case GIMPLE_OMP_TASKGROUP:
12222 case GIMPLE_OMP_ORDERED:
12223 case GIMPLE_OMP_CRITICAL:
12224 case GIMPLE_OMP_SECTION:
12225 cur_region = new_omp_region (bb, code, cur_region);
12226 fallthru = true;
12227 break;
12229 case GIMPLE_OMP_TARGET:
12230 cur_region = new_omp_region (bb, code, cur_region);
12231 fallthru = true;
12232 switch (gimple_omp_target_kind (last))
12234 case GF_OMP_TARGET_KIND_REGION:
12235 case GF_OMP_TARGET_KIND_DATA:
12236 case GF_OMP_TARGET_KIND_OACC_PARALLEL:
12237 case GF_OMP_TARGET_KIND_OACC_KERNELS:
12238 case GF_OMP_TARGET_KIND_OACC_DATA:
12239 break;
12240 case GF_OMP_TARGET_KIND_UPDATE:
12241 case GF_OMP_TARGET_KIND_OACC_UPDATE:
12242 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
12243 cur_region = cur_region->outer;
12244 break;
12245 default:
12246 gcc_unreachable ();
12248 break;
12250 case GIMPLE_OMP_SECTIONS:
12251 cur_region = new_omp_region (bb, code, cur_region);
12252 fallthru = true;
12253 break;
12255 case GIMPLE_OMP_SECTIONS_SWITCH:
12256 fallthru = false;
12257 break;
12259 case GIMPLE_OMP_ATOMIC_LOAD:
12260 case GIMPLE_OMP_ATOMIC_STORE:
12261 fallthru = true;
12262 break;
12264 case GIMPLE_OMP_RETURN:
12265 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12266 somewhere other than the next block. This will be
12267 created later. */
12268 cur_region->exit = bb;
12269 fallthru = cur_region->type != GIMPLE_OMP_SECTION;
12270 cur_region = cur_region->outer;
12271 break;
12273 case GIMPLE_OMP_CONTINUE:
12274 cur_region->cont = bb;
12275 switch (cur_region->type)
12277 case GIMPLE_OMP_FOR:
12278 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12279 succs edges as abnormal to prevent splitting
12280 them. */
12281 single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL;
12282 /* Make the loopback edge. */
12283 make_edge (bb, single_succ (cur_region->entry),
12284 EDGE_ABNORMAL);
12286 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12287 corresponds to the case that the body of the loop
12288 is not executed at all. */
12289 make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL);
12290 make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL);
12291 fallthru = false;
12292 break;
12294 case GIMPLE_OMP_SECTIONS:
12295 /* Wire up the edges into and out of the nested sections. */
12297 basic_block switch_bb = single_succ (cur_region->entry);
12299 struct omp_region *i;
12300 for (i = cur_region->inner; i ; i = i->next)
12302 gcc_assert (i->type == GIMPLE_OMP_SECTION);
12303 make_edge (switch_bb, i->entry, 0);
12304 make_edge (i->exit, bb, EDGE_FALLTHRU);
12307 /* Make the loopback edge to the block with
12308 GIMPLE_OMP_SECTIONS_SWITCH. */
12309 make_edge (bb, switch_bb, 0);
12311 /* Make the edge from the switch to exit. */
12312 make_edge (switch_bb, bb->next_bb, 0);
12313 fallthru = false;
12315 break;
12317 default:
12318 gcc_unreachable ();
12320 break;
12322 default:
12323 gcc_unreachable ();
12326 if (*region != cur_region)
12328 *region = cur_region;
12329 if (cur_region)
12330 *region_idx = cur_region->entry->index;
12331 else
12332 *region_idx = 0;
12335 return fallthru;
12338 static unsigned int
12339 diagnose_omp_structured_block_errors (void)
12341 struct walk_stmt_info wi;
12342 gimple_seq body = gimple_body (current_function_decl);
12344 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
12346 memset (&wi, 0, sizeof (wi));
12347 walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
12349 memset (&wi, 0, sizeof (wi));
12350 wi.want_locations = true;
12351 walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
12353 gimple_set_body (current_function_decl, body);
12355 splay_tree_delete (all_labels);
12356 all_labels = NULL;
12358 return 0;
12361 namespace {
12363 const pass_data pass_data_diagnose_omp_blocks =
12365 GIMPLE_PASS, /* type */
12366 "*diagnose_omp_blocks", /* name */
12367 OPTGROUP_NONE, /* optinfo_flags */
12368 TV_NONE, /* tv_id */
12369 PROP_gimple_any, /* properties_required */
12370 0, /* properties_provided */
12371 0, /* properties_destroyed */
12372 0, /* todo_flags_start */
12373 0, /* todo_flags_finish */
12376 class pass_diagnose_omp_blocks : public gimple_opt_pass
12378 public:
12379 pass_diagnose_omp_blocks (gcc::context *ctxt)
12380 : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
12383 /* opt_pass methods: */
12384 virtual bool gate (function *)
12386 return flag_cilkplus || flag_openacc || flag_openmp;
12388 virtual unsigned int execute (function *)
12390 return diagnose_omp_structured_block_errors ();
12393 }; // class pass_diagnose_omp_blocks
12395 } // anon namespace
12397 gimple_opt_pass *
12398 make_pass_diagnose_omp_blocks (gcc::context *ctxt)
12400 return new pass_diagnose_omp_blocks (ctxt);
12403 /* SIMD clone supporting code. */
12405 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12406 of arguments to reserve space for. */
12408 static struct cgraph_simd_clone *
12409 simd_clone_struct_alloc (int nargs)
12411 struct cgraph_simd_clone *clone_info;
12412 size_t len = (sizeof (struct cgraph_simd_clone)
12413 + nargs * sizeof (struct cgraph_simd_clone_arg));
12414 clone_info = (struct cgraph_simd_clone *)
12415 ggc_internal_cleared_alloc (len);
12416 return clone_info;
12419 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12421 static inline void
12422 simd_clone_struct_copy (struct cgraph_simd_clone *to,
12423 struct cgraph_simd_clone *from)
12425 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
12426 + ((from->nargs - from->inbranch)
12427 * sizeof (struct cgraph_simd_clone_arg))));
12430 /* Return vector of parameter types of function FNDECL. This uses
12431 TYPE_ARG_TYPES if available, otherwise falls back to types of
12432 DECL_ARGUMENTS types. */
12434 vec<tree>
12435 simd_clone_vector_of_formal_parm_types (tree fndecl)
12437 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
12438 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
12439 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
12440 unsigned int i;
12441 tree arg;
12442 FOR_EACH_VEC_ELT (args, i, arg)
12443 args[i] = TREE_TYPE (args[i]);
12444 return args;
12447 /* Given a simd function in NODE, extract the simd specific
12448 information from the OMP clauses passed in CLAUSES, and return
12449 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12450 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12451 otherwise set to FALSE. */
12453 static struct cgraph_simd_clone *
12454 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
12455 bool *inbranch_specified)
12457 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
12458 tree t;
12459 int n;
12460 *inbranch_specified = false;
12462 n = args.length ();
12463 if (n > 0 && args.last () == void_type_node)
12464 n--;
12466 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12467 be cloned have a distinctive artificial label in addition to "omp
12468 declare simd". */
12469 bool cilk_clone
12470 = (flag_cilkplus
12471 && lookup_attribute ("cilk simd function",
12472 DECL_ATTRIBUTES (node->decl)));
12474 /* Allocate one more than needed just in case this is an in-branch
12475 clone which will require a mask argument. */
12476 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
12477 clone_info->nargs = n;
12478 clone_info->cilk_elemental = cilk_clone;
12480 if (!clauses)
12482 args.release ();
12483 return clone_info;
12485 clauses = TREE_VALUE (clauses);
12486 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
12487 return clone_info;
12489 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
12491 switch (OMP_CLAUSE_CODE (t))
12493 case OMP_CLAUSE_INBRANCH:
12494 clone_info->inbranch = 1;
12495 *inbranch_specified = true;
12496 break;
12497 case OMP_CLAUSE_NOTINBRANCH:
12498 clone_info->inbranch = 0;
12499 *inbranch_specified = true;
12500 break;
12501 case OMP_CLAUSE_SIMDLEN:
12502 clone_info->simdlen
12503 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
12504 break;
12505 case OMP_CLAUSE_LINEAR:
12507 tree decl = OMP_CLAUSE_DECL (t);
12508 tree step = OMP_CLAUSE_LINEAR_STEP (t);
12509 int argno = TREE_INT_CST_LOW (decl);
12510 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
12512 clone_info->args[argno].arg_type
12513 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
12514 clone_info->args[argno].linear_step = tree_to_shwi (step);
12515 gcc_assert (clone_info->args[argno].linear_step >= 0
12516 && clone_info->args[argno].linear_step < n);
12518 else
12520 if (POINTER_TYPE_P (args[argno]))
12521 step = fold_convert (ssizetype, step);
12522 if (!tree_fits_shwi_p (step))
12524 warning_at (OMP_CLAUSE_LOCATION (t), 0,
12525 "ignoring large linear step");
12526 args.release ();
12527 return NULL;
12529 else if (integer_zerop (step))
12531 warning_at (OMP_CLAUSE_LOCATION (t), 0,
12532 "ignoring zero linear step");
12533 args.release ();
12534 return NULL;
12536 else
12538 clone_info->args[argno].arg_type
12539 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
12540 clone_info->args[argno].linear_step = tree_to_shwi (step);
12543 break;
12545 case OMP_CLAUSE_UNIFORM:
12547 tree decl = OMP_CLAUSE_DECL (t);
12548 int argno = tree_to_uhwi (decl);
12549 clone_info->args[argno].arg_type
12550 = SIMD_CLONE_ARG_TYPE_UNIFORM;
12551 break;
12553 case OMP_CLAUSE_ALIGNED:
12555 tree decl = OMP_CLAUSE_DECL (t);
12556 int argno = tree_to_uhwi (decl);
12557 clone_info->args[argno].alignment
12558 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
12559 break;
12561 default:
12562 break;
12565 args.release ();
12566 return clone_info;
12569 /* Given a SIMD clone in NODE, calculate the characteristic data
12570 type and return the coresponding type. The characteristic data
12571 type is computed as described in the Intel Vector ABI. */
12573 static tree
12574 simd_clone_compute_base_data_type (struct cgraph_node *node,
12575 struct cgraph_simd_clone *clone_info)
12577 tree type = integer_type_node;
12578 tree fndecl = node->decl;
12580 /* a) For non-void function, the characteristic data type is the
12581 return type. */
12582 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
12583 type = TREE_TYPE (TREE_TYPE (fndecl));
12585 /* b) If the function has any non-uniform, non-linear parameters,
12586 then the characteristic data type is the type of the first
12587 such parameter. */
12588 else
12590 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
12591 for (unsigned int i = 0; i < clone_info->nargs; ++i)
12592 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
12594 type = map[i];
12595 break;
12597 map.release ();
12600 /* c) If the characteristic data type determined by a) or b) above
12601 is struct, union, or class type which is pass-by-value (except
12602 for the type that maps to the built-in complex data type), the
12603 characteristic data type is int. */
12604 if (RECORD_OR_UNION_TYPE_P (type)
12605 && !aggregate_value_p (type, NULL)
12606 && TREE_CODE (type) != COMPLEX_TYPE)
12607 return integer_type_node;
12609 /* d) If none of the above three classes is applicable, the
12610 characteristic data type is int. */
12612 return type;
12614 /* e) For Intel Xeon Phi native and offload compilation, if the
12615 resulting characteristic data type is 8-bit or 16-bit integer
12616 data type, the characteristic data type is int. */
12617 /* Well, we don't handle Xeon Phi yet. */
12620 static tree
12621 simd_clone_mangle (struct cgraph_node *node,
12622 struct cgraph_simd_clone *clone_info)
12624 char vecsize_mangle = clone_info->vecsize_mangle;
12625 char mask = clone_info->inbranch ? 'M' : 'N';
12626 unsigned int simdlen = clone_info->simdlen;
12627 unsigned int n;
12628 pretty_printer pp;
12630 gcc_assert (vecsize_mangle && simdlen);
12632 pp_string (&pp, "_ZGV");
12633 pp_character (&pp, vecsize_mangle);
12634 pp_character (&pp, mask);
12635 pp_decimal_int (&pp, simdlen);
12637 for (n = 0; n < clone_info->nargs; ++n)
12639 struct cgraph_simd_clone_arg arg = clone_info->args[n];
12641 if (arg.arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM)
12642 pp_character (&pp, 'u');
12643 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
12645 gcc_assert (arg.linear_step != 0);
12646 pp_character (&pp, 'l');
12647 if (arg.linear_step > 1)
12648 pp_unsigned_wide_integer (&pp, arg.linear_step);
12649 else if (arg.linear_step < 0)
12651 pp_character (&pp, 'n');
12652 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
12653 arg.linear_step));
12656 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
12658 pp_character (&pp, 's');
12659 pp_unsigned_wide_integer (&pp, arg.linear_step);
12661 else
12662 pp_character (&pp, 'v');
12663 if (arg.alignment)
12665 pp_character (&pp, 'a');
12666 pp_decimal_int (&pp, arg.alignment);
12670 pp_underscore (&pp);
12671 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
12672 if (*str == '*')
12673 ++str;
12674 pp_string (&pp, str);
12675 str = pp_formatted_text (&pp);
12677 /* If there already is a SIMD clone with the same mangled name, don't
12678 add another one. This can happen e.g. for
12679 #pragma omp declare simd
12680 #pragma omp declare simd simdlen(8)
12681 int foo (int, int);
12682 if the simdlen is assumed to be 8 for the first one, etc. */
12683 for (struct cgraph_node *clone = node->simd_clones; clone;
12684 clone = clone->simdclone->next_clone)
12685 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)),
12686 str) == 0)
12687 return NULL_TREE;
12689 return get_identifier (str);
12692 /* Create a simd clone of OLD_NODE and return it. */
12694 static struct cgraph_node *
12695 simd_clone_create (struct cgraph_node *old_node)
12697 struct cgraph_node *new_node;
12698 if (old_node->definition)
12700 if (!old_node->has_gimple_body_p ())
12701 return NULL;
12702 old_node->get_body ();
12703 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
12704 false, NULL, NULL,
12705 "simdclone");
12707 else
12709 tree old_decl = old_node->decl;
12710 tree new_decl = copy_node (old_node->decl);
12711 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
12712 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
12713 SET_DECL_RTL (new_decl, NULL);
12714 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
12715 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
12716 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
12717 symtab->call_cgraph_insertion_hooks (new_node);
12719 if (new_node == NULL)
12720 return new_node;
12722 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
12724 /* The function cgraph_function_versioning () will force the new
12725 symbol local. Undo this, and inherit external visability from
12726 the old node. */
12727 new_node->local.local = old_node->local.local;
12728 new_node->externally_visible = old_node->externally_visible;
12730 return new_node;
12733 /* Adjust the return type of the given function to its appropriate
12734 vector counterpart. Returns a simd array to be used throughout the
12735 function as a return value. */
12737 static tree
12738 simd_clone_adjust_return_type (struct cgraph_node *node)
12740 tree fndecl = node->decl;
12741 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
12742 unsigned int veclen;
12743 tree t;
12745 /* Adjust the function return type. */
12746 if (orig_rettype == void_type_node)
12747 return NULL_TREE;
12748 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
12749 t = TREE_TYPE (TREE_TYPE (fndecl));
12750 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
12751 veclen = node->simdclone->vecsize_int;
12752 else
12753 veclen = node->simdclone->vecsize_float;
12754 veclen /= GET_MODE_BITSIZE (TYPE_MODE (t));
12755 if (veclen > node->simdclone->simdlen)
12756 veclen = node->simdclone->simdlen;
12757 if (POINTER_TYPE_P (t))
12758 t = pointer_sized_int_node;
12759 if (veclen == node->simdclone->simdlen)
12760 t = build_vector_type (t, node->simdclone->simdlen);
12761 else
12763 t = build_vector_type (t, veclen);
12764 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
12766 TREE_TYPE (TREE_TYPE (fndecl)) = t;
12767 if (!node->definition)
12768 return NULL_TREE;
12770 t = DECL_RESULT (fndecl);
12771 /* Adjust the DECL_RESULT. */
12772 gcc_assert (TREE_TYPE (t) != void_type_node);
12773 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
12774 relayout_decl (t);
12776 tree atype = build_array_type_nelts (orig_rettype,
12777 node->simdclone->simdlen);
12778 if (veclen != node->simdclone->simdlen)
12779 return build1 (VIEW_CONVERT_EXPR, atype, t);
12781 /* Set up a SIMD array to use as the return value. */
12782 tree retval = create_tmp_var_raw (atype, "retval");
12783 gimple_add_tmp_var (retval);
12784 return retval;
12787 /* Each vector argument has a corresponding array to be used locally
12788 as part of the eventual loop. Create such temporary array and
12789 return it.
12791 PREFIX is the prefix to be used for the temporary.
12793 TYPE is the inner element type.
12795 SIMDLEN is the number of elements. */
12797 static tree
12798 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
12800 tree atype = build_array_type_nelts (type, simdlen);
12801 tree avar = create_tmp_var_raw (atype, prefix);
12802 gimple_add_tmp_var (avar);
12803 return avar;
12806 /* Modify the function argument types to their corresponding vector
12807 counterparts if appropriate. Also, create one array for each simd
12808 argument to be used locally when using the function arguments as
12809 part of the loop.
12811 NODE is the function whose arguments are to be adjusted.
12813 Returns an adjustment vector that will be filled describing how the
12814 argument types will be adjusted. */
12816 static ipa_parm_adjustment_vec
12817 simd_clone_adjust_argument_types (struct cgraph_node *node)
12819 vec<tree> args;
12820 ipa_parm_adjustment_vec adjustments;
12822 if (node->definition)
12823 args = ipa_get_vector_of_formal_parms (node->decl);
12824 else
12825 args = simd_clone_vector_of_formal_parm_types (node->decl);
12826 adjustments.create (args.length ());
12827 unsigned i, j, veclen;
12828 struct ipa_parm_adjustment adj;
12829 for (i = 0; i < node->simdclone->nargs; ++i)
12831 memset (&adj, 0, sizeof (adj));
12832 tree parm = args[i];
12833 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
12834 adj.base_index = i;
12835 adj.base = parm;
12837 node->simdclone->args[i].orig_arg = node->definition ? parm : NULL_TREE;
12838 node->simdclone->args[i].orig_type = parm_type;
12840 if (node->simdclone->args[i].arg_type != SIMD_CLONE_ARG_TYPE_VECTOR)
12842 /* No adjustment necessary for scalar arguments. */
12843 adj.op = IPA_PARM_OP_COPY;
12845 else
12847 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
12848 veclen = node->simdclone->vecsize_int;
12849 else
12850 veclen = node->simdclone->vecsize_float;
12851 veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type));
12852 if (veclen > node->simdclone->simdlen)
12853 veclen = node->simdclone->simdlen;
12854 adj.arg_prefix = "simd";
12855 if (POINTER_TYPE_P (parm_type))
12856 adj.type = build_vector_type (pointer_sized_int_node, veclen);
12857 else
12858 adj.type = build_vector_type (parm_type, veclen);
12859 node->simdclone->args[i].vector_type = adj.type;
12860 for (j = veclen; j < node->simdclone->simdlen; j += veclen)
12862 adjustments.safe_push (adj);
12863 if (j == veclen)
12865 memset (&adj, 0, sizeof (adj));
12866 adj.op = IPA_PARM_OP_NEW;
12867 adj.arg_prefix = "simd";
12868 adj.base_index = i;
12869 adj.type = node->simdclone->args[i].vector_type;
12873 if (node->definition)
12874 node->simdclone->args[i].simd_array
12875 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
12876 parm_type, node->simdclone->simdlen);
12878 adjustments.safe_push (adj);
12881 if (node->simdclone->inbranch)
12883 tree base_type
12884 = simd_clone_compute_base_data_type (node->simdclone->origin,
12885 node->simdclone);
12887 memset (&adj, 0, sizeof (adj));
12888 adj.op = IPA_PARM_OP_NEW;
12889 adj.arg_prefix = "mask";
12891 adj.base_index = i;
12892 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
12893 veclen = node->simdclone->vecsize_int;
12894 else
12895 veclen = node->simdclone->vecsize_float;
12896 veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
12897 if (veclen > node->simdclone->simdlen)
12898 veclen = node->simdclone->simdlen;
12899 if (POINTER_TYPE_P (base_type))
12900 adj.type = build_vector_type (pointer_sized_int_node, veclen);
12901 else
12902 adj.type = build_vector_type (base_type, veclen);
12903 adjustments.safe_push (adj);
12905 for (j = veclen; j < node->simdclone->simdlen; j += veclen)
12906 adjustments.safe_push (adj);
12908 /* We have previously allocated one extra entry for the mask. Use
12909 it and fill it. */
12910 struct cgraph_simd_clone *sc = node->simdclone;
12911 sc->nargs++;
12912 if (node->definition)
12914 sc->args[i].orig_arg
12915 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
12916 sc->args[i].simd_array
12917 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
12919 sc->args[i].orig_type = base_type;
12920 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
12923 if (node->definition)
12924 ipa_modify_formal_parameters (node->decl, adjustments);
12925 else
12927 tree new_arg_types = NULL_TREE, new_reversed;
12928 bool last_parm_void = false;
12929 if (args.length () > 0 && args.last () == void_type_node)
12930 last_parm_void = true;
12932 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
12933 j = adjustments.length ();
12934 for (i = 0; i < j; i++)
12936 struct ipa_parm_adjustment *adj = &adjustments[i];
12937 tree ptype;
12938 if (adj->op == IPA_PARM_OP_COPY)
12939 ptype = args[adj->base_index];
12940 else
12941 ptype = adj->type;
12942 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
12944 new_reversed = nreverse (new_arg_types);
12945 if (last_parm_void)
12947 if (new_reversed)
12948 TREE_CHAIN (new_arg_types) = void_list_node;
12949 else
12950 new_reversed = void_list_node;
12953 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
12954 TYPE_ARG_TYPES (new_type) = new_reversed;
12955 TREE_TYPE (node->decl) = new_type;
12957 adjustments.release ();
12959 args.release ();
12960 return adjustments;
12963 /* Initialize and copy the function arguments in NODE to their
12964 corresponding local simd arrays. Returns a fresh gimple_seq with
12965 the instruction sequence generated. */
12967 static gimple_seq
12968 simd_clone_init_simd_arrays (struct cgraph_node *node,
12969 ipa_parm_adjustment_vec adjustments)
12971 gimple_seq seq = NULL;
12972 unsigned i = 0, j = 0, k;
12974 for (tree arg = DECL_ARGUMENTS (node->decl);
12975 arg;
12976 arg = DECL_CHAIN (arg), i++, j++)
12978 if (adjustments[j].op == IPA_PARM_OP_COPY)
12979 continue;
12981 node->simdclone->args[i].vector_arg = arg;
12983 tree array = node->simdclone->args[i].simd_array;
12984 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
12986 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
12987 tree ptr = build_fold_addr_expr (array);
12988 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
12989 build_int_cst (ptype, 0));
12990 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
12991 gimplify_and_add (t, &seq);
12993 else
12995 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
12996 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
12997 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
12999 tree ptr = build_fold_addr_expr (array);
13000 int elemsize;
13001 if (k)
13003 arg = DECL_CHAIN (arg);
13004 j++;
13006 elemsize
13007 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg))));
13008 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
13009 build_int_cst (ptype, k * elemsize));
13010 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
13011 gimplify_and_add (t, &seq);
13015 return seq;
13018 /* Callback info for ipa_simd_modify_stmt_ops below. */
13020 struct modify_stmt_info {
13021 ipa_parm_adjustment_vec adjustments;
13022 gimple stmt;
13023 /* True if the parent statement was modified by
13024 ipa_simd_modify_stmt_ops. */
13025 bool modified;
13028 /* Callback for walk_gimple_op.
13030 Adjust operands from a given statement as specified in the
13031 adjustments vector in the callback data. */
13033 static tree
13034 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
13036 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
13037 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
13038 tree *orig_tp = tp;
13039 if (TREE_CODE (*tp) == ADDR_EXPR)
13040 tp = &TREE_OPERAND (*tp, 0);
13041 struct ipa_parm_adjustment *cand = NULL;
13042 if (TREE_CODE (*tp) == PARM_DECL)
13043 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
13044 else
13046 if (TYPE_P (*tp))
13047 *walk_subtrees = 0;
13050 tree repl = NULL_TREE;
13051 if (cand)
13052 repl = unshare_expr (cand->new_decl);
13053 else
13055 if (tp != orig_tp)
13057 *walk_subtrees = 0;
13058 bool modified = info->modified;
13059 info->modified = false;
13060 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
13061 if (!info->modified)
13063 info->modified = modified;
13064 return NULL_TREE;
13066 info->modified = modified;
13067 repl = *tp;
13069 else
13070 return NULL_TREE;
13073 if (tp != orig_tp)
13075 repl = build_fold_addr_expr (repl);
13076 gimple stmt;
13077 if (is_gimple_debug (info->stmt))
13079 tree vexpr = make_node (DEBUG_EXPR_DECL);
13080 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
13081 DECL_ARTIFICIAL (vexpr) = 1;
13082 TREE_TYPE (vexpr) = TREE_TYPE (repl);
13083 DECL_MODE (vexpr) = TYPE_MODE (TREE_TYPE (repl));
13084 repl = vexpr;
13086 else
13088 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
13089 repl = gimple_assign_lhs (stmt);
13091 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
13092 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
13093 *orig_tp = repl;
13095 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
13097 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
13098 *tp = vce;
13100 else
13101 *tp = repl;
13103 info->modified = true;
13104 return NULL_TREE;
13107 /* Traverse the function body and perform all modifications as
13108 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13109 modified such that the replacement/reduction value will now be an
13110 offset into the corresponding simd_array.
13112 This function will replace all function argument uses with their
13113 corresponding simd array elements, and ajust the return values
13114 accordingly. */
13116 static void
13117 ipa_simd_modify_function_body (struct cgraph_node *node,
13118 ipa_parm_adjustment_vec adjustments,
13119 tree retval_array, tree iter)
13121 basic_block bb;
13122 unsigned int i, j, l;
13124 /* Re-use the adjustments array, but this time use it to replace
13125 every function argument use to an offset into the corresponding
13126 simd_array. */
13127 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
13129 if (!node->simdclone->args[i].vector_arg)
13130 continue;
13132 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
13133 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
13134 adjustments[j].new_decl
13135 = build4 (ARRAY_REF,
13136 basetype,
13137 node->simdclone->args[i].simd_array,
13138 iter,
13139 NULL_TREE, NULL_TREE);
13140 if (adjustments[j].op == IPA_PARM_OP_NONE
13141 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
13142 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
13145 l = adjustments.length ();
13146 for (i = 1; i < num_ssa_names; i++)
13148 tree name = ssa_name (i);
13149 if (name
13150 && SSA_NAME_VAR (name)
13151 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
13153 for (j = 0; j < l; j++)
13154 if (SSA_NAME_VAR (name) == adjustments[j].base
13155 && adjustments[j].new_decl)
13157 tree base_var;
13158 if (adjustments[j].new_ssa_base == NULL_TREE)
13160 base_var
13161 = copy_var_decl (adjustments[j].base,
13162 DECL_NAME (adjustments[j].base),
13163 TREE_TYPE (adjustments[j].base));
13164 adjustments[j].new_ssa_base = base_var;
13166 else
13167 base_var = adjustments[j].new_ssa_base;
13168 if (SSA_NAME_IS_DEFAULT_DEF (name))
13170 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
13171 gimple_stmt_iterator gsi = gsi_after_labels (bb);
13172 tree new_decl = unshare_expr (adjustments[j].new_decl);
13173 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
13174 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
13175 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
13176 gimple stmt = gimple_build_assign (name, new_decl);
13177 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
13179 else
13180 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
13185 struct modify_stmt_info info;
13186 info.adjustments = adjustments;
13188 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
13190 gimple_stmt_iterator gsi;
13192 gsi = gsi_start_bb (bb);
13193 while (!gsi_end_p (gsi))
13195 gimple stmt = gsi_stmt (gsi);
13196 info.stmt = stmt;
13197 struct walk_stmt_info wi;
13199 memset (&wi, 0, sizeof (wi));
13200 info.modified = false;
13201 wi.info = &info;
13202 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
13204 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
13206 tree retval = gimple_return_retval (return_stmt);
13207 if (!retval)
13209 gsi_remove (&gsi, true);
13210 continue;
13213 /* Replace `return foo' with `retval_array[iter] = foo'. */
13214 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
13215 retval_array, iter, NULL, NULL);
13216 stmt = gimple_build_assign (ref, retval);
13217 gsi_replace (&gsi, stmt, true);
13218 info.modified = true;
13221 if (info.modified)
13223 update_stmt (stmt);
13224 if (maybe_clean_eh_stmt (stmt))
13225 gimple_purge_dead_eh_edges (gimple_bb (stmt));
13227 gsi_next (&gsi);
13232 /* Adjust the argument types in NODE to their appropriate vector
13233 counterparts. */
13235 static void
13236 simd_clone_adjust (struct cgraph_node *node)
13238 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
13240 targetm.simd_clone.adjust (node);
13242 tree retval = simd_clone_adjust_return_type (node);
13243 ipa_parm_adjustment_vec adjustments
13244 = simd_clone_adjust_argument_types (node);
13246 push_gimplify_context ();
13248 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
13250 /* Adjust all uses of vector arguments accordingly. Adjust all
13251 return values accordingly. */
13252 tree iter = create_tmp_var (unsigned_type_node, "iter");
13253 tree iter1 = make_ssa_name (iter);
13254 tree iter2 = make_ssa_name (iter);
13255 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
13257 /* Initialize the iteration variable. */
13258 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
13259 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
13260 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
13261 /* Insert the SIMD array and iv initialization at function
13262 entry. */
13263 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
13265 pop_gimplify_context (NULL);
13267 /* Create a new BB right before the original exit BB, to hold the
13268 iteration increment and the condition/branch. */
13269 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
13270 basic_block incr_bb = create_empty_bb (orig_exit);
13271 add_bb_to_loop (incr_bb, body_bb->loop_father);
13272 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13273 flag. Set it now to be a FALLTHRU_EDGE. */
13274 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
13275 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
13276 for (unsigned i = 0;
13277 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
13279 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
13280 redirect_edge_succ (e, incr_bb);
13282 edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
13283 e->probability = REG_BR_PROB_BASE;
13284 gsi = gsi_last_bb (incr_bb);
13285 gimple g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
13286 build_int_cst (unsigned_type_node, 1));
13287 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13289 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13290 struct loop *loop = alloc_loop ();
13291 cfun->has_force_vectorize_loops = true;
13292 loop->safelen = node->simdclone->simdlen;
13293 loop->force_vectorize = true;
13294 loop->header = body_bb;
13296 /* Branch around the body if the mask applies. */
13297 if (node->simdclone->inbranch)
13299 gimple_stmt_iterator gsi = gsi_last_bb (loop->header);
13300 tree mask_array
13301 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
13302 tree mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
13303 tree aref = build4 (ARRAY_REF,
13304 TREE_TYPE (TREE_TYPE (mask_array)),
13305 mask_array, iter1,
13306 NULL, NULL);
13307 g = gimple_build_assign (mask, aref);
13308 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13309 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref)));
13310 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
13312 aref = build1 (VIEW_CONVERT_EXPR,
13313 build_nonstandard_integer_type (bitsize, 0), mask);
13314 mask = make_ssa_name (TREE_TYPE (aref));
13315 g = gimple_build_assign (mask, aref);
13316 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13319 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
13320 NULL, NULL);
13321 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13322 make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
13323 FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE;
13326 /* Generate the condition. */
13327 g = gimple_build_cond (LT_EXPR,
13328 iter2,
13329 build_int_cst (unsigned_type_node,
13330 node->simdclone->simdlen),
13331 NULL, NULL);
13332 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13333 e = split_block (incr_bb, gsi_stmt (gsi));
13334 basic_block latch_bb = e->dest;
13335 basic_block new_exit_bb;
13336 new_exit_bb = split_block (latch_bb, NULL)->dest;
13337 loop->latch = latch_bb;
13339 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
13341 make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
13342 /* The successor of incr_bb is already pointing to latch_bb; just
13343 change the flags.
13344 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13345 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
13347 gphi *phi = create_phi_node (iter1, body_bb);
13348 edge preheader_edge = find_edge (entry_bb, body_bb);
13349 edge latch_edge = single_succ_edge (latch_bb);
13350 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
13351 UNKNOWN_LOCATION);
13352 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
13354 /* Generate the new return. */
13355 gsi = gsi_last_bb (new_exit_bb);
13356 if (retval
13357 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
13358 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
13359 retval = TREE_OPERAND (retval, 0);
13360 else if (retval)
13362 retval = build1 (VIEW_CONVERT_EXPR,
13363 TREE_TYPE (TREE_TYPE (node->decl)),
13364 retval);
13365 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
13366 false, GSI_CONTINUE_LINKING);
13368 g = gimple_build_return (retval);
13369 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13371 /* Handle aligned clauses by replacing default defs of the aligned
13372 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13373 lhs. Handle linear by adding PHIs. */
13374 for (unsigned i = 0; i < node->simdclone->nargs; i++)
13375 if (node->simdclone->args[i].alignment
13376 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
13377 && (node->simdclone->args[i].alignment
13378 & (node->simdclone->args[i].alignment - 1)) == 0
13379 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
13380 == POINTER_TYPE)
13382 unsigned int alignment = node->simdclone->args[i].alignment;
13383 tree orig_arg = node->simdclone->args[i].orig_arg;
13384 tree def = ssa_default_def (cfun, orig_arg);
13385 if (def && !has_zero_uses (def))
13387 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
13388 gimple_seq seq = NULL;
13389 bool need_cvt = false;
13390 gcall *call
13391 = gimple_build_call (fn, 2, def, size_int (alignment));
13392 g = call;
13393 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
13394 ptr_type_node))
13395 need_cvt = true;
13396 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
13397 gimple_call_set_lhs (g, t);
13398 gimple_seq_add_stmt_without_update (&seq, g);
13399 if (need_cvt)
13401 t = make_ssa_name (orig_arg);
13402 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
13403 gimple_seq_add_stmt_without_update (&seq, g);
13405 gsi_insert_seq_on_edge_immediate
13406 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
13408 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
13409 int freq = compute_call_stmt_bb_frequency (current_function_decl,
13410 entry_bb);
13411 node->create_edge (cgraph_node::get_create (fn),
13412 call, entry_bb->count, freq);
13414 imm_use_iterator iter;
13415 use_operand_p use_p;
13416 gimple use_stmt;
13417 tree repl = gimple_get_lhs (g);
13418 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
13419 if (is_gimple_debug (use_stmt) || use_stmt == call)
13420 continue;
13421 else
13422 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
13423 SET_USE (use_p, repl);
13426 else if (node->simdclone->args[i].arg_type
13427 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
13429 tree orig_arg = node->simdclone->args[i].orig_arg;
13430 tree def = ssa_default_def (cfun, orig_arg);
13431 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
13432 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
13433 if (def && !has_zero_uses (def))
13435 iter1 = make_ssa_name (orig_arg);
13436 iter2 = make_ssa_name (orig_arg);
13437 phi = create_phi_node (iter1, body_bb);
13438 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
13439 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
13440 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
13441 ? PLUS_EXPR : POINTER_PLUS_EXPR;
13442 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
13443 ? TREE_TYPE (orig_arg) : sizetype;
13444 tree addcst
13445 = build_int_cst (addtype, node->simdclone->args[i].linear_step);
13446 g = gimple_build_assign (iter2, code, iter1, addcst);
13447 gsi = gsi_last_bb (incr_bb);
13448 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
13450 imm_use_iterator iter;
13451 use_operand_p use_p;
13452 gimple use_stmt;
13453 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
13454 if (use_stmt == phi)
13455 continue;
13456 else
13457 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
13458 SET_USE (use_p, iter1);
13462 calculate_dominance_info (CDI_DOMINATORS);
13463 add_loop (loop, loop->header->loop_father);
13464 update_ssa (TODO_update_ssa);
13466 pop_cfun ();
13469 /* If the function in NODE is tagged as an elemental SIMD function,
13470 create the appropriate SIMD clones. */
13472 static void
13473 expand_simd_clones (struct cgraph_node *node)
13475 tree attr = lookup_attribute ("omp declare simd",
13476 DECL_ATTRIBUTES (node->decl));
13477 if (attr == NULL_TREE
13478 || node->global.inlined_to
13479 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
13480 return;
13482 /* Ignore
13483 #pragma omp declare simd
13484 extern int foo ();
13485 in C, there we don't know the argument types at all. */
13486 if (!node->definition
13487 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
13488 return;
13492 /* Start with parsing the "omp declare simd" attribute(s). */
13493 bool inbranch_clause_specified;
13494 struct cgraph_simd_clone *clone_info
13495 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
13496 &inbranch_clause_specified);
13497 if (clone_info == NULL)
13498 continue;
13500 int orig_simdlen = clone_info->simdlen;
13501 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
13502 /* The target can return 0 (no simd clones should be created),
13503 1 (just one ISA of simd clones should be created) or higher
13504 count of ISA variants. In that case, clone_info is initialized
13505 for the first ISA variant. */
13506 int count
13507 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
13508 base_type, 0);
13509 if (count == 0)
13510 continue;
13512 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13513 also create one inbranch and one !inbranch clone of it. */
13514 for (int i = 0; i < count * 2; i++)
13516 struct cgraph_simd_clone *clone = clone_info;
13517 if (inbranch_clause_specified && (i & 1) != 0)
13518 continue;
13520 if (i != 0)
13522 clone = simd_clone_struct_alloc (clone_info->nargs
13523 + ((i & 1) != 0));
13524 simd_clone_struct_copy (clone, clone_info);
13525 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13526 and simd_clone_adjust_argument_types did to the first
13527 clone's info. */
13528 clone->nargs -= clone_info->inbranch;
13529 clone->simdlen = orig_simdlen;
13530 /* And call the target hook again to get the right ISA. */
13531 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
13532 base_type,
13533 i / 2);
13534 if ((i & 1) != 0)
13535 clone->inbranch = 1;
13538 /* simd_clone_mangle might fail if such a clone has been created
13539 already. */
13540 tree id = simd_clone_mangle (node, clone);
13541 if (id == NULL_TREE)
13542 continue;
13544 /* Only when we are sure we want to create the clone actually
13545 clone the function (or definitions) or create another
13546 extern FUNCTION_DECL (for prototypes without definitions). */
13547 struct cgraph_node *n = simd_clone_create (node);
13548 if (n == NULL)
13549 continue;
13551 n->simdclone = clone;
13552 clone->origin = node;
13553 clone->next_clone = NULL;
13554 if (node->simd_clones == NULL)
13556 clone->prev_clone = n;
13557 node->simd_clones = n;
13559 else
13561 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
13562 clone->prev_clone->simdclone->next_clone = n;
13563 node->simd_clones->simdclone->prev_clone = n;
13565 symtab->change_decl_assembler_name (n->decl, id);
13566 /* And finally adjust the return type, parameters and for
13567 definitions also function body. */
13568 if (node->definition)
13569 simd_clone_adjust (n);
13570 else
13572 simd_clone_adjust_return_type (n);
13573 simd_clone_adjust_argument_types (n);
13577 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
13580 /* Entry point for IPA simd clone creation pass. */
13582 static unsigned int
13583 ipa_omp_simd_clone (void)
13585 struct cgraph_node *node;
13586 FOR_EACH_FUNCTION (node)
13587 expand_simd_clones (node);
13588 return 0;
13591 namespace {
13593 const pass_data pass_data_omp_simd_clone =
13595 SIMPLE_IPA_PASS, /* type */
13596 "simdclone", /* name */
13597 OPTGROUP_NONE, /* optinfo_flags */
13598 TV_NONE, /* tv_id */
13599 ( PROP_ssa | PROP_cfg ), /* properties_required */
13600 0, /* properties_provided */
13601 0, /* properties_destroyed */
13602 0, /* todo_flags_start */
13603 0, /* todo_flags_finish */
13606 class pass_omp_simd_clone : public simple_ipa_opt_pass
13608 public:
13609 pass_omp_simd_clone(gcc::context *ctxt)
13610 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
13613 /* opt_pass methods: */
13614 virtual bool gate (function *);
13615 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
13618 bool
13619 pass_omp_simd_clone::gate (function *)
13621 return ((flag_openmp || flag_openmp_simd
13622 || flag_cilkplus
13623 || (in_lto_p && !flag_wpa))
13624 && (targetm.simd_clone.compute_vecsize_and_simdlen != NULL));
13627 } // anon namespace
13629 simple_ipa_opt_pass *
13630 make_pass_omp_simd_clone (gcc::context *ctxt)
13632 return new pass_omp_simd_clone (ctxt);
13635 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
13636 adds their addresses and sizes to constructor-vector V_CTOR. */
13637 static void
13638 add_decls_addresses_to_decl_constructor (vec<tree, va_gc> *v_decls,
13639 vec<constructor_elt, va_gc> *v_ctor)
13641 unsigned len = vec_safe_length (v_decls);
13642 for (unsigned i = 0; i < len; i++)
13644 tree it = (*v_decls)[i];
13645 bool is_function = TREE_CODE (it) != VAR_DECL;
13647 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, build_fold_addr_expr (it));
13648 if (!is_function)
13649 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE,
13650 fold_convert (const_ptr_type_node,
13651 DECL_SIZE_UNIT (it)));
13655 /* Create new symbols containing (address, size) pairs for global variables,
13656 marked with "omp declare target" attribute, as well as addresses for the
13657 functions, which are outlined offloading regions. */
13658 void
13659 omp_finish_file (void)
13661 unsigned num_funcs = vec_safe_length (offload_funcs);
13662 unsigned num_vars = vec_safe_length (offload_vars);
13664 if (num_funcs == 0 && num_vars == 0)
13665 return;
13667 if (targetm_common.have_named_sections)
13669 vec<constructor_elt, va_gc> *v_f, *v_v;
13670 vec_alloc (v_f, num_funcs);
13671 vec_alloc (v_v, num_vars * 2);
13673 add_decls_addresses_to_decl_constructor (offload_funcs, v_f);
13674 add_decls_addresses_to_decl_constructor (offload_vars, v_v);
13676 tree vars_decl_type = build_array_type_nelts (pointer_sized_int_node,
13677 num_vars * 2);
13678 tree funcs_decl_type = build_array_type_nelts (pointer_sized_int_node,
13679 num_funcs);
13680 TYPE_ALIGN (vars_decl_type) = TYPE_ALIGN (pointer_sized_int_node);
13681 TYPE_ALIGN (funcs_decl_type) = TYPE_ALIGN (pointer_sized_int_node);
13682 tree ctor_v = build_constructor (vars_decl_type, v_v);
13683 tree ctor_f = build_constructor (funcs_decl_type, v_f);
13684 TREE_CONSTANT (ctor_v) = TREE_CONSTANT (ctor_f) = 1;
13685 TREE_STATIC (ctor_v) = TREE_STATIC (ctor_f) = 1;
13686 tree funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
13687 get_identifier (".offload_func_table"),
13688 funcs_decl_type);
13689 tree vars_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
13690 get_identifier (".offload_var_table"),
13691 vars_decl_type);
13692 TREE_STATIC (funcs_decl) = TREE_STATIC (vars_decl) = 1;
13693 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
13694 otherwise a joint table in a binary will contain padding between
13695 tables from multiple object files. */
13696 DECL_USER_ALIGN (funcs_decl) = DECL_USER_ALIGN (vars_decl) = 1;
13697 DECL_ALIGN (funcs_decl) = TYPE_ALIGN (funcs_decl_type);
13698 DECL_ALIGN (vars_decl) = TYPE_ALIGN (vars_decl_type);
13699 DECL_INITIAL (funcs_decl) = ctor_f;
13700 DECL_INITIAL (vars_decl) = ctor_v;
13701 set_decl_section_name (funcs_decl, OFFLOAD_FUNC_TABLE_SECTION_NAME);
13702 set_decl_section_name (vars_decl, OFFLOAD_VAR_TABLE_SECTION_NAME);
13704 varpool_node::finalize_decl (vars_decl);
13705 varpool_node::finalize_decl (funcs_decl);
13707 else
13709 for (unsigned i = 0; i < num_funcs; i++)
13711 tree it = (*offload_funcs)[i];
13712 targetm.record_offload_symbol (it);
13714 for (unsigned i = 0; i < num_vars; i++)
13716 tree it = (*offload_vars)[i];
13717 targetm.record_offload_symbol (it);
13722 #include "gt-omp-low.h"