* gcc.target/i386/387-3.c, gcc.target/i386/387-4.c,
[official-gcc.git] / gcc / omp-low.c
blob11bb2d35a151aa0b563254d2148d00fd163f466a
1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
13 version.
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 for more details.
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
24 #include "config.h"
25 #include "system.h"
26 #include "coretypes.h"
27 #include "tm.h"
28 #include "tree.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
31 #include "rtl.h"
32 #include "pointer-set.h"
33 #include "basic-block.h"
34 #include "tree-ssa-alias.h"
35 #include "internal-fn.h"
36 #include "gimple-fold.h"
37 #include "gimple-expr.h"
38 #include "is-a.h"
39 #include "gimple.h"
40 #include "gimplify.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
48 #include "gimple-ssa.h"
49 #include "cgraph.h"
50 #include "tree-cfg.h"
51 #include "tree-phinodes.h"
52 #include "ssa-iterators.h"
53 #include "tree-ssanames.h"
54 #include "tree-into-ssa.h"
55 #include "expr.h"
56 #include "tree-dfa.h"
57 #include "tree-ssa.h"
58 #include "flags.h"
59 #include "function.h"
60 #include "expr.h"
61 #include "tree-pass.h"
62 #include "except.h"
63 #include "splay-tree.h"
64 #include "optabs.h"
65 #include "cfgloop.h"
66 #include "target.h"
67 #include "omp-low.h"
68 #include "gimple-low.h"
69 #include "tree-cfgcleanup.h"
70 #include "pretty-print.h"
71 #include "ipa-prop.h"
72 #include "tree-nested.h"
73 #include "tree-eh.h"
76 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
77 phases. The first phase scans the function looking for OMP statements
78 and then for variables that must be replaced to satisfy data sharing
79 clauses. The second phase expands code for the constructs, as well as
80 re-gimplifying things when variables have been replaced with complex
81 expressions.
83 Final code generation is done by pass_expand_omp. The flowgraph is
84 scanned for parallel regions which are then moved to a new
85 function, to be invoked by the thread library. */
87 /* Parallel region information. Every parallel and workshare
88 directive is enclosed between two markers, the OMP_* directive
89 and a corresponding OMP_RETURN statement. */
91 struct omp_region
93 /* The enclosing region. */
94 struct omp_region *outer;
96 /* First child region. */
97 struct omp_region *inner;
99 /* Next peer region. */
100 struct omp_region *next;
102 /* Block containing the omp directive as its last stmt. */
103 basic_block entry;
105 /* Block containing the OMP_RETURN as its last stmt. */
106 basic_block exit;
108 /* Block containing the OMP_CONTINUE as its last stmt. */
109 basic_block cont;
111 /* If this is a combined parallel+workshare region, this is a list
112 of additional arguments needed by the combined parallel+workshare
113 library call. */
114 vec<tree, va_gc> *ws_args;
116 /* The code for the omp directive of this region. */
117 enum gimple_code type;
119 /* Schedule kind, only used for OMP_FOR type regions. */
120 enum omp_clause_schedule_kind sched_kind;
122 /* True if this is a combined parallel+workshare region. */
123 bool is_combined_parallel;
126 /* Context structure. Used to store information about each parallel
127 directive in the code. */
129 typedef struct omp_context
131 /* This field must be at the beginning, as we do "inheritance": Some
132 callback functions for tree-inline.c (e.g., omp_copy_decl)
133 receive a copy_body_data pointer that is up-casted to an
134 omp_context pointer. */
135 copy_body_data cb;
137 /* The tree of contexts corresponding to the encountered constructs. */
138 struct omp_context *outer;
139 gimple stmt;
141 /* Map variables to fields in a structure that allows communication
142 between sending and receiving threads. */
143 splay_tree field_map;
144 tree record_type;
145 tree sender_decl;
146 tree receiver_decl;
148 /* These are used just by task contexts, if task firstprivate fn is
149 needed. srecord_type is used to communicate from the thread
150 that encountered the task construct to task firstprivate fn,
151 record_type is allocated by GOMP_task, initialized by task firstprivate
152 fn and passed to the task body fn. */
153 splay_tree sfield_map;
154 tree srecord_type;
156 /* A chain of variables to add to the top-level block surrounding the
157 construct. In the case of a parallel, this is in the child function. */
158 tree block_vars;
160 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
161 barriers should jump to during omplower pass. */
162 tree cancel_label;
164 /* What to do with variables with implicitly determined sharing
165 attributes. */
166 enum omp_clause_default_kind default_kind;
168 /* Nesting depth of this context. Used to beautify error messages re
169 invalid gotos. The outermost ctx is depth 1, with depth 0 being
170 reserved for the main body of the function. */
171 int depth;
173 /* True if this parallel directive is nested within another. */
174 bool is_nested;
176 /* True if this construct can be cancelled. */
177 bool cancellable;
178 } omp_context;
181 struct omp_for_data_loop
183 tree v, n1, n2, step;
184 enum tree_code cond_code;
187 /* A structure describing the main elements of a parallel loop. */
189 struct omp_for_data
191 struct omp_for_data_loop loop;
192 tree chunk_size;
193 gimple for_stmt;
194 tree pre, iter_type;
195 int collapse;
196 bool have_nowait, have_ordered;
197 enum omp_clause_schedule_kind sched_kind;
198 struct omp_for_data_loop *loops;
202 static splay_tree all_contexts;
203 static int taskreg_nesting_level;
204 static int target_nesting_level;
205 static struct omp_region *root_omp_region;
206 static bitmap task_shared_vars;
208 static void scan_omp (gimple_seq *, omp_context *);
209 static tree scan_omp_1_op (tree *, int *, void *);
211 #define WALK_SUBSTMTS \
212 case GIMPLE_BIND: \
213 case GIMPLE_TRY: \
214 case GIMPLE_CATCH: \
215 case GIMPLE_EH_FILTER: \
216 case GIMPLE_TRANSACTION: \
217 /* The sub-statements for these should be walked. */ \
218 *handled_ops_p = false; \
219 break;
221 /* Convenience function for calling scan_omp_1_op on tree operands. */
223 static inline tree
224 scan_omp_op (tree *tp, omp_context *ctx)
226 struct walk_stmt_info wi;
228 memset (&wi, 0, sizeof (wi));
229 wi.info = ctx;
230 wi.want_locations = true;
232 return walk_tree (tp, scan_omp_1_op, &wi, NULL);
235 static void lower_omp (gimple_seq *, omp_context *);
236 static tree lookup_decl_in_outer_ctx (tree, omp_context *);
237 static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
239 /* Find an OpenMP clause of type KIND within CLAUSES. */
241 tree
242 find_omp_clause (tree clauses, enum omp_clause_code kind)
244 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
245 if (OMP_CLAUSE_CODE (clauses) == kind)
246 return clauses;
248 return NULL_TREE;
251 /* Return true if CTX is for an omp parallel. */
253 static inline bool
254 is_parallel_ctx (omp_context *ctx)
256 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
260 /* Return true if CTX is for an omp task. */
262 static inline bool
263 is_task_ctx (omp_context *ctx)
265 return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
269 /* Return true if CTX is for an omp parallel or omp task. */
271 static inline bool
272 is_taskreg_ctx (omp_context *ctx)
274 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL
275 || gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
279 /* Return true if REGION is a combined parallel+workshare region. */
281 static inline bool
282 is_combined_parallel (struct omp_region *region)
284 return region->is_combined_parallel;
288 /* Extract the header elements of parallel loop FOR_STMT and store
289 them into *FD. */
291 static void
292 extract_omp_for_data (gimple for_stmt, struct omp_for_data *fd,
293 struct omp_for_data_loop *loops)
295 tree t, var, *collapse_iter, *collapse_count;
296 tree count = NULL_TREE, iter_type = long_integer_type_node;
297 struct omp_for_data_loop *loop;
298 int i;
299 struct omp_for_data_loop dummy_loop;
300 location_t loc = gimple_location (for_stmt);
301 bool simd = gimple_omp_for_kind (for_stmt) & GF_OMP_FOR_KIND_SIMD;
302 bool distribute = gimple_omp_for_kind (for_stmt)
303 == GF_OMP_FOR_KIND_DISTRIBUTE;
305 fd->for_stmt = for_stmt;
306 fd->pre = NULL;
307 fd->collapse = gimple_omp_for_collapse (for_stmt);
308 if (fd->collapse > 1)
309 fd->loops = loops;
310 else
311 fd->loops = &fd->loop;
313 fd->have_nowait = distribute || simd;
314 fd->have_ordered = false;
315 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
316 fd->chunk_size = NULL_TREE;
317 collapse_iter = NULL;
318 collapse_count = NULL;
320 for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t))
321 switch (OMP_CLAUSE_CODE (t))
323 case OMP_CLAUSE_NOWAIT:
324 fd->have_nowait = true;
325 break;
326 case OMP_CLAUSE_ORDERED:
327 fd->have_ordered = true;
328 break;
329 case OMP_CLAUSE_SCHEDULE:
330 gcc_assert (!distribute);
331 fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t);
332 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t);
333 break;
334 case OMP_CLAUSE_DIST_SCHEDULE:
335 gcc_assert (distribute);
336 fd->chunk_size = OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t);
337 break;
338 case OMP_CLAUSE_COLLAPSE:
339 if (fd->collapse > 1)
341 collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t);
342 collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t);
344 default:
345 break;
348 /* FIXME: for now map schedule(auto) to schedule(static).
349 There should be analysis to determine whether all iterations
350 are approximately the same amount of work (then schedule(static)
351 is best) or if it varies (then schedule(dynamic,N) is better). */
352 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO)
354 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
355 gcc_assert (fd->chunk_size == NULL);
357 gcc_assert (fd->collapse == 1 || collapse_iter != NULL);
358 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
359 gcc_assert (fd->chunk_size == NULL);
360 else if (fd->chunk_size == NULL)
362 /* We only need to compute a default chunk size for ordered
363 static loops and dynamic loops. */
364 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
365 || fd->have_ordered)
366 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
367 ? integer_zero_node : integer_one_node;
370 for (i = 0; i < fd->collapse; i++)
372 if (fd->collapse == 1)
373 loop = &fd->loop;
374 else if (loops != NULL)
375 loop = loops + i;
376 else
377 loop = &dummy_loop;
379 loop->v = gimple_omp_for_index (for_stmt, i);
380 gcc_assert (SSA_VAR_P (loop->v));
381 gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
382 || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE);
383 var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v;
384 loop->n1 = gimple_omp_for_initial (for_stmt, i);
386 loop->cond_code = gimple_omp_for_cond (for_stmt, i);
387 loop->n2 = gimple_omp_for_final (for_stmt, i);
388 switch (loop->cond_code)
390 case LT_EXPR:
391 case GT_EXPR:
392 break;
393 case NE_EXPR:
394 gcc_assert (gimple_omp_for_kind (for_stmt)
395 == GF_OMP_FOR_KIND_CILKSIMD);
396 break;
397 case LE_EXPR:
398 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
399 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, 1);
400 else
401 loop->n2 = fold_build2_loc (loc,
402 PLUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
403 build_int_cst (TREE_TYPE (loop->n2), 1));
404 loop->cond_code = LT_EXPR;
405 break;
406 case GE_EXPR:
407 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
408 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, -1);
409 else
410 loop->n2 = fold_build2_loc (loc,
411 MINUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
412 build_int_cst (TREE_TYPE (loop->n2), 1));
413 loop->cond_code = GT_EXPR;
414 break;
415 default:
416 gcc_unreachable ();
419 t = gimple_omp_for_incr (for_stmt, i);
420 gcc_assert (TREE_OPERAND (t, 0) == var);
421 switch (TREE_CODE (t))
423 case PLUS_EXPR:
424 loop->step = TREE_OPERAND (t, 1);
425 break;
426 case POINTER_PLUS_EXPR:
427 loop->step = fold_convert (ssizetype, TREE_OPERAND (t, 1));
428 break;
429 case MINUS_EXPR:
430 loop->step = TREE_OPERAND (t, 1);
431 loop->step = fold_build1_loc (loc,
432 NEGATE_EXPR, TREE_TYPE (loop->step),
433 loop->step);
434 break;
435 default:
436 gcc_unreachable ();
439 if (simd
440 || (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
441 && !fd->have_ordered))
443 if (fd->collapse == 1)
444 iter_type = TREE_TYPE (loop->v);
445 else if (i == 0
446 || TYPE_PRECISION (iter_type)
447 < TYPE_PRECISION (TREE_TYPE (loop->v)))
448 iter_type
449 = build_nonstandard_integer_type
450 (TYPE_PRECISION (TREE_TYPE (loop->v)), 1);
452 else if (iter_type != long_long_unsigned_type_node)
454 if (POINTER_TYPE_P (TREE_TYPE (loop->v)))
455 iter_type = long_long_unsigned_type_node;
456 else if (TYPE_UNSIGNED (TREE_TYPE (loop->v))
457 && TYPE_PRECISION (TREE_TYPE (loop->v))
458 >= TYPE_PRECISION (iter_type))
460 tree n;
462 if (loop->cond_code == LT_EXPR)
463 n = fold_build2_loc (loc,
464 PLUS_EXPR, TREE_TYPE (loop->v),
465 loop->n2, loop->step);
466 else
467 n = loop->n1;
468 if (TREE_CODE (n) != INTEGER_CST
469 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n))
470 iter_type = long_long_unsigned_type_node;
472 else if (TYPE_PRECISION (TREE_TYPE (loop->v))
473 > TYPE_PRECISION (iter_type))
475 tree n1, n2;
477 if (loop->cond_code == LT_EXPR)
479 n1 = loop->n1;
480 n2 = fold_build2_loc (loc,
481 PLUS_EXPR, TREE_TYPE (loop->v),
482 loop->n2, loop->step);
484 else
486 n1 = fold_build2_loc (loc,
487 MINUS_EXPR, TREE_TYPE (loop->v),
488 loop->n2, loop->step);
489 n2 = loop->n1;
491 if (TREE_CODE (n1) != INTEGER_CST
492 || TREE_CODE (n2) != INTEGER_CST
493 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1)
494 || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type)))
495 iter_type = long_long_unsigned_type_node;
499 if (collapse_count && *collapse_count == NULL)
501 t = fold_binary (loop->cond_code, boolean_type_node,
502 fold_convert (TREE_TYPE (loop->v), loop->n1),
503 fold_convert (TREE_TYPE (loop->v), loop->n2));
504 if (t && integer_zerop (t))
505 count = build_zero_cst (long_long_unsigned_type_node);
506 else if ((i == 0 || count != NULL_TREE)
507 && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
508 && TREE_CONSTANT (loop->n1)
509 && TREE_CONSTANT (loop->n2)
510 && TREE_CODE (loop->step) == INTEGER_CST)
512 tree itype = TREE_TYPE (loop->v);
514 if (POINTER_TYPE_P (itype))
515 itype = signed_type_for (itype);
516 t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1));
517 t = fold_build2_loc (loc,
518 PLUS_EXPR, itype,
519 fold_convert_loc (loc, itype, loop->step), t);
520 t = fold_build2_loc (loc, PLUS_EXPR, itype, t,
521 fold_convert_loc (loc, itype, loop->n2));
522 t = fold_build2_loc (loc, MINUS_EXPR, itype, t,
523 fold_convert_loc (loc, itype, loop->n1));
524 if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR)
525 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype,
526 fold_build1_loc (loc, NEGATE_EXPR, itype, t),
527 fold_build1_loc (loc, NEGATE_EXPR, itype,
528 fold_convert_loc (loc, itype,
529 loop->step)));
530 else
531 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t,
532 fold_convert_loc (loc, itype, loop->step));
533 t = fold_convert_loc (loc, long_long_unsigned_type_node, t);
534 if (count != NULL_TREE)
535 count = fold_build2_loc (loc,
536 MULT_EXPR, long_long_unsigned_type_node,
537 count, t);
538 else
539 count = t;
540 if (TREE_CODE (count) != INTEGER_CST)
541 count = NULL_TREE;
543 else if (count && !integer_zerop (count))
544 count = NULL_TREE;
548 if (count
549 && !simd
550 && (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
551 || fd->have_ordered))
553 if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node)))
554 iter_type = long_long_unsigned_type_node;
555 else
556 iter_type = long_integer_type_node;
558 else if (collapse_iter && *collapse_iter != NULL)
559 iter_type = TREE_TYPE (*collapse_iter);
560 fd->iter_type = iter_type;
561 if (collapse_iter && *collapse_iter == NULL)
562 *collapse_iter = create_tmp_var (iter_type, ".iter");
563 if (collapse_count && *collapse_count == NULL)
565 if (count)
566 *collapse_count = fold_convert_loc (loc, iter_type, count);
567 else
568 *collapse_count = create_tmp_var (iter_type, ".count");
571 if (fd->collapse > 1)
573 fd->loop.v = *collapse_iter;
574 fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0);
575 fd->loop.n2 = *collapse_count;
576 fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1);
577 fd->loop.cond_code = LT_EXPR;
582 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
583 is the immediate dominator of PAR_ENTRY_BB, return true if there
584 are no data dependencies that would prevent expanding the parallel
585 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
587 When expanding a combined parallel+workshare region, the call to
588 the child function may need additional arguments in the case of
589 GIMPLE_OMP_FOR regions. In some cases, these arguments are
590 computed out of variables passed in from the parent to the child
591 via 'struct .omp_data_s'. For instance:
593 #pragma omp parallel for schedule (guided, i * 4)
594 for (j ...)
596 Is lowered into:
598 # BLOCK 2 (PAR_ENTRY_BB)
599 .omp_data_o.i = i;
600 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
602 # BLOCK 3 (WS_ENTRY_BB)
603 .omp_data_i = &.omp_data_o;
604 D.1667 = .omp_data_i->i;
605 D.1598 = D.1667 * 4;
606 #pragma omp for schedule (guided, D.1598)
608 When we outline the parallel region, the call to the child function
609 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
610 that value is computed *after* the call site. So, in principle we
611 cannot do the transformation.
613 To see whether the code in WS_ENTRY_BB blocks the combined
614 parallel+workshare call, we collect all the variables used in the
615 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
616 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
617 call.
619 FIXME. If we had the SSA form built at this point, we could merely
620 hoist the code in block 3 into block 2 and be done with it. But at
621 this point we don't have dataflow information and though we could
622 hack something up here, it is really not worth the aggravation. */
624 static bool
625 workshare_safe_to_combine_p (basic_block ws_entry_bb)
627 struct omp_for_data fd;
628 gimple ws_stmt = last_stmt (ws_entry_bb);
630 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
631 return true;
633 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
635 extract_omp_for_data (ws_stmt, &fd, NULL);
637 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
638 return false;
639 if (fd.iter_type != long_integer_type_node)
640 return false;
642 /* FIXME. We give up too easily here. If any of these arguments
643 are not constants, they will likely involve variables that have
644 been mapped into fields of .omp_data_s for sharing with the child
645 function. With appropriate data flow, it would be possible to
646 see through this. */
647 if (!is_gimple_min_invariant (fd.loop.n1)
648 || !is_gimple_min_invariant (fd.loop.n2)
649 || !is_gimple_min_invariant (fd.loop.step)
650 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
651 return false;
653 return true;
657 /* Collect additional arguments needed to emit a combined
658 parallel+workshare call. WS_STMT is the workshare directive being
659 expanded. */
661 static vec<tree, va_gc> *
662 get_ws_args_for (gimple par_stmt, gimple ws_stmt)
664 tree t;
665 location_t loc = gimple_location (ws_stmt);
666 vec<tree, va_gc> *ws_args;
668 if (gimple_code (ws_stmt) == GIMPLE_OMP_FOR)
670 struct omp_for_data fd;
671 tree n1, n2;
673 extract_omp_for_data (ws_stmt, &fd, NULL);
674 n1 = fd.loop.n1;
675 n2 = fd.loop.n2;
677 if (gimple_omp_for_combined_into_p (ws_stmt))
679 tree innerc
680 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt),
681 OMP_CLAUSE__LOOPTEMP_);
682 gcc_assert (innerc);
683 n1 = OMP_CLAUSE_DECL (innerc);
684 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
685 OMP_CLAUSE__LOOPTEMP_);
686 gcc_assert (innerc);
687 n2 = OMP_CLAUSE_DECL (innerc);
690 vec_alloc (ws_args, 3 + (fd.chunk_size != 0));
692 t = fold_convert_loc (loc, long_integer_type_node, n1);
693 ws_args->quick_push (t);
695 t = fold_convert_loc (loc, long_integer_type_node, n2);
696 ws_args->quick_push (t);
698 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step);
699 ws_args->quick_push (t);
701 if (fd.chunk_size)
703 t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size);
704 ws_args->quick_push (t);
707 return ws_args;
709 else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
711 /* Number of sections is equal to the number of edges from the
712 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
713 the exit of the sections region. */
714 basic_block bb = single_succ (gimple_bb (ws_stmt));
715 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
716 vec_alloc (ws_args, 1);
717 ws_args->quick_push (t);
718 return ws_args;
721 gcc_unreachable ();
725 /* Discover whether REGION is a combined parallel+workshare region. */
727 static void
728 determine_parallel_type (struct omp_region *region)
730 basic_block par_entry_bb, par_exit_bb;
731 basic_block ws_entry_bb, ws_exit_bb;
733 if (region == NULL || region->inner == NULL
734 || region->exit == NULL || region->inner->exit == NULL
735 || region->inner->cont == NULL)
736 return;
738 /* We only support parallel+for and parallel+sections. */
739 if (region->type != GIMPLE_OMP_PARALLEL
740 || (region->inner->type != GIMPLE_OMP_FOR
741 && region->inner->type != GIMPLE_OMP_SECTIONS))
742 return;
744 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
745 WS_EXIT_BB -> PAR_EXIT_BB. */
746 par_entry_bb = region->entry;
747 par_exit_bb = region->exit;
748 ws_entry_bb = region->inner->entry;
749 ws_exit_bb = region->inner->exit;
751 if (single_succ (par_entry_bb) == ws_entry_bb
752 && single_succ (ws_exit_bb) == par_exit_bb
753 && workshare_safe_to_combine_p (ws_entry_bb)
754 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb))
755 || (last_and_only_stmt (ws_entry_bb)
756 && last_and_only_stmt (par_exit_bb))))
758 gimple par_stmt = last_stmt (par_entry_bb);
759 gimple ws_stmt = last_stmt (ws_entry_bb);
761 if (region->inner->type == GIMPLE_OMP_FOR)
763 /* If this is a combined parallel loop, we need to determine
764 whether or not to use the combined library calls. There
765 are two cases where we do not apply the transformation:
766 static loops and any kind of ordered loop. In the first
767 case, we already open code the loop so there is no need
768 to do anything else. In the latter case, the combined
769 parallel loop call would still need extra synchronization
770 to implement ordered semantics, so there would not be any
771 gain in using the combined call. */
772 tree clauses = gimple_omp_for_clauses (ws_stmt);
773 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE);
774 if (c == NULL
775 || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC
776 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED))
778 region->is_combined_parallel = false;
779 region->inner->is_combined_parallel = false;
780 return;
784 region->is_combined_parallel = true;
785 region->inner->is_combined_parallel = true;
786 region->ws_args = get_ws_args_for (par_stmt, ws_stmt);
791 /* Return true if EXPR is variable sized. */
793 static inline bool
794 is_variable_sized (const_tree expr)
796 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
799 /* Return true if DECL is a reference type. */
801 static inline bool
802 is_reference (tree decl)
804 return lang_hooks.decls.omp_privatize_by_reference (decl);
807 /* Lookup variables in the decl or field splay trees. The "maybe" form
808 allows for the variable form to not have been entered, otherwise we
809 assert that the variable must have been entered. */
811 static inline tree
812 lookup_decl (tree var, omp_context *ctx)
814 tree *n;
815 n = (tree *) pointer_map_contains (ctx->cb.decl_map, var);
816 return *n;
819 static inline tree
820 maybe_lookup_decl (const_tree var, omp_context *ctx)
822 tree *n;
823 n = (tree *) pointer_map_contains (ctx->cb.decl_map, var);
824 return n ? *n : NULL_TREE;
827 static inline tree
828 lookup_field (tree var, omp_context *ctx)
830 splay_tree_node n;
831 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
832 return (tree) n->value;
835 static inline tree
836 lookup_sfield (tree var, omp_context *ctx)
838 splay_tree_node n;
839 n = splay_tree_lookup (ctx->sfield_map
840 ? ctx->sfield_map : ctx->field_map,
841 (splay_tree_key) var);
842 return (tree) n->value;
845 static inline tree
846 maybe_lookup_field (tree var, omp_context *ctx)
848 splay_tree_node n;
849 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
850 return n ? (tree) n->value : NULL_TREE;
853 /* Return true if DECL should be copied by pointer. SHARED_CTX is
854 the parallel context if DECL is to be shared. */
856 static bool
857 use_pointer_for_field (tree decl, omp_context *shared_ctx)
859 if (AGGREGATE_TYPE_P (TREE_TYPE (decl)))
860 return true;
862 /* We can only use copy-in/copy-out semantics for shared variables
863 when we know the value is not accessible from an outer scope. */
864 if (shared_ctx)
866 /* ??? Trivially accessible from anywhere. But why would we even
867 be passing an address in this case? Should we simply assert
868 this to be false, or should we have a cleanup pass that removes
869 these from the list of mappings? */
870 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
871 return true;
873 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
874 without analyzing the expression whether or not its location
875 is accessible to anyone else. In the case of nested parallel
876 regions it certainly may be. */
877 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
878 return true;
880 /* Do not use copy-in/copy-out for variables that have their
881 address taken. */
882 if (TREE_ADDRESSABLE (decl))
883 return true;
885 /* lower_send_shared_vars only uses copy-in, but not copy-out
886 for these. */
887 if (TREE_READONLY (decl)
888 || ((TREE_CODE (decl) == RESULT_DECL
889 || TREE_CODE (decl) == PARM_DECL)
890 && DECL_BY_REFERENCE (decl)))
891 return false;
893 /* Disallow copy-in/out in nested parallel if
894 decl is shared in outer parallel, otherwise
895 each thread could store the shared variable
896 in its own copy-in location, making the
897 variable no longer really shared. */
898 if (shared_ctx->is_nested)
900 omp_context *up;
902 for (up = shared_ctx->outer; up; up = up->outer)
903 if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up))
904 break;
906 if (up)
908 tree c;
910 for (c = gimple_omp_taskreg_clauses (up->stmt);
911 c; c = OMP_CLAUSE_CHAIN (c))
912 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
913 && OMP_CLAUSE_DECL (c) == decl)
914 break;
916 if (c)
917 goto maybe_mark_addressable_and_ret;
921 /* For tasks avoid using copy-in/out. As tasks can be
922 deferred or executed in different thread, when GOMP_task
923 returns, the task hasn't necessarily terminated. */
924 if (is_task_ctx (shared_ctx))
926 tree outer;
927 maybe_mark_addressable_and_ret:
928 outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
929 if (is_gimple_reg (outer))
931 /* Taking address of OUTER in lower_send_shared_vars
932 might need regimplification of everything that uses the
933 variable. */
934 if (!task_shared_vars)
935 task_shared_vars = BITMAP_ALLOC (NULL);
936 bitmap_set_bit (task_shared_vars, DECL_UID (outer));
937 TREE_ADDRESSABLE (outer) = 1;
939 return true;
943 return false;
946 /* Construct a new automatic decl similar to VAR. */
948 static tree
949 omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
951 tree copy = copy_var_decl (var, name, type);
953 DECL_CONTEXT (copy) = current_function_decl;
954 DECL_CHAIN (copy) = ctx->block_vars;
955 ctx->block_vars = copy;
957 return copy;
960 static tree
961 omp_copy_decl_1 (tree var, omp_context *ctx)
963 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
966 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
967 as appropriate. */
968 static tree
969 omp_build_component_ref (tree obj, tree field)
971 tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL);
972 if (TREE_THIS_VOLATILE (field))
973 TREE_THIS_VOLATILE (ret) |= 1;
974 if (TREE_READONLY (field))
975 TREE_READONLY (ret) |= 1;
976 return ret;
979 /* Build tree nodes to access the field for VAR on the receiver side. */
981 static tree
982 build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
984 tree x, field = lookup_field (var, ctx);
986 /* If the receiver record type was remapped in the child function,
987 remap the field into the new record type. */
988 x = maybe_lookup_field (field, ctx);
989 if (x != NULL)
990 field = x;
992 x = build_simple_mem_ref (ctx->receiver_decl);
993 x = omp_build_component_ref (x, field);
994 if (by_ref)
995 x = build_simple_mem_ref (x);
997 return x;
1000 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1001 of a parallel, this is a component reference; for workshare constructs
1002 this is some variable. */
1004 static tree
1005 build_outer_var_ref (tree var, omp_context *ctx)
1007 tree x;
1009 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
1010 x = var;
1011 else if (is_variable_sized (var))
1013 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
1014 x = build_outer_var_ref (x, ctx);
1015 x = build_simple_mem_ref (x);
1017 else if (is_taskreg_ctx (ctx))
1019 bool by_ref = use_pointer_for_field (var, NULL);
1020 x = build_receiver_ref (var, by_ref, ctx);
1022 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1023 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD)
1025 /* #pragma omp simd isn't a worksharing construct, and can reference even
1026 private vars in its linear etc. clauses. */
1027 x = NULL_TREE;
1028 if (ctx->outer && is_taskreg_ctx (ctx))
1029 x = lookup_decl (var, ctx->outer);
1030 else if (ctx->outer)
1031 x = maybe_lookup_decl_in_outer_ctx (var, ctx);
1032 if (x == NULL_TREE)
1033 x = var;
1035 else if (ctx->outer)
1036 x = lookup_decl (var, ctx->outer);
1037 else if (is_reference (var))
1038 /* This can happen with orphaned constructs. If var is reference, it is
1039 possible it is shared and as such valid. */
1040 x = var;
1041 else
1042 gcc_unreachable ();
1044 if (is_reference (var))
1045 x = build_simple_mem_ref (x);
1047 return x;
1050 /* Build tree nodes to access the field for VAR on the sender side. */
1052 static tree
1053 build_sender_ref (tree var, omp_context *ctx)
1055 tree field = lookup_sfield (var, ctx);
1056 return omp_build_component_ref (ctx->sender_decl, field);
1059 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1061 static void
1062 install_var_field (tree var, bool by_ref, int mask, omp_context *ctx)
1064 tree field, type, sfield = NULL_TREE;
1066 gcc_assert ((mask & 1) == 0
1067 || !splay_tree_lookup (ctx->field_map, (splay_tree_key) var));
1068 gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
1069 || !splay_tree_lookup (ctx->sfield_map, (splay_tree_key) var));
1071 type = TREE_TYPE (var);
1072 if (mask & 4)
1074 gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
1075 type = build_pointer_type (build_pointer_type (type));
1077 else if (by_ref)
1078 type = build_pointer_type (type);
1079 else if ((mask & 3) == 1 && is_reference (var))
1080 type = TREE_TYPE (type);
1082 field = build_decl (DECL_SOURCE_LOCATION (var),
1083 FIELD_DECL, DECL_NAME (var), type);
1085 /* Remember what variable this field was created for. This does have a
1086 side effect of making dwarf2out ignore this member, so for helpful
1087 debugging we clear it later in delete_omp_context. */
1088 DECL_ABSTRACT_ORIGIN (field) = var;
1089 if (type == TREE_TYPE (var))
1091 DECL_ALIGN (field) = DECL_ALIGN (var);
1092 DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
1093 TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
1095 else
1096 DECL_ALIGN (field) = TYPE_ALIGN (type);
1098 if ((mask & 3) == 3)
1100 insert_field_into_struct (ctx->record_type, field);
1101 if (ctx->srecord_type)
1103 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1104 FIELD_DECL, DECL_NAME (var), type);
1105 DECL_ABSTRACT_ORIGIN (sfield) = var;
1106 DECL_ALIGN (sfield) = DECL_ALIGN (field);
1107 DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
1108 TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
1109 insert_field_into_struct (ctx->srecord_type, sfield);
1112 else
1114 if (ctx->srecord_type == NULL_TREE)
1116 tree t;
1118 ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
1119 ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1120 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
1122 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1123 FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
1124 DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
1125 insert_field_into_struct (ctx->srecord_type, sfield);
1126 splay_tree_insert (ctx->sfield_map,
1127 (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
1128 (splay_tree_value) sfield);
1131 sfield = field;
1132 insert_field_into_struct ((mask & 1) ? ctx->record_type
1133 : ctx->srecord_type, field);
1136 if (mask & 1)
1137 splay_tree_insert (ctx->field_map, (splay_tree_key) var,
1138 (splay_tree_value) field);
1139 if ((mask & 2) && ctx->sfield_map)
1140 splay_tree_insert (ctx->sfield_map, (splay_tree_key) var,
1141 (splay_tree_value) sfield);
1144 static tree
1145 install_var_local (tree var, omp_context *ctx)
1147 tree new_var = omp_copy_decl_1 (var, ctx);
1148 insert_decl_map (&ctx->cb, var, new_var);
1149 return new_var;
1152 /* Adjust the replacement for DECL in CTX for the new context. This means
1153 copying the DECL_VALUE_EXPR, and fixing up the type. */
1155 static void
1156 fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
1158 tree new_decl, size;
1160 new_decl = lookup_decl (decl, ctx);
1162 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
1164 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
1165 && DECL_HAS_VALUE_EXPR_P (decl))
1167 tree ve = DECL_VALUE_EXPR (decl);
1168 walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
1169 SET_DECL_VALUE_EXPR (new_decl, ve);
1170 DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
1173 if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
1175 size = remap_decl (DECL_SIZE (decl), &ctx->cb);
1176 if (size == error_mark_node)
1177 size = TYPE_SIZE (TREE_TYPE (new_decl));
1178 DECL_SIZE (new_decl) = size;
1180 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
1181 if (size == error_mark_node)
1182 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
1183 DECL_SIZE_UNIT (new_decl) = size;
1187 /* The callback for remap_decl. Search all containing contexts for a
1188 mapping of the variable; this avoids having to duplicate the splay
1189 tree ahead of time. We know a mapping doesn't already exist in the
1190 given context. Create new mappings to implement default semantics. */
1192 static tree
1193 omp_copy_decl (tree var, copy_body_data *cb)
1195 omp_context *ctx = (omp_context *) cb;
1196 tree new_var;
1198 if (TREE_CODE (var) == LABEL_DECL)
1200 new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
1201 DECL_CONTEXT (new_var) = current_function_decl;
1202 insert_decl_map (&ctx->cb, var, new_var);
1203 return new_var;
1206 while (!is_taskreg_ctx (ctx))
1208 ctx = ctx->outer;
1209 if (ctx == NULL)
1210 return var;
1211 new_var = maybe_lookup_decl (var, ctx);
1212 if (new_var)
1213 return new_var;
1216 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
1217 return var;
1219 return error_mark_node;
1223 /* Debugging dumps for parallel regions. */
1224 void dump_omp_region (FILE *, struct omp_region *, int);
1225 void debug_omp_region (struct omp_region *);
1226 void debug_all_omp_regions (void);
1228 /* Dump the parallel region tree rooted at REGION. */
1230 void
1231 dump_omp_region (FILE *file, struct omp_region *region, int indent)
1233 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
1234 gimple_code_name[region->type]);
1236 if (region->inner)
1237 dump_omp_region (file, region->inner, indent + 4);
1239 if (region->cont)
1241 fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "",
1242 region->cont->index);
1245 if (region->exit)
1246 fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "",
1247 region->exit->index);
1248 else
1249 fprintf (file, "%*s[no exit marker]\n", indent, "");
1251 if (region->next)
1252 dump_omp_region (file, region->next, indent);
1255 DEBUG_FUNCTION void
1256 debug_omp_region (struct omp_region *region)
1258 dump_omp_region (stderr, region, 0);
1261 DEBUG_FUNCTION void
1262 debug_all_omp_regions (void)
1264 dump_omp_region (stderr, root_omp_region, 0);
1268 /* Create a new parallel region starting at STMT inside region PARENT. */
1270 static struct omp_region *
1271 new_omp_region (basic_block bb, enum gimple_code type,
1272 struct omp_region *parent)
1274 struct omp_region *region = XCNEW (struct omp_region);
1276 region->outer = parent;
1277 region->entry = bb;
1278 region->type = type;
1280 if (parent)
1282 /* This is a nested region. Add it to the list of inner
1283 regions in PARENT. */
1284 region->next = parent->inner;
1285 parent->inner = region;
1287 else
1289 /* This is a toplevel region. Add it to the list of toplevel
1290 regions in ROOT_OMP_REGION. */
1291 region->next = root_omp_region;
1292 root_omp_region = region;
1295 return region;
1298 /* Release the memory associated with the region tree rooted at REGION. */
1300 static void
1301 free_omp_region_1 (struct omp_region *region)
1303 struct omp_region *i, *n;
1305 for (i = region->inner; i ; i = n)
1307 n = i->next;
1308 free_omp_region_1 (i);
1311 free (region);
1314 /* Release the memory for the entire omp region tree. */
1316 void
1317 free_omp_regions (void)
1319 struct omp_region *r, *n;
1320 for (r = root_omp_region; r ; r = n)
1322 n = r->next;
1323 free_omp_region_1 (r);
1325 root_omp_region = NULL;
1329 /* Create a new context, with OUTER_CTX being the surrounding context. */
1331 static omp_context *
1332 new_omp_context (gimple stmt, omp_context *outer_ctx)
1334 omp_context *ctx = XCNEW (omp_context);
1336 splay_tree_insert (all_contexts, (splay_tree_key) stmt,
1337 (splay_tree_value) ctx);
1338 ctx->stmt = stmt;
1340 if (outer_ctx)
1342 ctx->outer = outer_ctx;
1343 ctx->cb = outer_ctx->cb;
1344 ctx->cb.block = NULL;
1345 ctx->depth = outer_ctx->depth + 1;
1347 else
1349 ctx->cb.src_fn = current_function_decl;
1350 ctx->cb.dst_fn = current_function_decl;
1351 ctx->cb.src_node = cgraph_get_node (current_function_decl);
1352 gcc_checking_assert (ctx->cb.src_node);
1353 ctx->cb.dst_node = ctx->cb.src_node;
1354 ctx->cb.src_cfun = cfun;
1355 ctx->cb.copy_decl = omp_copy_decl;
1356 ctx->cb.eh_lp_nr = 0;
1357 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1358 ctx->depth = 1;
1361 ctx->cb.decl_map = pointer_map_create ();
1363 return ctx;
1366 static gimple_seq maybe_catch_exception (gimple_seq);
1368 /* Finalize task copyfn. */
1370 static void
1371 finalize_task_copyfn (gimple task_stmt)
1373 struct function *child_cfun;
1374 tree child_fn;
1375 gimple_seq seq = NULL, new_seq;
1376 gimple bind;
1378 child_fn = gimple_omp_task_copy_fn (task_stmt);
1379 if (child_fn == NULL_TREE)
1380 return;
1382 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1383 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1385 push_cfun (child_cfun);
1386 bind = gimplify_body (child_fn, false);
1387 gimple_seq_add_stmt (&seq, bind);
1388 new_seq = maybe_catch_exception (seq);
1389 if (new_seq != seq)
1391 bind = gimple_build_bind (NULL, new_seq, NULL);
1392 seq = NULL;
1393 gimple_seq_add_stmt (&seq, bind);
1395 gimple_set_body (child_fn, seq);
1396 pop_cfun ();
1398 /* Inform the callgraph about the new function. */
1399 cgraph_add_new_function (child_fn, false);
1402 /* Destroy a omp_context data structures. Called through the splay tree
1403 value delete callback. */
1405 static void
1406 delete_omp_context (splay_tree_value value)
1408 omp_context *ctx = (omp_context *) value;
1410 pointer_map_destroy (ctx->cb.decl_map);
1412 if (ctx->field_map)
1413 splay_tree_delete (ctx->field_map);
1414 if (ctx->sfield_map)
1415 splay_tree_delete (ctx->sfield_map);
1417 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1418 it produces corrupt debug information. */
1419 if (ctx->record_type)
1421 tree t;
1422 for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1423 DECL_ABSTRACT_ORIGIN (t) = NULL;
1425 if (ctx->srecord_type)
1427 tree t;
1428 for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1429 DECL_ABSTRACT_ORIGIN (t) = NULL;
1432 if (is_task_ctx (ctx))
1433 finalize_task_copyfn (ctx->stmt);
1435 XDELETE (ctx);
1438 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1439 context. */
1441 static void
1442 fixup_child_record_type (omp_context *ctx)
1444 tree f, type = ctx->record_type;
1446 /* ??? It isn't sufficient to just call remap_type here, because
1447 variably_modified_type_p doesn't work the way we expect for
1448 record types. Testing each field for whether it needs remapping
1449 and creating a new record by hand works, however. */
1450 for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1451 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1452 break;
1453 if (f)
1455 tree name, new_fields = NULL;
1457 type = lang_hooks.types.make_type (RECORD_TYPE);
1458 name = DECL_NAME (TYPE_NAME (ctx->record_type));
1459 name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1460 TYPE_DECL, name, type);
1461 TYPE_NAME (type) = name;
1463 for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1465 tree new_f = copy_node (f);
1466 DECL_CONTEXT (new_f) = type;
1467 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1468 DECL_CHAIN (new_f) = new_fields;
1469 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1470 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1471 &ctx->cb, NULL);
1472 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1473 &ctx->cb, NULL);
1474 new_fields = new_f;
1476 /* Arrange to be able to look up the receiver field
1477 given the sender field. */
1478 splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1479 (splay_tree_value) new_f);
1481 TYPE_FIELDS (type) = nreverse (new_fields);
1482 layout_type (type);
1485 TREE_TYPE (ctx->receiver_decl) = build_pointer_type (type);
1488 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1489 specified by CLAUSES. */
1491 static void
1492 scan_sharing_clauses (tree clauses, omp_context *ctx)
1494 tree c, decl;
1495 bool scan_array_reductions = false;
1497 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1499 bool by_ref;
1501 switch (OMP_CLAUSE_CODE (c))
1503 case OMP_CLAUSE_PRIVATE:
1504 decl = OMP_CLAUSE_DECL (c);
1505 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1506 goto do_private;
1507 else if (!is_variable_sized (decl))
1508 install_var_local (decl, ctx);
1509 break;
1511 case OMP_CLAUSE_SHARED:
1512 /* Ignore shared directives in teams construct. */
1513 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1514 break;
1515 gcc_assert (is_taskreg_ctx (ctx));
1516 decl = OMP_CLAUSE_DECL (c);
1517 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1518 || !is_variable_sized (decl));
1519 /* Global variables don't need to be copied,
1520 the receiver side will use them directly. */
1521 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1522 break;
1523 by_ref = use_pointer_for_field (decl, ctx);
1524 if (! TREE_READONLY (decl)
1525 || TREE_ADDRESSABLE (decl)
1526 || by_ref
1527 || is_reference (decl))
1529 install_var_field (decl, by_ref, 3, ctx);
1530 install_var_local (decl, ctx);
1531 break;
1533 /* We don't need to copy const scalar vars back. */
1534 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1535 goto do_private;
1537 case OMP_CLAUSE_LASTPRIVATE:
1538 /* Let the corresponding firstprivate clause create
1539 the variable. */
1540 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1541 break;
1542 /* FALLTHRU */
1544 case OMP_CLAUSE_FIRSTPRIVATE:
1545 case OMP_CLAUSE_REDUCTION:
1546 case OMP_CLAUSE_LINEAR:
1547 decl = OMP_CLAUSE_DECL (c);
1548 do_private:
1549 if (is_variable_sized (decl))
1551 if (is_task_ctx (ctx))
1552 install_var_field (decl, false, 1, ctx);
1553 break;
1555 else if (is_taskreg_ctx (ctx))
1557 bool global
1558 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1559 by_ref = use_pointer_for_field (decl, NULL);
1561 if (is_task_ctx (ctx)
1562 && (global || by_ref || is_reference (decl)))
1564 install_var_field (decl, false, 1, ctx);
1565 if (!global)
1566 install_var_field (decl, by_ref, 2, ctx);
1568 else if (!global)
1569 install_var_field (decl, by_ref, 3, ctx);
1571 install_var_local (decl, ctx);
1572 break;
1574 case OMP_CLAUSE__LOOPTEMP_:
1575 gcc_assert (is_parallel_ctx (ctx));
1576 decl = OMP_CLAUSE_DECL (c);
1577 install_var_field (decl, false, 3, ctx);
1578 install_var_local (decl, ctx);
1579 break;
1581 case OMP_CLAUSE_COPYPRIVATE:
1582 case OMP_CLAUSE_COPYIN:
1583 decl = OMP_CLAUSE_DECL (c);
1584 by_ref = use_pointer_for_field (decl, NULL);
1585 install_var_field (decl, by_ref, 3, ctx);
1586 break;
1588 case OMP_CLAUSE_DEFAULT:
1589 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c);
1590 break;
1592 case OMP_CLAUSE_FINAL:
1593 case OMP_CLAUSE_IF:
1594 case OMP_CLAUSE_NUM_THREADS:
1595 case OMP_CLAUSE_NUM_TEAMS:
1596 case OMP_CLAUSE_THREAD_LIMIT:
1597 case OMP_CLAUSE_DEVICE:
1598 case OMP_CLAUSE_SCHEDULE:
1599 case OMP_CLAUSE_DIST_SCHEDULE:
1600 case OMP_CLAUSE_DEPEND:
1601 if (ctx->outer)
1602 scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1603 break;
1605 case OMP_CLAUSE_TO:
1606 case OMP_CLAUSE_FROM:
1607 case OMP_CLAUSE_MAP:
1608 if (ctx->outer)
1609 scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1610 decl = OMP_CLAUSE_DECL (c);
1611 /* Global variables with "omp declare target" attribute
1612 don't need to be copied, the receiver side will use them
1613 directly. */
1614 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1615 && DECL_P (decl)
1616 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1617 && lookup_attribute ("omp declare target",
1618 DECL_ATTRIBUTES (decl)))
1619 break;
1620 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1621 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER)
1623 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1624 #pragma omp target data, there is nothing to map for
1625 those. */
1626 if (gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_DATA
1627 && !POINTER_TYPE_P (TREE_TYPE (decl)))
1628 break;
1630 if (DECL_P (decl))
1632 if (DECL_SIZE (decl)
1633 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1635 tree decl2 = DECL_VALUE_EXPR (decl);
1636 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
1637 decl2 = TREE_OPERAND (decl2, 0);
1638 gcc_assert (DECL_P (decl2));
1639 install_var_field (decl2, true, 3, ctx);
1640 install_var_local (decl2, ctx);
1641 install_var_local (decl, ctx);
1643 else
1645 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1646 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
1647 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1648 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1649 install_var_field (decl, true, 7, ctx);
1650 else
1651 install_var_field (decl, true, 3, ctx);
1652 if (gimple_omp_target_kind (ctx->stmt)
1653 == GF_OMP_TARGET_KIND_REGION)
1654 install_var_local (decl, ctx);
1657 else
1659 tree base = get_base_address (decl);
1660 tree nc = OMP_CLAUSE_CHAIN (c);
1661 if (DECL_P (base)
1662 && nc != NULL_TREE
1663 && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1664 && OMP_CLAUSE_DECL (nc) == base
1665 && OMP_CLAUSE_MAP_KIND (nc) == OMP_CLAUSE_MAP_POINTER
1666 && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1668 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1669 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1671 else
1673 gcc_assert (!splay_tree_lookup (ctx->field_map,
1674 (splay_tree_key) decl));
1675 tree field
1676 = build_decl (OMP_CLAUSE_LOCATION (c),
1677 FIELD_DECL, NULL_TREE, ptr_type_node);
1678 DECL_ALIGN (field) = TYPE_ALIGN (ptr_type_node);
1679 insert_field_into_struct (ctx->record_type, field);
1680 splay_tree_insert (ctx->field_map, (splay_tree_key) decl,
1681 (splay_tree_value) field);
1684 break;
1686 case OMP_CLAUSE_NOWAIT:
1687 case OMP_CLAUSE_ORDERED:
1688 case OMP_CLAUSE_COLLAPSE:
1689 case OMP_CLAUSE_UNTIED:
1690 case OMP_CLAUSE_MERGEABLE:
1691 case OMP_CLAUSE_PROC_BIND:
1692 case OMP_CLAUSE_SAFELEN:
1693 break;
1695 case OMP_CLAUSE_ALIGNED:
1696 decl = OMP_CLAUSE_DECL (c);
1697 if (is_global_var (decl)
1698 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1699 install_var_local (decl, ctx);
1700 break;
1702 default:
1703 gcc_unreachable ();
1707 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1709 switch (OMP_CLAUSE_CODE (c))
1711 case OMP_CLAUSE_LASTPRIVATE:
1712 /* Let the corresponding firstprivate clause create
1713 the variable. */
1714 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1715 scan_array_reductions = true;
1716 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1717 break;
1718 /* FALLTHRU */
1720 case OMP_CLAUSE_PRIVATE:
1721 case OMP_CLAUSE_FIRSTPRIVATE:
1722 case OMP_CLAUSE_REDUCTION:
1723 case OMP_CLAUSE_LINEAR:
1724 decl = OMP_CLAUSE_DECL (c);
1725 if (is_variable_sized (decl))
1726 install_var_local (decl, ctx);
1727 fixup_remapped_decl (decl, ctx,
1728 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1729 && OMP_CLAUSE_PRIVATE_DEBUG (c));
1730 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1731 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1732 scan_array_reductions = true;
1733 break;
1735 case OMP_CLAUSE_SHARED:
1736 /* Ignore shared directives in teams construct. */
1737 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1738 break;
1739 decl = OMP_CLAUSE_DECL (c);
1740 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1741 fixup_remapped_decl (decl, ctx, false);
1742 break;
1744 case OMP_CLAUSE_MAP:
1745 if (gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_DATA)
1746 break;
1747 decl = OMP_CLAUSE_DECL (c);
1748 if (DECL_P (decl)
1749 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1750 && lookup_attribute ("omp declare target",
1751 DECL_ATTRIBUTES (decl)))
1752 break;
1753 if (DECL_P (decl))
1755 if (OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
1756 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
1757 && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
1759 tree new_decl = lookup_decl (decl, ctx);
1760 TREE_TYPE (new_decl)
1761 = remap_type (TREE_TYPE (decl), &ctx->cb);
1763 else if (DECL_SIZE (decl)
1764 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1766 tree decl2 = DECL_VALUE_EXPR (decl);
1767 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
1768 decl2 = TREE_OPERAND (decl2, 0);
1769 gcc_assert (DECL_P (decl2));
1770 fixup_remapped_decl (decl2, ctx, false);
1771 fixup_remapped_decl (decl, ctx, true);
1773 else
1774 fixup_remapped_decl (decl, ctx, false);
1776 break;
1778 case OMP_CLAUSE_COPYPRIVATE:
1779 case OMP_CLAUSE_COPYIN:
1780 case OMP_CLAUSE_DEFAULT:
1781 case OMP_CLAUSE_IF:
1782 case OMP_CLAUSE_NUM_THREADS:
1783 case OMP_CLAUSE_NUM_TEAMS:
1784 case OMP_CLAUSE_THREAD_LIMIT:
1785 case OMP_CLAUSE_DEVICE:
1786 case OMP_CLAUSE_SCHEDULE:
1787 case OMP_CLAUSE_DIST_SCHEDULE:
1788 case OMP_CLAUSE_NOWAIT:
1789 case OMP_CLAUSE_ORDERED:
1790 case OMP_CLAUSE_COLLAPSE:
1791 case OMP_CLAUSE_UNTIED:
1792 case OMP_CLAUSE_FINAL:
1793 case OMP_CLAUSE_MERGEABLE:
1794 case OMP_CLAUSE_PROC_BIND:
1795 case OMP_CLAUSE_SAFELEN:
1796 case OMP_CLAUSE_ALIGNED:
1797 case OMP_CLAUSE_DEPEND:
1798 case OMP_CLAUSE__LOOPTEMP_:
1799 case OMP_CLAUSE_TO:
1800 case OMP_CLAUSE_FROM:
1801 break;
1803 default:
1804 gcc_unreachable ();
1808 if (scan_array_reductions)
1809 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1810 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1811 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1813 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx);
1814 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
1816 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
1817 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1818 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
1821 /* Create a new name for omp child function. Returns an identifier. */
1823 static tree
1824 create_omp_child_function_name (bool task_copy)
1826 return (clone_function_name (current_function_decl,
1827 task_copy ? "_omp_cpyfn" : "_omp_fn"));
1830 /* Build a decl for the omp child function. It'll not contain a body
1831 yet, just the bare decl. */
1833 static void
1834 create_omp_child_function (omp_context *ctx, bool task_copy)
1836 tree decl, type, name, t;
1838 name = create_omp_child_function_name (task_copy);
1839 if (task_copy)
1840 type = build_function_type_list (void_type_node, ptr_type_node,
1841 ptr_type_node, NULL_TREE);
1842 else
1843 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
1845 decl = build_decl (gimple_location (ctx->stmt),
1846 FUNCTION_DECL, name, type);
1848 if (!task_copy)
1849 ctx->cb.dst_fn = decl;
1850 else
1851 gimple_omp_task_set_copy_fn (ctx->stmt, decl);
1853 TREE_STATIC (decl) = 1;
1854 TREE_USED (decl) = 1;
1855 DECL_ARTIFICIAL (decl) = 1;
1856 DECL_NAMELESS (decl) = 1;
1857 DECL_IGNORED_P (decl) = 0;
1858 TREE_PUBLIC (decl) = 0;
1859 DECL_UNINLINABLE (decl) = 1;
1860 DECL_EXTERNAL (decl) = 0;
1861 DECL_CONTEXT (decl) = NULL_TREE;
1862 DECL_INITIAL (decl) = make_node (BLOCK);
1863 bool target_p = false;
1864 if (lookup_attribute ("omp declare target",
1865 DECL_ATTRIBUTES (current_function_decl)))
1866 target_p = true;
1867 else
1869 omp_context *octx;
1870 for (octx = ctx; octx; octx = octx->outer)
1871 if (gimple_code (octx->stmt) == GIMPLE_OMP_TARGET
1872 && gimple_omp_target_kind (octx->stmt)
1873 == GF_OMP_TARGET_KIND_REGION)
1875 target_p = true;
1876 break;
1879 if (target_p)
1880 DECL_ATTRIBUTES (decl)
1881 = tree_cons (get_identifier ("omp declare target"),
1882 NULL_TREE, DECL_ATTRIBUTES (decl));
1884 t = build_decl (DECL_SOURCE_LOCATION (decl),
1885 RESULT_DECL, NULL_TREE, void_type_node);
1886 DECL_ARTIFICIAL (t) = 1;
1887 DECL_IGNORED_P (t) = 1;
1888 DECL_CONTEXT (t) = decl;
1889 DECL_RESULT (decl) = t;
1891 t = build_decl (DECL_SOURCE_LOCATION (decl),
1892 PARM_DECL, get_identifier (".omp_data_i"), ptr_type_node);
1893 DECL_ARTIFICIAL (t) = 1;
1894 DECL_NAMELESS (t) = 1;
1895 DECL_ARG_TYPE (t) = ptr_type_node;
1896 DECL_CONTEXT (t) = current_function_decl;
1897 TREE_USED (t) = 1;
1898 DECL_ARGUMENTS (decl) = t;
1899 if (!task_copy)
1900 ctx->receiver_decl = t;
1901 else
1903 t = build_decl (DECL_SOURCE_LOCATION (decl),
1904 PARM_DECL, get_identifier (".omp_data_o"),
1905 ptr_type_node);
1906 DECL_ARTIFICIAL (t) = 1;
1907 DECL_NAMELESS (t) = 1;
1908 DECL_ARG_TYPE (t) = ptr_type_node;
1909 DECL_CONTEXT (t) = current_function_decl;
1910 TREE_USED (t) = 1;
1911 TREE_ADDRESSABLE (t) = 1;
1912 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
1913 DECL_ARGUMENTS (decl) = t;
1916 /* Allocate memory for the function structure. The call to
1917 allocate_struct_function clobbers CFUN, so we need to restore
1918 it afterward. */
1919 push_struct_function (decl);
1920 cfun->function_end_locus = gimple_location (ctx->stmt);
1921 pop_cfun ();
1924 /* Callback for walk_gimple_seq. Check if combined parallel
1925 contains gimple_omp_for_combined_into_p OMP_FOR. */
1927 static tree
1928 find_combined_for (gimple_stmt_iterator *gsi_p,
1929 bool *handled_ops_p,
1930 struct walk_stmt_info *wi)
1932 gimple stmt = gsi_stmt (*gsi_p);
1934 *handled_ops_p = true;
1935 switch (gimple_code (stmt))
1937 WALK_SUBSTMTS;
1939 case GIMPLE_OMP_FOR:
1940 if (gimple_omp_for_combined_into_p (stmt)
1941 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
1943 wi->info = stmt;
1944 return integer_zero_node;
1946 break;
1947 default:
1948 break;
1950 return NULL;
1953 /* Scan an OpenMP parallel directive. */
1955 static void
1956 scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
1958 omp_context *ctx;
1959 tree name;
1960 gimple stmt = gsi_stmt (*gsi);
1962 /* Ignore parallel directives with empty bodies, unless there
1963 are copyin clauses. */
1964 if (optimize > 0
1965 && empty_body_p (gimple_omp_body (stmt))
1966 && find_omp_clause (gimple_omp_parallel_clauses (stmt),
1967 OMP_CLAUSE_COPYIN) == NULL)
1969 gsi_replace (gsi, gimple_build_nop (), false);
1970 return;
1973 if (gimple_omp_parallel_combined_p (stmt))
1975 gimple for_stmt;
1976 struct walk_stmt_info wi;
1978 memset (&wi, 0, sizeof (wi));
1979 wi.val_only = true;
1980 walk_gimple_seq (gimple_omp_body (stmt),
1981 find_combined_for, NULL, &wi);
1982 for_stmt = (gimple) wi.info;
1983 if (for_stmt)
1985 struct omp_for_data fd;
1986 extract_omp_for_data (for_stmt, &fd, NULL);
1987 /* We need two temporaries with fd.loop.v type (istart/iend)
1988 and then (fd.collapse - 1) temporaries with the same
1989 type for count2 ... countN-1 vars if not constant. */
1990 size_t count = 2, i;
1991 tree type = fd.iter_type;
1992 if (fd.collapse > 1
1993 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
1994 count += fd.collapse - 1;
1995 for (i = 0; i < count; i++)
1997 tree temp = create_tmp_var (type, NULL);
1998 tree c = build_omp_clause (UNKNOWN_LOCATION,
1999 OMP_CLAUSE__LOOPTEMP_);
2000 OMP_CLAUSE_DECL (c) = temp;
2001 OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
2002 gimple_omp_parallel_set_clauses (stmt, c);
2007 ctx = new_omp_context (stmt, outer_ctx);
2008 if (taskreg_nesting_level > 1)
2009 ctx->is_nested = true;
2010 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2011 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2012 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2013 name = create_tmp_var_name (".omp_data_s");
2014 name = build_decl (gimple_location (stmt),
2015 TYPE_DECL, name, ctx->record_type);
2016 DECL_ARTIFICIAL (name) = 1;
2017 DECL_NAMELESS (name) = 1;
2018 TYPE_NAME (ctx->record_type) = name;
2019 create_omp_child_function (ctx, false);
2020 gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2022 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2023 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2025 if (TYPE_FIELDS (ctx->record_type) == NULL)
2026 ctx->record_type = ctx->receiver_decl = NULL;
2027 else
2029 layout_type (ctx->record_type);
2030 fixup_child_record_type (ctx);
2034 /* Scan an OpenMP task directive. */
2036 static void
2037 scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2039 omp_context *ctx;
2040 tree name, t;
2041 gimple stmt = gsi_stmt (*gsi);
2042 location_t loc = gimple_location (stmt);
2044 /* Ignore task directives with empty bodies. */
2045 if (optimize > 0
2046 && empty_body_p (gimple_omp_body (stmt)))
2048 gsi_replace (gsi, gimple_build_nop (), false);
2049 return;
2052 ctx = new_omp_context (stmt, outer_ctx);
2053 if (taskreg_nesting_level > 1)
2054 ctx->is_nested = true;
2055 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2056 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2057 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2058 name = create_tmp_var_name (".omp_data_s");
2059 name = build_decl (gimple_location (stmt),
2060 TYPE_DECL, name, ctx->record_type);
2061 DECL_ARTIFICIAL (name) = 1;
2062 DECL_NAMELESS (name) = 1;
2063 TYPE_NAME (ctx->record_type) = name;
2064 create_omp_child_function (ctx, false);
2065 gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2067 scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2069 if (ctx->srecord_type)
2071 name = create_tmp_var_name (".omp_data_a");
2072 name = build_decl (gimple_location (stmt),
2073 TYPE_DECL, name, ctx->srecord_type);
2074 DECL_ARTIFICIAL (name) = 1;
2075 DECL_NAMELESS (name) = 1;
2076 TYPE_NAME (ctx->srecord_type) = name;
2077 create_omp_child_function (ctx, true);
2080 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2082 if (TYPE_FIELDS (ctx->record_type) == NULL)
2084 ctx->record_type = ctx->receiver_decl = NULL;
2085 t = build_int_cst (long_integer_type_node, 0);
2086 gimple_omp_task_set_arg_size (stmt, t);
2087 t = build_int_cst (long_integer_type_node, 1);
2088 gimple_omp_task_set_arg_align (stmt, t);
2090 else
2092 tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2093 /* Move VLA fields to the end. */
2094 p = &TYPE_FIELDS (ctx->record_type);
2095 while (*p)
2096 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2097 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2099 *q = *p;
2100 *p = TREE_CHAIN (*p);
2101 TREE_CHAIN (*q) = NULL_TREE;
2102 q = &TREE_CHAIN (*q);
2104 else
2105 p = &DECL_CHAIN (*p);
2106 *p = vla_fields;
2107 layout_type (ctx->record_type);
2108 fixup_child_record_type (ctx);
2109 if (ctx->srecord_type)
2110 layout_type (ctx->srecord_type);
2111 t = fold_convert_loc (loc, long_integer_type_node,
2112 TYPE_SIZE_UNIT (ctx->record_type));
2113 gimple_omp_task_set_arg_size (stmt, t);
2114 t = build_int_cst (long_integer_type_node,
2115 TYPE_ALIGN_UNIT (ctx->record_type));
2116 gimple_omp_task_set_arg_align (stmt, t);
2121 /* Scan an OpenMP loop directive. */
2123 static void
2124 scan_omp_for (gimple stmt, omp_context *outer_ctx)
2126 omp_context *ctx;
2127 size_t i;
2129 ctx = new_omp_context (stmt, outer_ctx);
2131 scan_sharing_clauses (gimple_omp_for_clauses (stmt), ctx);
2133 scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
2134 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
2136 scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
2137 scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
2138 scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
2139 scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
2141 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2144 /* Scan an OpenMP sections directive. */
2146 static void
2147 scan_omp_sections (gimple stmt, omp_context *outer_ctx)
2149 omp_context *ctx;
2151 ctx = new_omp_context (stmt, outer_ctx);
2152 scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
2153 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2156 /* Scan an OpenMP single directive. */
2158 static void
2159 scan_omp_single (gimple stmt, omp_context *outer_ctx)
2161 omp_context *ctx;
2162 tree name;
2164 ctx = new_omp_context (stmt, outer_ctx);
2165 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2166 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2167 name = create_tmp_var_name (".omp_copy_s");
2168 name = build_decl (gimple_location (stmt),
2169 TYPE_DECL, name, ctx->record_type);
2170 TYPE_NAME (ctx->record_type) = name;
2172 scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
2173 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2175 if (TYPE_FIELDS (ctx->record_type) == NULL)
2176 ctx->record_type = NULL;
2177 else
2178 layout_type (ctx->record_type);
2181 /* Scan an OpenMP target{, data, update} directive. */
2183 static void
2184 scan_omp_target (gimple stmt, omp_context *outer_ctx)
2186 omp_context *ctx;
2187 tree name;
2188 int kind = gimple_omp_target_kind (stmt);
2190 ctx = new_omp_context (stmt, outer_ctx);
2191 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2192 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2193 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2194 name = create_tmp_var_name (".omp_data_t");
2195 name = build_decl (gimple_location (stmt),
2196 TYPE_DECL, name, ctx->record_type);
2197 DECL_ARTIFICIAL (name) = 1;
2198 DECL_NAMELESS (name) = 1;
2199 TYPE_NAME (ctx->record_type) = name;
2200 if (kind == GF_OMP_TARGET_KIND_REGION)
2202 create_omp_child_function (ctx, false);
2203 gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
2206 scan_sharing_clauses (gimple_omp_target_clauses (stmt), ctx);
2207 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2209 if (TYPE_FIELDS (ctx->record_type) == NULL)
2210 ctx->record_type = ctx->receiver_decl = NULL;
2211 else
2213 TYPE_FIELDS (ctx->record_type)
2214 = nreverse (TYPE_FIELDS (ctx->record_type));
2215 #ifdef ENABLE_CHECKING
2216 tree field;
2217 unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
2218 for (field = TYPE_FIELDS (ctx->record_type);
2219 field;
2220 field = DECL_CHAIN (field))
2221 gcc_assert (DECL_ALIGN (field) == align);
2222 #endif
2223 layout_type (ctx->record_type);
2224 if (kind == GF_OMP_TARGET_KIND_REGION)
2225 fixup_child_record_type (ctx);
2229 /* Scan an OpenMP teams directive. */
2231 static void
2232 scan_omp_teams (gimple stmt, omp_context *outer_ctx)
2234 omp_context *ctx = new_omp_context (stmt, outer_ctx);
2235 scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
2236 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2239 /* Check OpenMP nesting restrictions. */
2240 static bool
2241 check_omp_nesting_restrictions (gimple stmt, omp_context *ctx)
2243 if (ctx != NULL)
2245 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
2246 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD)
2248 error_at (gimple_location (stmt),
2249 "OpenMP constructs may not be nested inside simd region");
2250 return false;
2252 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2254 if ((gimple_code (stmt) != GIMPLE_OMP_FOR
2255 || (gimple_omp_for_kind (stmt)
2256 != GF_OMP_FOR_KIND_DISTRIBUTE))
2257 && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
2259 error_at (gimple_location (stmt),
2260 "only distribute or parallel constructs are allowed to "
2261 "be closely nested inside teams construct");
2262 return false;
2266 switch (gimple_code (stmt))
2268 case GIMPLE_OMP_FOR:
2269 if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_KIND_SIMD)
2270 return true;
2271 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
2273 if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
2275 error_at (gimple_location (stmt),
2276 "distribute construct must be closely nested inside "
2277 "teams construct");
2278 return false;
2280 return true;
2282 /* FALLTHRU */
2283 case GIMPLE_CALL:
2284 if (is_gimple_call (stmt)
2285 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2286 == BUILT_IN_GOMP_CANCEL
2287 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2288 == BUILT_IN_GOMP_CANCELLATION_POINT))
2290 const char *bad = NULL;
2291 const char *kind = NULL;
2292 if (ctx == NULL)
2294 error_at (gimple_location (stmt), "orphaned %qs construct",
2295 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2296 == BUILT_IN_GOMP_CANCEL
2297 ? "#pragma omp cancel"
2298 : "#pragma omp cancellation point");
2299 return false;
2301 switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
2302 ? tree_to_shwi (gimple_call_arg (stmt, 0))
2303 : 0)
2305 case 1:
2306 if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
2307 bad = "#pragma omp parallel";
2308 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2309 == BUILT_IN_GOMP_CANCEL
2310 && !integer_zerop (gimple_call_arg (stmt, 1)))
2311 ctx->cancellable = true;
2312 kind = "parallel";
2313 break;
2314 case 2:
2315 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
2316 || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
2317 bad = "#pragma omp for";
2318 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2319 == BUILT_IN_GOMP_CANCEL
2320 && !integer_zerop (gimple_call_arg (stmt, 1)))
2322 ctx->cancellable = true;
2323 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2324 OMP_CLAUSE_NOWAIT))
2325 warning_at (gimple_location (stmt), 0,
2326 "%<#pragma omp cancel for%> inside "
2327 "%<nowait%> for construct");
2328 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2329 OMP_CLAUSE_ORDERED))
2330 warning_at (gimple_location (stmt), 0,
2331 "%<#pragma omp cancel for%> inside "
2332 "%<ordered%> for construct");
2334 kind = "for";
2335 break;
2336 case 4:
2337 if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
2338 && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
2339 bad = "#pragma omp sections";
2340 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2341 == BUILT_IN_GOMP_CANCEL
2342 && !integer_zerop (gimple_call_arg (stmt, 1)))
2344 if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
2346 ctx->cancellable = true;
2347 if (find_omp_clause (gimple_omp_sections_clauses
2348 (ctx->stmt),
2349 OMP_CLAUSE_NOWAIT))
2350 warning_at (gimple_location (stmt), 0,
2351 "%<#pragma omp cancel sections%> inside "
2352 "%<nowait%> sections construct");
2354 else
2356 gcc_assert (ctx->outer
2357 && gimple_code (ctx->outer->stmt)
2358 == GIMPLE_OMP_SECTIONS);
2359 ctx->outer->cancellable = true;
2360 if (find_omp_clause (gimple_omp_sections_clauses
2361 (ctx->outer->stmt),
2362 OMP_CLAUSE_NOWAIT))
2363 warning_at (gimple_location (stmt), 0,
2364 "%<#pragma omp cancel sections%> inside "
2365 "%<nowait%> sections construct");
2368 kind = "sections";
2369 break;
2370 case 8:
2371 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK)
2372 bad = "#pragma omp task";
2373 else
2374 ctx->cancellable = true;
2375 kind = "taskgroup";
2376 break;
2377 default:
2378 error_at (gimple_location (stmt), "invalid arguments");
2379 return false;
2381 if (bad)
2383 error_at (gimple_location (stmt),
2384 "%<%s %s%> construct not closely nested inside of %qs",
2385 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2386 == BUILT_IN_GOMP_CANCEL
2387 ? "#pragma omp cancel"
2388 : "#pragma omp cancellation point", kind, bad);
2389 return false;
2392 /* FALLTHRU */
2393 case GIMPLE_OMP_SECTIONS:
2394 case GIMPLE_OMP_SINGLE:
2395 for (; ctx != NULL; ctx = ctx->outer)
2396 switch (gimple_code (ctx->stmt))
2398 case GIMPLE_OMP_FOR:
2399 case GIMPLE_OMP_SECTIONS:
2400 case GIMPLE_OMP_SINGLE:
2401 case GIMPLE_OMP_ORDERED:
2402 case GIMPLE_OMP_MASTER:
2403 case GIMPLE_OMP_TASK:
2404 case GIMPLE_OMP_CRITICAL:
2405 if (is_gimple_call (stmt))
2407 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2408 != BUILT_IN_GOMP_BARRIER)
2409 return true;
2410 error_at (gimple_location (stmt),
2411 "barrier region may not be closely nested inside "
2412 "of work-sharing, critical, ordered, master or "
2413 "explicit task region");
2414 return false;
2416 error_at (gimple_location (stmt),
2417 "work-sharing region may not be closely nested inside "
2418 "of work-sharing, critical, ordered, master or explicit "
2419 "task region");
2420 return false;
2421 case GIMPLE_OMP_PARALLEL:
2422 return true;
2423 default:
2424 break;
2426 break;
2427 case GIMPLE_OMP_MASTER:
2428 for (; ctx != NULL; ctx = ctx->outer)
2429 switch (gimple_code (ctx->stmt))
2431 case GIMPLE_OMP_FOR:
2432 case GIMPLE_OMP_SECTIONS:
2433 case GIMPLE_OMP_SINGLE:
2434 case GIMPLE_OMP_TASK:
2435 error_at (gimple_location (stmt),
2436 "master region may not be closely nested inside "
2437 "of work-sharing or explicit task region");
2438 return false;
2439 case GIMPLE_OMP_PARALLEL:
2440 return true;
2441 default:
2442 break;
2444 break;
2445 case GIMPLE_OMP_ORDERED:
2446 for (; ctx != NULL; ctx = ctx->outer)
2447 switch (gimple_code (ctx->stmt))
2449 case GIMPLE_OMP_CRITICAL:
2450 case GIMPLE_OMP_TASK:
2451 error_at (gimple_location (stmt),
2452 "ordered region may not be closely nested inside "
2453 "of critical or explicit task region");
2454 return false;
2455 case GIMPLE_OMP_FOR:
2456 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2457 OMP_CLAUSE_ORDERED) == NULL)
2459 error_at (gimple_location (stmt),
2460 "ordered region must be closely nested inside "
2461 "a loop region with an ordered clause");
2462 return false;
2464 return true;
2465 case GIMPLE_OMP_PARALLEL:
2466 error_at (gimple_location (stmt),
2467 "ordered region must be closely nested inside "
2468 "a loop region with an ordered clause");
2469 return false;
2470 default:
2471 break;
2473 break;
2474 case GIMPLE_OMP_CRITICAL:
2475 for (; ctx != NULL; ctx = ctx->outer)
2476 if (gimple_code (ctx->stmt) == GIMPLE_OMP_CRITICAL
2477 && (gimple_omp_critical_name (stmt)
2478 == gimple_omp_critical_name (ctx->stmt)))
2480 error_at (gimple_location (stmt),
2481 "critical region may not be nested inside a critical "
2482 "region with the same name");
2483 return false;
2485 break;
2486 case GIMPLE_OMP_TEAMS:
2487 if (ctx == NULL
2488 || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
2489 || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION)
2491 error_at (gimple_location (stmt),
2492 "teams construct not closely nested inside of target "
2493 "region");
2494 return false;
2496 break;
2497 default:
2498 break;
2500 return true;
2504 /* Helper function scan_omp.
2506 Callback for walk_tree or operators in walk_gimple_stmt used to
2507 scan for OpenMP directives in TP. */
2509 static tree
2510 scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
2512 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
2513 omp_context *ctx = (omp_context *) wi->info;
2514 tree t = *tp;
2516 switch (TREE_CODE (t))
2518 case VAR_DECL:
2519 case PARM_DECL:
2520 case LABEL_DECL:
2521 case RESULT_DECL:
2522 if (ctx)
2523 *tp = remap_decl (t, &ctx->cb);
2524 break;
2526 default:
2527 if (ctx && TYPE_P (t))
2528 *tp = remap_type (t, &ctx->cb);
2529 else if (!DECL_P (t))
2531 *walk_subtrees = 1;
2532 if (ctx)
2534 tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
2535 if (tem != TREE_TYPE (t))
2537 if (TREE_CODE (t) == INTEGER_CST)
2538 *tp = build_int_cst_wide (tem,
2539 TREE_INT_CST_LOW (t),
2540 TREE_INT_CST_HIGH (t));
2541 else
2542 TREE_TYPE (t) = tem;
2546 break;
2549 return NULL_TREE;
2552 /* Return true if FNDECL is a setjmp or a longjmp. */
2554 static bool
2555 setjmp_or_longjmp_p (const_tree fndecl)
2557 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
2558 && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP
2559 || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP))
2560 return true;
2562 tree declname = DECL_NAME (fndecl);
2563 if (!declname)
2564 return false;
2565 const char *name = IDENTIFIER_POINTER (declname);
2566 return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
2570 /* Helper function for scan_omp.
2572 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2573 the current statement in GSI. */
2575 static tree
2576 scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
2577 struct walk_stmt_info *wi)
2579 gimple stmt = gsi_stmt (*gsi);
2580 omp_context *ctx = (omp_context *) wi->info;
2582 if (gimple_has_location (stmt))
2583 input_location = gimple_location (stmt);
2585 /* Check the OpenMP nesting restrictions. */
2586 bool remove = false;
2587 if (is_gimple_omp (stmt))
2588 remove = !check_omp_nesting_restrictions (stmt, ctx);
2589 else if (is_gimple_call (stmt))
2591 tree fndecl = gimple_call_fndecl (stmt);
2592 if (fndecl)
2594 if (setjmp_or_longjmp_p (fndecl)
2595 && ctx
2596 && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
2597 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD)
2599 remove = true;
2600 error_at (gimple_location (stmt),
2601 "setjmp/longjmp inside simd construct");
2603 else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
2604 switch (DECL_FUNCTION_CODE (fndecl))
2606 case BUILT_IN_GOMP_BARRIER:
2607 case BUILT_IN_GOMP_CANCEL:
2608 case BUILT_IN_GOMP_CANCELLATION_POINT:
2609 case BUILT_IN_GOMP_TASKYIELD:
2610 case BUILT_IN_GOMP_TASKWAIT:
2611 case BUILT_IN_GOMP_TASKGROUP_START:
2612 case BUILT_IN_GOMP_TASKGROUP_END:
2613 remove = !check_omp_nesting_restrictions (stmt, ctx);
2614 break;
2615 default:
2616 break;
2620 if (remove)
2622 stmt = gimple_build_nop ();
2623 gsi_replace (gsi, stmt, false);
2626 *handled_ops_p = true;
2628 switch (gimple_code (stmt))
2630 case GIMPLE_OMP_PARALLEL:
2631 taskreg_nesting_level++;
2632 scan_omp_parallel (gsi, ctx);
2633 taskreg_nesting_level--;
2634 break;
2636 case GIMPLE_OMP_TASK:
2637 taskreg_nesting_level++;
2638 scan_omp_task (gsi, ctx);
2639 taskreg_nesting_level--;
2640 break;
2642 case GIMPLE_OMP_FOR:
2643 scan_omp_for (stmt, ctx);
2644 break;
2646 case GIMPLE_OMP_SECTIONS:
2647 scan_omp_sections (stmt, ctx);
2648 break;
2650 case GIMPLE_OMP_SINGLE:
2651 scan_omp_single (stmt, ctx);
2652 break;
2654 case GIMPLE_OMP_SECTION:
2655 case GIMPLE_OMP_MASTER:
2656 case GIMPLE_OMP_TASKGROUP:
2657 case GIMPLE_OMP_ORDERED:
2658 case GIMPLE_OMP_CRITICAL:
2659 ctx = new_omp_context (stmt, ctx);
2660 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2661 break;
2663 case GIMPLE_OMP_TARGET:
2664 scan_omp_target (stmt, ctx);
2665 break;
2667 case GIMPLE_OMP_TEAMS:
2668 scan_omp_teams (stmt, ctx);
2669 break;
2671 case GIMPLE_BIND:
2673 tree var;
2675 *handled_ops_p = false;
2676 if (ctx)
2677 for (var = gimple_bind_vars (stmt); var ; var = DECL_CHAIN (var))
2678 insert_decl_map (&ctx->cb, var, var);
2680 break;
2681 default:
2682 *handled_ops_p = false;
2683 break;
2686 return NULL_TREE;
2690 /* Scan all the statements starting at the current statement. CTX
2691 contains context information about the OpenMP directives and
2692 clauses found during the scan. */
2694 static void
2695 scan_omp (gimple_seq *body_p, omp_context *ctx)
2697 location_t saved_location;
2698 struct walk_stmt_info wi;
2700 memset (&wi, 0, sizeof (wi));
2701 wi.info = ctx;
2702 wi.want_locations = true;
2704 saved_location = input_location;
2705 walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
2706 input_location = saved_location;
2709 /* Re-gimplification and code generation routines. */
2711 /* Build a call to GOMP_barrier. */
2713 static gimple
2714 build_omp_barrier (tree lhs)
2716 tree fndecl = builtin_decl_explicit (lhs ? BUILT_IN_GOMP_BARRIER_CANCEL
2717 : BUILT_IN_GOMP_BARRIER);
2718 gimple g = gimple_build_call (fndecl, 0);
2719 if (lhs)
2720 gimple_call_set_lhs (g, lhs);
2721 return g;
2724 /* If a context was created for STMT when it was scanned, return it. */
2726 static omp_context *
2727 maybe_lookup_ctx (gimple stmt)
2729 splay_tree_node n;
2730 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
2731 return n ? (omp_context *) n->value : NULL;
2735 /* Find the mapping for DECL in CTX or the immediately enclosing
2736 context that has a mapping for DECL.
2738 If CTX is a nested parallel directive, we may have to use the decl
2739 mappings created in CTX's parent context. Suppose that we have the
2740 following parallel nesting (variable UIDs showed for clarity):
2742 iD.1562 = 0;
2743 #omp parallel shared(iD.1562) -> outer parallel
2744 iD.1562 = iD.1562 + 1;
2746 #omp parallel shared (iD.1562) -> inner parallel
2747 iD.1562 = iD.1562 - 1;
2749 Each parallel structure will create a distinct .omp_data_s structure
2750 for copying iD.1562 in/out of the directive:
2752 outer parallel .omp_data_s.1.i -> iD.1562
2753 inner parallel .omp_data_s.2.i -> iD.1562
2755 A shared variable mapping will produce a copy-out operation before
2756 the parallel directive and a copy-in operation after it. So, in
2757 this case we would have:
2759 iD.1562 = 0;
2760 .omp_data_o.1.i = iD.1562;
2761 #omp parallel shared(iD.1562) -> outer parallel
2762 .omp_data_i.1 = &.omp_data_o.1
2763 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2765 .omp_data_o.2.i = iD.1562; -> **
2766 #omp parallel shared(iD.1562) -> inner parallel
2767 .omp_data_i.2 = &.omp_data_o.2
2768 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2771 ** This is a problem. The symbol iD.1562 cannot be referenced
2772 inside the body of the outer parallel region. But since we are
2773 emitting this copy operation while expanding the inner parallel
2774 directive, we need to access the CTX structure of the outer
2775 parallel directive to get the correct mapping:
2777 .omp_data_o.2.i = .omp_data_i.1->i
2779 Since there may be other workshare or parallel directives enclosing
2780 the parallel directive, it may be necessary to walk up the context
2781 parent chain. This is not a problem in general because nested
2782 parallelism happens only rarely. */
2784 static tree
2785 lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
2787 tree t;
2788 omp_context *up;
2790 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
2791 t = maybe_lookup_decl (decl, up);
2793 gcc_assert (!ctx->is_nested || t || is_global_var (decl));
2795 return t ? t : decl;
2799 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2800 in outer contexts. */
2802 static tree
2803 maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
2805 tree t = NULL;
2806 omp_context *up;
2808 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
2809 t = maybe_lookup_decl (decl, up);
2811 return t ? t : decl;
2815 /* Construct the initialization value for reduction CLAUSE. */
2817 tree
2818 omp_reduction_init (tree clause, tree type)
2820 location_t loc = OMP_CLAUSE_LOCATION (clause);
2821 switch (OMP_CLAUSE_REDUCTION_CODE (clause))
2823 case PLUS_EXPR:
2824 case MINUS_EXPR:
2825 case BIT_IOR_EXPR:
2826 case BIT_XOR_EXPR:
2827 case TRUTH_OR_EXPR:
2828 case TRUTH_ORIF_EXPR:
2829 case TRUTH_XOR_EXPR:
2830 case NE_EXPR:
2831 return build_zero_cst (type);
2833 case MULT_EXPR:
2834 case TRUTH_AND_EXPR:
2835 case TRUTH_ANDIF_EXPR:
2836 case EQ_EXPR:
2837 return fold_convert_loc (loc, type, integer_one_node);
2839 case BIT_AND_EXPR:
2840 return fold_convert_loc (loc, type, integer_minus_one_node);
2842 case MAX_EXPR:
2843 if (SCALAR_FLOAT_TYPE_P (type))
2845 REAL_VALUE_TYPE max, min;
2846 if (HONOR_INFINITIES (TYPE_MODE (type)))
2848 real_inf (&max);
2849 real_arithmetic (&min, NEGATE_EXPR, &max, NULL);
2851 else
2852 real_maxval (&min, 1, TYPE_MODE (type));
2853 return build_real (type, min);
2855 else
2857 gcc_assert (INTEGRAL_TYPE_P (type));
2858 return TYPE_MIN_VALUE (type);
2861 case MIN_EXPR:
2862 if (SCALAR_FLOAT_TYPE_P (type))
2864 REAL_VALUE_TYPE max;
2865 if (HONOR_INFINITIES (TYPE_MODE (type)))
2866 real_inf (&max);
2867 else
2868 real_maxval (&max, 0, TYPE_MODE (type));
2869 return build_real (type, max);
2871 else
2873 gcc_assert (INTEGRAL_TYPE_P (type));
2874 return TYPE_MAX_VALUE (type);
2877 default:
2878 gcc_unreachable ();
2882 /* Return alignment to be assumed for var in CLAUSE, which should be
2883 OMP_CLAUSE_ALIGNED. */
2885 static tree
2886 omp_clause_aligned_alignment (tree clause)
2888 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
2889 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
2891 /* Otherwise return implementation defined alignment. */
2892 unsigned int al = 1;
2893 enum machine_mode mode, vmode;
2894 int vs = targetm.vectorize.autovectorize_vector_sizes ();
2895 if (vs)
2896 vs = 1 << floor_log2 (vs);
2897 static enum mode_class classes[]
2898 = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
2899 for (int i = 0; i < 4; i += 2)
2900 for (mode = GET_CLASS_NARROWEST_MODE (classes[i]);
2901 mode != VOIDmode;
2902 mode = GET_MODE_WIDER_MODE (mode))
2904 vmode = targetm.vectorize.preferred_simd_mode (mode);
2905 if (GET_MODE_CLASS (vmode) != classes[i + 1])
2906 continue;
2907 while (vs
2908 && GET_MODE_SIZE (vmode) < vs
2909 && GET_MODE_2XWIDER_MODE (vmode) != VOIDmode)
2910 vmode = GET_MODE_2XWIDER_MODE (vmode);
2912 tree type = lang_hooks.types.type_for_mode (mode, 1);
2913 if (type == NULL_TREE || TYPE_MODE (type) != mode)
2914 continue;
2915 type = build_vector_type (type, GET_MODE_SIZE (vmode)
2916 / GET_MODE_SIZE (mode));
2917 if (TYPE_MODE (type) != vmode)
2918 continue;
2919 if (TYPE_ALIGN_UNIT (type) > al)
2920 al = TYPE_ALIGN_UNIT (type);
2922 return build_int_cst (integer_type_node, al);
2925 /* Return maximum possible vectorization factor for the target. */
2927 static int
2928 omp_max_vf (void)
2930 if (!optimize
2931 || optimize_debug
2932 || !flag_tree_loop_optimize
2933 || (!flag_tree_loop_vectorize
2934 && (global_options_set.x_flag_tree_loop_vectorize
2935 || global_options_set.x_flag_tree_vectorize)))
2936 return 1;
2938 int vs = targetm.vectorize.autovectorize_vector_sizes ();
2939 if (vs)
2941 vs = 1 << floor_log2 (vs);
2942 return vs;
2944 enum machine_mode vqimode = targetm.vectorize.preferred_simd_mode (QImode);
2945 if (GET_MODE_CLASS (vqimode) == MODE_VECTOR_INT)
2946 return GET_MODE_NUNITS (vqimode);
2947 return 1;
2950 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2951 privatization. */
2953 static bool
2954 lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, int &max_vf,
2955 tree &idx, tree &lane, tree &ivar, tree &lvar)
2957 if (max_vf == 0)
2959 max_vf = omp_max_vf ();
2960 if (max_vf > 1)
2962 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2963 OMP_CLAUSE_SAFELEN);
2964 if (c
2965 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c), max_vf) == -1)
2966 max_vf = tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c));
2968 if (max_vf > 1)
2970 idx = create_tmp_var (unsigned_type_node, NULL);
2971 lane = create_tmp_var (unsigned_type_node, NULL);
2974 if (max_vf == 1)
2975 return false;
2977 tree atype = build_array_type_nelts (TREE_TYPE (new_var), max_vf);
2978 tree avar = create_tmp_var_raw (atype, NULL);
2979 if (TREE_ADDRESSABLE (new_var))
2980 TREE_ADDRESSABLE (avar) = 1;
2981 DECL_ATTRIBUTES (avar)
2982 = tree_cons (get_identifier ("omp simd array"), NULL,
2983 DECL_ATTRIBUTES (avar));
2984 gimple_add_tmp_var (avar);
2985 ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, idx,
2986 NULL_TREE, NULL_TREE);
2987 lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, lane,
2988 NULL_TREE, NULL_TREE);
2989 if (DECL_P (new_var))
2991 SET_DECL_VALUE_EXPR (new_var, lvar);
2992 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
2994 return true;
2997 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
2998 from the receiver (aka child) side and initializers for REFERENCE_TYPE
2999 private variables. Initialization statements go in ILIST, while calls
3000 to destructors go in DLIST. */
3002 static void
3003 lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
3004 omp_context *ctx, struct omp_for_data *fd)
3006 tree c, dtor, copyin_seq, x, ptr;
3007 bool copyin_by_ref = false;
3008 bool lastprivate_firstprivate = false;
3009 bool reduction_omp_orig_ref = false;
3010 int pass;
3011 bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3012 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD);
3013 int max_vf = 0;
3014 tree lane = NULL_TREE, idx = NULL_TREE;
3015 tree ivar = NULL_TREE, lvar = NULL_TREE;
3016 gimple_seq llist[2] = { NULL, NULL };
3018 copyin_seq = NULL;
3020 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3021 with data sharing clauses referencing variable sized vars. That
3022 is unnecessarily hard to support and very unlikely to result in
3023 vectorized code anyway. */
3024 if (is_simd)
3025 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3026 switch (OMP_CLAUSE_CODE (c))
3028 case OMP_CLAUSE_REDUCTION:
3029 case OMP_CLAUSE_PRIVATE:
3030 case OMP_CLAUSE_FIRSTPRIVATE:
3031 case OMP_CLAUSE_LASTPRIVATE:
3032 case OMP_CLAUSE_LINEAR:
3033 if (is_variable_sized (OMP_CLAUSE_DECL (c)))
3034 max_vf = 1;
3035 break;
3036 default:
3037 continue;
3040 /* Do all the fixed sized types in the first pass, and the variable sized
3041 types in the second pass. This makes sure that the scalar arguments to
3042 the variable sized types are processed before we use them in the
3043 variable sized operations. */
3044 for (pass = 0; pass < 2; ++pass)
3046 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3048 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
3049 tree var, new_var;
3050 bool by_ref;
3051 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3053 switch (c_kind)
3055 case OMP_CLAUSE_PRIVATE:
3056 if (OMP_CLAUSE_PRIVATE_DEBUG (c))
3057 continue;
3058 break;
3059 case OMP_CLAUSE_SHARED:
3060 /* Ignore shared directives in teams construct. */
3061 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3062 continue;
3063 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
3065 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c)));
3066 continue;
3068 case OMP_CLAUSE_FIRSTPRIVATE:
3069 case OMP_CLAUSE_COPYIN:
3070 case OMP_CLAUSE_LINEAR:
3071 break;
3072 case OMP_CLAUSE_REDUCTION:
3073 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
3074 reduction_omp_orig_ref = true;
3075 break;
3076 case OMP_CLAUSE__LOOPTEMP_:
3077 /* Handle _looptemp_ clauses only on parallel. */
3078 if (fd)
3079 continue;
3080 break;
3081 case OMP_CLAUSE_LASTPRIVATE:
3082 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3084 lastprivate_firstprivate = true;
3085 if (pass != 0)
3086 continue;
3088 break;
3089 case OMP_CLAUSE_ALIGNED:
3090 if (pass == 0)
3091 continue;
3092 var = OMP_CLAUSE_DECL (c);
3093 if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
3094 && !is_global_var (var))
3096 new_var = maybe_lookup_decl (var, ctx);
3097 if (new_var == NULL_TREE)
3098 new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
3099 x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3100 x = build_call_expr_loc (clause_loc, x, 2, new_var,
3101 omp_clause_aligned_alignment (c));
3102 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3103 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
3104 gimplify_and_add (x, ilist);
3106 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
3107 && is_global_var (var))
3109 tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
3110 new_var = lookup_decl (var, ctx);
3111 t = maybe_lookup_decl_in_outer_ctx (var, ctx);
3112 t = build_fold_addr_expr_loc (clause_loc, t);
3113 t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3114 t = build_call_expr_loc (clause_loc, t2, 2, t,
3115 omp_clause_aligned_alignment (c));
3116 t = fold_convert_loc (clause_loc, ptype, t);
3117 x = create_tmp_var (ptype, NULL);
3118 t = build2 (MODIFY_EXPR, ptype, x, t);
3119 gimplify_and_add (t, ilist);
3120 t = build_simple_mem_ref_loc (clause_loc, x);
3121 SET_DECL_VALUE_EXPR (new_var, t);
3122 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3124 continue;
3125 default:
3126 continue;
3129 new_var = var = OMP_CLAUSE_DECL (c);
3130 if (c_kind != OMP_CLAUSE_COPYIN)
3131 new_var = lookup_decl (var, ctx);
3133 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
3135 if (pass != 0)
3136 continue;
3138 else if (is_variable_sized (var))
3140 /* For variable sized types, we need to allocate the
3141 actual storage here. Call alloca and store the
3142 result in the pointer decl that we created elsewhere. */
3143 if (pass == 0)
3144 continue;
3146 if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
3148 gimple stmt;
3149 tree tmp, atmp;
3151 ptr = DECL_VALUE_EXPR (new_var);
3152 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
3153 ptr = TREE_OPERAND (ptr, 0);
3154 gcc_assert (DECL_P (ptr));
3155 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
3157 /* void *tmp = __builtin_alloca */
3158 atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3159 stmt = gimple_build_call (atmp, 1, x);
3160 tmp = create_tmp_var_raw (ptr_type_node, NULL);
3161 gimple_add_tmp_var (tmp);
3162 gimple_call_set_lhs (stmt, tmp);
3164 gimple_seq_add_stmt (ilist, stmt);
3166 x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
3167 gimplify_assign (ptr, x, ilist);
3170 else if (is_reference (var))
3172 /* For references that are being privatized for Fortran,
3173 allocate new backing storage for the new pointer
3174 variable. This allows us to avoid changing all the
3175 code that expects a pointer to something that expects
3176 a direct variable. */
3177 if (pass == 0)
3178 continue;
3180 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
3181 if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
3183 x = build_receiver_ref (var, false, ctx);
3184 x = build_fold_addr_expr_loc (clause_loc, x);
3186 else if (TREE_CONSTANT (x))
3188 /* For reduction with placeholder in SIMD loop,
3189 defer adding the initialization of the reference,
3190 because if we decide to use SIMD array for it,
3191 the initilization could cause expansion ICE. */
3192 if (c_kind == OMP_CLAUSE_REDUCTION
3193 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
3194 && is_simd)
3195 x = NULL_TREE;
3196 else
3198 const char *name = NULL;
3199 if (DECL_NAME (var))
3200 name = IDENTIFIER_POINTER (DECL_NAME (new_var));
3202 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
3203 name);
3204 gimple_add_tmp_var (x);
3205 TREE_ADDRESSABLE (x) = 1;
3206 x = build_fold_addr_expr_loc (clause_loc, x);
3209 else
3211 tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3212 x = build_call_expr_loc (clause_loc, atmp, 1, x);
3215 if (x)
3217 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3218 gimplify_assign (new_var, x, ilist);
3221 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3223 else if (c_kind == OMP_CLAUSE_REDUCTION
3224 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3226 if (pass == 0)
3227 continue;
3229 else if (pass != 0)
3230 continue;
3232 switch (OMP_CLAUSE_CODE (c))
3234 case OMP_CLAUSE_SHARED:
3235 /* Ignore shared directives in teams construct. */
3236 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3237 continue;
3238 /* Shared global vars are just accessed directly. */
3239 if (is_global_var (new_var))
3240 break;
3241 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3242 needs to be delayed until after fixup_child_record_type so
3243 that we get the correct type during the dereference. */
3244 by_ref = use_pointer_for_field (var, ctx);
3245 x = build_receiver_ref (var, by_ref, ctx);
3246 SET_DECL_VALUE_EXPR (new_var, x);
3247 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3249 /* ??? If VAR is not passed by reference, and the variable
3250 hasn't been initialized yet, then we'll get a warning for
3251 the store into the omp_data_s structure. Ideally, we'd be
3252 able to notice this and not store anything at all, but
3253 we're generating code too early. Suppress the warning. */
3254 if (!by_ref)
3255 TREE_NO_WARNING (var) = 1;
3256 break;
3258 case OMP_CLAUSE_LASTPRIVATE:
3259 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3260 break;
3261 /* FALLTHRU */
3263 case OMP_CLAUSE_PRIVATE:
3264 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
3265 x = build_outer_var_ref (var, ctx);
3266 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
3268 if (is_task_ctx (ctx))
3269 x = build_receiver_ref (var, false, ctx);
3270 else
3271 x = build_outer_var_ref (var, ctx);
3273 else
3274 x = NULL;
3275 do_private:
3276 tree nx;
3277 nx = lang_hooks.decls.omp_clause_default_ctor (c, new_var, x);
3278 if (is_simd)
3280 tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
3281 if ((TREE_ADDRESSABLE (new_var) || nx || y
3282 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
3283 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3284 idx, lane, ivar, lvar))
3286 if (nx)
3287 x = lang_hooks.decls.omp_clause_default_ctor
3288 (c, unshare_expr (ivar), x);
3289 if (nx && x)
3290 gimplify_and_add (x, &llist[0]);
3291 if (y)
3293 y = lang_hooks.decls.omp_clause_dtor (c, ivar);
3294 if (y)
3296 gimple_seq tseq = NULL;
3298 dtor = y;
3299 gimplify_stmt (&dtor, &tseq);
3300 gimple_seq_add_seq (&llist[1], tseq);
3303 break;
3306 if (nx)
3307 gimplify_and_add (nx, ilist);
3308 /* FALLTHRU */
3310 do_dtor:
3311 x = lang_hooks.decls.omp_clause_dtor (c, new_var);
3312 if (x)
3314 gimple_seq tseq = NULL;
3316 dtor = x;
3317 gimplify_stmt (&dtor, &tseq);
3318 gimple_seq_add_seq (dlist, tseq);
3320 break;
3322 case OMP_CLAUSE_LINEAR:
3323 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
3324 goto do_firstprivate;
3325 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
3326 x = NULL;
3327 else
3328 x = build_outer_var_ref (var, ctx);
3329 goto do_private;
3331 case OMP_CLAUSE_FIRSTPRIVATE:
3332 if (is_task_ctx (ctx))
3334 if (is_reference (var) || is_variable_sized (var))
3335 goto do_dtor;
3336 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
3337 ctx))
3338 || use_pointer_for_field (var, NULL))
3340 x = build_receiver_ref (var, false, ctx);
3341 SET_DECL_VALUE_EXPR (new_var, x);
3342 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3343 goto do_dtor;
3346 do_firstprivate:
3347 x = build_outer_var_ref (var, ctx);
3348 if (is_simd)
3350 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
3351 && gimple_omp_for_combined_into_p (ctx->stmt))
3353 tree stept = POINTER_TYPE_P (TREE_TYPE (x))
3354 ? sizetype : TREE_TYPE (x);
3355 tree t = fold_convert (stept,
3356 OMP_CLAUSE_LINEAR_STEP (c));
3357 tree c = find_omp_clause (clauses,
3358 OMP_CLAUSE__LOOPTEMP_);
3359 gcc_assert (c);
3360 tree l = OMP_CLAUSE_DECL (c);
3361 if (fd->collapse == 1)
3363 tree n1 = fd->loop.n1;
3364 tree step = fd->loop.step;
3365 tree itype = TREE_TYPE (l);
3366 if (POINTER_TYPE_P (itype))
3367 itype = signed_type_for (itype);
3368 l = fold_build2 (MINUS_EXPR, itype, l, n1);
3369 if (TYPE_UNSIGNED (itype)
3370 && fd->loop.cond_code == GT_EXPR)
3371 l = fold_build2 (TRUNC_DIV_EXPR, itype,
3372 fold_build1 (NEGATE_EXPR,
3373 itype, l),
3374 fold_build1 (NEGATE_EXPR,
3375 itype, step));
3376 else
3377 l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
3379 t = fold_build2 (MULT_EXPR, stept,
3380 fold_convert (stept, l), t);
3381 if (POINTER_TYPE_P (TREE_TYPE (x)))
3382 x = fold_build2 (POINTER_PLUS_EXPR,
3383 TREE_TYPE (x), x, t);
3384 else
3385 x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, t);
3388 if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
3389 || TREE_ADDRESSABLE (new_var))
3390 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3391 idx, lane, ivar, lvar))
3393 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
3395 tree iv = create_tmp_var (TREE_TYPE (new_var), NULL);
3396 x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
3397 gimplify_and_add (x, ilist);
3398 gimple_stmt_iterator gsi
3399 = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
3400 gimple g
3401 = gimple_build_assign (unshare_expr (lvar), iv);
3402 gsi_insert_before_without_update (&gsi, g,
3403 GSI_SAME_STMT);
3404 tree stept = POINTER_TYPE_P (TREE_TYPE (x))
3405 ? sizetype : TREE_TYPE (x);
3406 tree t = fold_convert (stept,
3407 OMP_CLAUSE_LINEAR_STEP (c));
3408 enum tree_code code = PLUS_EXPR;
3409 if (POINTER_TYPE_P (TREE_TYPE (new_var)))
3410 code = POINTER_PLUS_EXPR;
3411 g = gimple_build_assign_with_ops (code, iv, iv, t);
3412 gsi_insert_before_without_update (&gsi, g,
3413 GSI_SAME_STMT);
3414 break;
3416 x = lang_hooks.decls.omp_clause_copy_ctor
3417 (c, unshare_expr (ivar), x);
3418 gimplify_and_add (x, &llist[0]);
3419 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
3420 if (x)
3422 gimple_seq tseq = NULL;
3424 dtor = x;
3425 gimplify_stmt (&dtor, &tseq);
3426 gimple_seq_add_seq (&llist[1], tseq);
3428 break;
3431 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
3432 gimplify_and_add (x, ilist);
3433 goto do_dtor;
3435 case OMP_CLAUSE__LOOPTEMP_:
3436 gcc_assert (is_parallel_ctx (ctx));
3437 x = build_outer_var_ref (var, ctx);
3438 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
3439 gimplify_and_add (x, ilist);
3440 break;
3442 case OMP_CLAUSE_COPYIN:
3443 by_ref = use_pointer_for_field (var, NULL);
3444 x = build_receiver_ref (var, by_ref, ctx);
3445 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
3446 append_to_statement_list (x, &copyin_seq);
3447 copyin_by_ref |= by_ref;
3448 break;
3450 case OMP_CLAUSE_REDUCTION:
3451 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3453 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
3454 gimple tseq;
3455 x = build_outer_var_ref (var, ctx);
3457 if (is_reference (var)
3458 && !useless_type_conversion_p (TREE_TYPE (placeholder),
3459 TREE_TYPE (x)))
3460 x = build_fold_addr_expr_loc (clause_loc, x);
3461 SET_DECL_VALUE_EXPR (placeholder, x);
3462 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
3463 tree new_vard = new_var;
3464 if (is_reference (var))
3466 gcc_assert (TREE_CODE (new_var) == MEM_REF);
3467 new_vard = TREE_OPERAND (new_var, 0);
3468 gcc_assert (DECL_P (new_vard));
3470 if (is_simd
3471 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3472 idx, lane, ivar, lvar))
3474 if (new_vard == new_var)
3476 gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
3477 SET_DECL_VALUE_EXPR (new_var, ivar);
3479 else
3481 SET_DECL_VALUE_EXPR (new_vard,
3482 build_fold_addr_expr (ivar));
3483 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
3485 x = lang_hooks.decls.omp_clause_default_ctor
3486 (c, unshare_expr (ivar),
3487 build_outer_var_ref (var, ctx));
3488 if (x)
3489 gimplify_and_add (x, &llist[0]);
3490 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
3492 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
3493 lower_omp (&tseq, ctx);
3494 gimple_seq_add_seq (&llist[0], tseq);
3496 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
3497 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
3498 lower_omp (&tseq, ctx);
3499 gimple_seq_add_seq (&llist[1], tseq);
3500 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
3501 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
3502 if (new_vard == new_var)
3503 SET_DECL_VALUE_EXPR (new_var, lvar);
3504 else
3505 SET_DECL_VALUE_EXPR (new_vard,
3506 build_fold_addr_expr (lvar));
3507 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
3508 if (x)
3510 tseq = NULL;
3511 dtor = x;
3512 gimplify_stmt (&dtor, &tseq);
3513 gimple_seq_add_seq (&llist[1], tseq);
3515 break;
3517 /* If this is a reference to constant size reduction var
3518 with placeholder, we haven't emitted the initializer
3519 for it because it is undesirable if SIMD arrays are used.
3520 But if they aren't used, we need to emit the deferred
3521 initialization now. */
3522 else if (is_reference (var) && is_simd)
3524 tree z
3525 = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
3526 if (TREE_CONSTANT (z))
3528 const char *name = NULL;
3529 if (DECL_NAME (var))
3530 name = IDENTIFIER_POINTER (DECL_NAME (new_vard));
3532 z = create_tmp_var_raw
3533 (TREE_TYPE (TREE_TYPE (new_vard)), name);
3534 gimple_add_tmp_var (z);
3535 TREE_ADDRESSABLE (z) = 1;
3536 z = build_fold_addr_expr_loc (clause_loc, z);
3537 gimplify_assign (new_vard, z, ilist);
3540 x = lang_hooks.decls.omp_clause_default_ctor
3541 (c, new_var, unshare_expr (x));
3542 if (x)
3543 gimplify_and_add (x, ilist);
3544 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
3546 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
3547 lower_omp (&tseq, ctx);
3548 gimple_seq_add_seq (ilist, tseq);
3550 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
3551 if (is_simd)
3553 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
3554 lower_omp (&tseq, ctx);
3555 gimple_seq_add_seq (dlist, tseq);
3556 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
3558 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
3559 goto do_dtor;
3561 else
3563 x = omp_reduction_init (c, TREE_TYPE (new_var));
3564 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
3565 enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
3567 /* reduction(-:var) sums up the partial results, so it
3568 acts identically to reduction(+:var). */
3569 if (code == MINUS_EXPR)
3570 code = PLUS_EXPR;
3572 if (is_simd
3573 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3574 idx, lane, ivar, lvar))
3576 tree ref = build_outer_var_ref (var, ctx);
3578 gimplify_assign (unshare_expr (ivar), x, &llist[0]);
3580 x = build2 (code, TREE_TYPE (ref), ref, ivar);
3581 ref = build_outer_var_ref (var, ctx);
3582 gimplify_assign (ref, x, &llist[1]);
3584 else
3586 gimplify_assign (new_var, x, ilist);
3587 if (is_simd)
3589 tree ref = build_outer_var_ref (var, ctx);
3591 x = build2 (code, TREE_TYPE (ref), ref, new_var);
3592 ref = build_outer_var_ref (var, ctx);
3593 gimplify_assign (ref, x, dlist);
3597 break;
3599 default:
3600 gcc_unreachable ();
3605 if (lane)
3607 tree uid = create_tmp_var (ptr_type_node, "simduid");
3608 /* Don't want uninit warnings on simduid, it is always uninitialized,
3609 but we use it not for the value, but for the DECL_UID only. */
3610 TREE_NO_WARNING (uid) = 1;
3611 gimple g
3612 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid);
3613 gimple_call_set_lhs (g, lane);
3614 gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
3615 gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
3616 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
3617 OMP_CLAUSE__SIMDUID__DECL (c) = uid;
3618 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
3619 gimple_omp_for_set_clauses (ctx->stmt, c);
3620 g = gimple_build_assign_with_ops (INTEGER_CST, lane,
3621 build_int_cst (unsigned_type_node, 0),
3622 NULL_TREE);
3623 gimple_seq_add_stmt (ilist, g);
3624 for (int i = 0; i < 2; i++)
3625 if (llist[i])
3627 tree vf = create_tmp_var (unsigned_type_node, NULL);
3628 g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
3629 gimple_call_set_lhs (g, vf);
3630 gimple_seq *seq = i == 0 ? ilist : dlist;
3631 gimple_seq_add_stmt (seq, g);
3632 tree t = build_int_cst (unsigned_type_node, 0);
3633 g = gimple_build_assign_with_ops (INTEGER_CST, idx, t, NULL_TREE);
3634 gimple_seq_add_stmt (seq, g);
3635 tree body = create_artificial_label (UNKNOWN_LOCATION);
3636 tree header = create_artificial_label (UNKNOWN_LOCATION);
3637 tree end = create_artificial_label (UNKNOWN_LOCATION);
3638 gimple_seq_add_stmt (seq, gimple_build_goto (header));
3639 gimple_seq_add_stmt (seq, gimple_build_label (body));
3640 gimple_seq_add_seq (seq, llist[i]);
3641 t = build_int_cst (unsigned_type_node, 1);
3642 g = gimple_build_assign_with_ops (PLUS_EXPR, idx, idx, t);
3643 gimple_seq_add_stmt (seq, g);
3644 gimple_seq_add_stmt (seq, gimple_build_label (header));
3645 g = gimple_build_cond (LT_EXPR, idx, vf, body, end);
3646 gimple_seq_add_stmt (seq, g);
3647 gimple_seq_add_stmt (seq, gimple_build_label (end));
3651 /* The copyin sequence is not to be executed by the main thread, since
3652 that would result in self-copies. Perhaps not visible to scalars,
3653 but it certainly is to C++ operator=. */
3654 if (copyin_seq)
3656 x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
3658 x = build2 (NE_EXPR, boolean_type_node, x,
3659 build_int_cst (TREE_TYPE (x), 0));
3660 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
3661 gimplify_and_add (x, ilist);
3664 /* If any copyin variable is passed by reference, we must ensure the
3665 master thread doesn't modify it before it is copied over in all
3666 threads. Similarly for variables in both firstprivate and
3667 lastprivate clauses we need to ensure the lastprivate copying
3668 happens after firstprivate copying in all threads. And similarly
3669 for UDRs if initializer expression refers to omp_orig. */
3670 if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref)
3672 /* Don't add any barrier for #pragma omp simd or
3673 #pragma omp distribute. */
3674 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3675 || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR)
3676 gimple_seq_add_stmt (ilist, build_omp_barrier (NULL_TREE));
3679 /* If max_vf is non-zero, then we can use only a vectorization factor
3680 up to the max_vf we chose. So stick it into the safelen clause. */
3681 if (max_vf)
3683 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3684 OMP_CLAUSE_SAFELEN);
3685 if (c == NULL_TREE
3686 || compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
3687 max_vf) == 1)
3689 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
3690 OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
3691 max_vf);
3692 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
3693 gimple_omp_for_set_clauses (ctx->stmt, c);
3699 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3700 both parallel and workshare constructs. PREDICATE may be NULL if it's
3701 always true. */
3703 static void
3704 lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list,
3705 omp_context *ctx)
3707 tree x, c, label = NULL, orig_clauses = clauses;
3708 bool par_clauses = false;
3709 tree simduid = NULL, lastlane = NULL;
3711 /* Early exit if there are no lastprivate or linear clauses. */
3712 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
3713 if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
3714 || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
3715 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
3716 break;
3717 if (clauses == NULL)
3719 /* If this was a workshare clause, see if it had been combined
3720 with its parallel. In that case, look for the clauses on the
3721 parallel statement itself. */
3722 if (is_parallel_ctx (ctx))
3723 return;
3725 ctx = ctx->outer;
3726 if (ctx == NULL || !is_parallel_ctx (ctx))
3727 return;
3729 clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
3730 OMP_CLAUSE_LASTPRIVATE);
3731 if (clauses == NULL)
3732 return;
3733 par_clauses = true;
3736 if (predicate)
3738 gimple stmt;
3739 tree label_true, arm1, arm2;
3741 label = create_artificial_label (UNKNOWN_LOCATION);
3742 label_true = create_artificial_label (UNKNOWN_LOCATION);
3743 arm1 = TREE_OPERAND (predicate, 0);
3744 arm2 = TREE_OPERAND (predicate, 1);
3745 gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
3746 gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
3747 stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2,
3748 label_true, label);
3749 gimple_seq_add_stmt (stmt_list, stmt);
3750 gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
3753 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3754 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD)
3756 simduid = find_omp_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
3757 if (simduid)
3758 simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
3761 for (c = clauses; c ;)
3763 tree var, new_var;
3764 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3766 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
3767 || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
3768 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
3770 var = OMP_CLAUSE_DECL (c);
3771 new_var = lookup_decl (var, ctx);
3773 if (simduid && DECL_HAS_VALUE_EXPR_P (new_var))
3775 tree val = DECL_VALUE_EXPR (new_var);
3776 if (TREE_CODE (val) == ARRAY_REF
3777 && VAR_P (TREE_OPERAND (val, 0))
3778 && lookup_attribute ("omp simd array",
3779 DECL_ATTRIBUTES (TREE_OPERAND (val,
3780 0))))
3782 if (lastlane == NULL)
3784 lastlane = create_tmp_var (unsigned_type_node, NULL);
3785 gimple g
3786 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
3787 2, simduid,
3788 TREE_OPERAND (val, 1));
3789 gimple_call_set_lhs (g, lastlane);
3790 gimple_seq_add_stmt (stmt_list, g);
3792 new_var = build4 (ARRAY_REF, TREE_TYPE (val),
3793 TREE_OPERAND (val, 0), lastlane,
3794 NULL_TREE, NULL_TREE);
3798 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
3799 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
3801 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
3802 gimple_seq_add_seq (stmt_list,
3803 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
3804 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
3807 x = build_outer_var_ref (var, ctx);
3808 if (is_reference (var))
3809 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3810 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
3811 gimplify_and_add (x, stmt_list);
3813 c = OMP_CLAUSE_CHAIN (c);
3814 if (c == NULL && !par_clauses)
3816 /* If this was a workshare clause, see if it had been combined
3817 with its parallel. In that case, continue looking for the
3818 clauses also on the parallel statement itself. */
3819 if (is_parallel_ctx (ctx))
3820 break;
3822 ctx = ctx->outer;
3823 if (ctx == NULL || !is_parallel_ctx (ctx))
3824 break;
3826 c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
3827 OMP_CLAUSE_LASTPRIVATE);
3828 par_clauses = true;
3832 if (label)
3833 gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
3837 /* Generate code to implement the REDUCTION clauses. */
3839 static void
3840 lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx)
3842 gimple_seq sub_seq = NULL;
3843 gimple stmt;
3844 tree x, c;
3845 int count = 0;
3847 /* SIMD reductions are handled in lower_rec_input_clauses. */
3848 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3849 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD)
3850 return;
3852 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3853 update in that case, otherwise use a lock. */
3854 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
3855 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
3857 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3859 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3860 count = -1;
3861 break;
3863 count++;
3866 if (count == 0)
3867 return;
3869 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3871 tree var, ref, new_var;
3872 enum tree_code code;
3873 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3875 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
3876 continue;
3878 var = OMP_CLAUSE_DECL (c);
3879 new_var = lookup_decl (var, ctx);
3880 if (is_reference (var))
3881 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3882 ref = build_outer_var_ref (var, ctx);
3883 code = OMP_CLAUSE_REDUCTION_CODE (c);
3885 /* reduction(-:var) sums up the partial results, so it acts
3886 identically to reduction(+:var). */
3887 if (code == MINUS_EXPR)
3888 code = PLUS_EXPR;
3890 if (count == 1)
3892 tree addr = build_fold_addr_expr_loc (clause_loc, ref);
3894 addr = save_expr (addr);
3895 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
3896 x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var);
3897 x = build2 (OMP_ATOMIC, void_type_node, addr, x);
3898 gimplify_and_add (x, stmt_seqp);
3899 return;
3902 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3904 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
3906 if (is_reference (var)
3907 && !useless_type_conversion_p (TREE_TYPE (placeholder),
3908 TREE_TYPE (ref)))
3909 ref = build_fold_addr_expr_loc (clause_loc, ref);
3910 SET_DECL_VALUE_EXPR (placeholder, ref);
3911 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
3912 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
3913 gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
3914 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
3915 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
3917 else
3919 x = build2 (code, TREE_TYPE (ref), ref, new_var);
3920 ref = build_outer_var_ref (var, ctx);
3921 gimplify_assign (ref, x, &sub_seq);
3925 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START),
3927 gimple_seq_add_stmt (stmt_seqp, stmt);
3929 gimple_seq_add_seq (stmt_seqp, sub_seq);
3931 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
3933 gimple_seq_add_stmt (stmt_seqp, stmt);
3937 /* Generate code to implement the COPYPRIVATE clauses. */
3939 static void
3940 lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
3941 omp_context *ctx)
3943 tree c;
3945 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3947 tree var, new_var, ref, x;
3948 bool by_ref;
3949 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3951 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
3952 continue;
3954 var = OMP_CLAUSE_DECL (c);
3955 by_ref = use_pointer_for_field (var, NULL);
3957 ref = build_sender_ref (var, ctx);
3958 x = new_var = lookup_decl_in_outer_ctx (var, ctx);
3959 if (by_ref)
3961 x = build_fold_addr_expr_loc (clause_loc, new_var);
3962 x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
3964 gimplify_assign (ref, x, slist);
3966 ref = build_receiver_ref (var, false, ctx);
3967 if (by_ref)
3969 ref = fold_convert_loc (clause_loc,
3970 build_pointer_type (TREE_TYPE (new_var)),
3971 ref);
3972 ref = build_fold_indirect_ref_loc (clause_loc, ref);
3974 if (is_reference (var))
3976 ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
3977 ref = build_simple_mem_ref_loc (clause_loc, ref);
3978 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3980 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
3981 gimplify_and_add (x, rlist);
3986 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
3987 and REDUCTION from the sender (aka parent) side. */
3989 static void
3990 lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
3991 omp_context *ctx)
3993 tree c;
3995 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3997 tree val, ref, x, var;
3998 bool by_ref, do_in = false, do_out = false;
3999 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4001 switch (OMP_CLAUSE_CODE (c))
4003 case OMP_CLAUSE_PRIVATE:
4004 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
4005 break;
4006 continue;
4007 case OMP_CLAUSE_FIRSTPRIVATE:
4008 case OMP_CLAUSE_COPYIN:
4009 case OMP_CLAUSE_LASTPRIVATE:
4010 case OMP_CLAUSE_REDUCTION:
4011 case OMP_CLAUSE__LOOPTEMP_:
4012 break;
4013 default:
4014 continue;
4017 val = OMP_CLAUSE_DECL (c);
4018 var = lookup_decl_in_outer_ctx (val, ctx);
4020 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
4021 && is_global_var (var))
4022 continue;
4023 if (is_variable_sized (val))
4024 continue;
4025 by_ref = use_pointer_for_field (val, NULL);
4027 switch (OMP_CLAUSE_CODE (c))
4029 case OMP_CLAUSE_PRIVATE:
4030 case OMP_CLAUSE_FIRSTPRIVATE:
4031 case OMP_CLAUSE_COPYIN:
4032 case OMP_CLAUSE__LOOPTEMP_:
4033 do_in = true;
4034 break;
4036 case OMP_CLAUSE_LASTPRIVATE:
4037 if (by_ref || is_reference (val))
4039 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
4040 continue;
4041 do_in = true;
4043 else
4045 do_out = true;
4046 if (lang_hooks.decls.omp_private_outer_ref (val))
4047 do_in = true;
4049 break;
4051 case OMP_CLAUSE_REDUCTION:
4052 do_in = true;
4053 do_out = !(by_ref || is_reference (val));
4054 break;
4056 default:
4057 gcc_unreachable ();
4060 if (do_in)
4062 ref = build_sender_ref (val, ctx);
4063 x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
4064 gimplify_assign (ref, x, ilist);
4065 if (is_task_ctx (ctx))
4066 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
4069 if (do_out)
4071 ref = build_sender_ref (val, ctx);
4072 gimplify_assign (var, ref, olist);
4077 /* Generate code to implement SHARED from the sender (aka parent)
4078 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4079 list things that got automatically shared. */
4081 static void
4082 lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
4084 tree var, ovar, nvar, f, x, record_type;
4086 if (ctx->record_type == NULL)
4087 return;
4089 record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
4090 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
4092 ovar = DECL_ABSTRACT_ORIGIN (f);
4093 nvar = maybe_lookup_decl (ovar, ctx);
4094 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar))
4095 continue;
4097 /* If CTX is a nested parallel directive. Find the immediately
4098 enclosing parallel or workshare construct that contains a
4099 mapping for OVAR. */
4100 var = lookup_decl_in_outer_ctx (ovar, ctx);
4102 if (use_pointer_for_field (ovar, ctx))
4104 x = build_sender_ref (ovar, ctx);
4105 var = build_fold_addr_expr (var);
4106 gimplify_assign (x, var, ilist);
4108 else
4110 x = build_sender_ref (ovar, ctx);
4111 gimplify_assign (x, var, ilist);
4113 if (!TREE_READONLY (var)
4114 /* We don't need to receive a new reference to a result
4115 or parm decl. In fact we may not store to it as we will
4116 invalidate any pending RSO and generate wrong gimple
4117 during inlining. */
4118 && !((TREE_CODE (var) == RESULT_DECL
4119 || TREE_CODE (var) == PARM_DECL)
4120 && DECL_BY_REFERENCE (var)))
4122 x = build_sender_ref (ovar, ctx);
4123 gimplify_assign (var, x, olist);
4130 /* A convenience function to build an empty GIMPLE_COND with just the
4131 condition. */
4133 static gimple
4134 gimple_build_cond_empty (tree cond)
4136 enum tree_code pred_code;
4137 tree lhs, rhs;
4139 gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs);
4140 return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE);
4144 /* Build the function calls to GOMP_parallel_start etc to actually
4145 generate the parallel operation. REGION is the parallel region
4146 being expanded. BB is the block where to insert the code. WS_ARGS
4147 will be set if this is a call to a combined parallel+workshare
4148 construct, it contains the list of additional arguments needed by
4149 the workshare construct. */
4151 static void
4152 expand_parallel_call (struct omp_region *region, basic_block bb,
4153 gimple entry_stmt, vec<tree, va_gc> *ws_args)
4155 tree t, t1, t2, val, cond, c, clauses, flags;
4156 gimple_stmt_iterator gsi;
4157 gimple stmt;
4158 enum built_in_function start_ix;
4159 int start_ix2;
4160 location_t clause_loc;
4161 vec<tree, va_gc> *args;
4163 clauses = gimple_omp_parallel_clauses (entry_stmt);
4165 /* Determine what flavor of GOMP_parallel we will be
4166 emitting. */
4167 start_ix = BUILT_IN_GOMP_PARALLEL;
4168 if (is_combined_parallel (region))
4170 switch (region->inner->type)
4172 case GIMPLE_OMP_FOR:
4173 gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
4174 start_ix2 = ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4175 + (region->inner->sched_kind
4176 == OMP_CLAUSE_SCHEDULE_RUNTIME
4177 ? 3 : region->inner->sched_kind));
4178 start_ix = (enum built_in_function)start_ix2;
4179 break;
4180 case GIMPLE_OMP_SECTIONS:
4181 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS;
4182 break;
4183 default:
4184 gcc_unreachable ();
4188 /* By default, the value of NUM_THREADS is zero (selected at run time)
4189 and there is no conditional. */
4190 cond = NULL_TREE;
4191 val = build_int_cst (unsigned_type_node, 0);
4192 flags = build_int_cst (unsigned_type_node, 0);
4194 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
4195 if (c)
4196 cond = OMP_CLAUSE_IF_EXPR (c);
4198 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS);
4199 if (c)
4201 val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
4202 clause_loc = OMP_CLAUSE_LOCATION (c);
4204 else
4205 clause_loc = gimple_location (entry_stmt);
4207 c = find_omp_clause (clauses, OMP_CLAUSE_PROC_BIND);
4208 if (c)
4209 flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c));
4211 /* Ensure 'val' is of the correct type. */
4212 val = fold_convert_loc (clause_loc, unsigned_type_node, val);
4214 /* If we found the clause 'if (cond)', build either
4215 (cond != 0) or (cond ? val : 1u). */
4216 if (cond)
4218 gimple_stmt_iterator gsi;
4220 cond = gimple_boolify (cond);
4222 if (integer_zerop (val))
4223 val = fold_build2_loc (clause_loc,
4224 EQ_EXPR, unsigned_type_node, cond,
4225 build_int_cst (TREE_TYPE (cond), 0));
4226 else
4228 basic_block cond_bb, then_bb, else_bb;
4229 edge e, e_then, e_else;
4230 tree tmp_then, tmp_else, tmp_join, tmp_var;
4232 tmp_var = create_tmp_var (TREE_TYPE (val), NULL);
4233 if (gimple_in_ssa_p (cfun))
4235 tmp_then = make_ssa_name (tmp_var, NULL);
4236 tmp_else = make_ssa_name (tmp_var, NULL);
4237 tmp_join = make_ssa_name (tmp_var, NULL);
4239 else
4241 tmp_then = tmp_var;
4242 tmp_else = tmp_var;
4243 tmp_join = tmp_var;
4246 e = split_block (bb, NULL);
4247 cond_bb = e->src;
4248 bb = e->dest;
4249 remove_edge (e);
4251 then_bb = create_empty_bb (cond_bb);
4252 else_bb = create_empty_bb (then_bb);
4253 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
4254 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
4256 stmt = gimple_build_cond_empty (cond);
4257 gsi = gsi_start_bb (cond_bb);
4258 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4260 gsi = gsi_start_bb (then_bb);
4261 stmt = gimple_build_assign (tmp_then, val);
4262 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4264 gsi = gsi_start_bb (else_bb);
4265 stmt = gimple_build_assign
4266 (tmp_else, build_int_cst (unsigned_type_node, 1));
4267 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4269 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
4270 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
4271 if (current_loops)
4273 add_bb_to_loop (then_bb, cond_bb->loop_father);
4274 add_bb_to_loop (else_bb, cond_bb->loop_father);
4276 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
4277 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
4279 if (gimple_in_ssa_p (cfun))
4281 gimple phi = create_phi_node (tmp_join, bb);
4282 add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION);
4283 add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION);
4286 val = tmp_join;
4289 gsi = gsi_start_bb (bb);
4290 val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE,
4291 false, GSI_CONTINUE_LINKING);
4294 gsi = gsi_last_bb (bb);
4295 t = gimple_omp_parallel_data_arg (entry_stmt);
4296 if (t == NULL)
4297 t1 = null_pointer_node;
4298 else
4299 t1 = build_fold_addr_expr (t);
4300 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
4302 vec_alloc (args, 4 + vec_safe_length (ws_args));
4303 args->quick_push (t2);
4304 args->quick_push (t1);
4305 args->quick_push (val);
4306 if (ws_args)
4307 args->splice (*ws_args);
4308 args->quick_push (flags);
4310 t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
4311 builtin_decl_explicit (start_ix), args);
4313 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4314 false, GSI_CONTINUE_LINKING);
4318 /* Build the function call to GOMP_task to actually
4319 generate the task operation. BB is the block where to insert the code. */
4321 static void
4322 expand_task_call (basic_block bb, gimple entry_stmt)
4324 tree t, t1, t2, t3, flags, cond, c, c2, clauses, depend;
4325 gimple_stmt_iterator gsi;
4326 location_t loc = gimple_location (entry_stmt);
4328 clauses = gimple_omp_task_clauses (entry_stmt);
4330 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
4331 if (c)
4332 cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (c));
4333 else
4334 cond = boolean_true_node;
4336 c = find_omp_clause (clauses, OMP_CLAUSE_UNTIED);
4337 c2 = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE);
4338 depend = find_omp_clause (clauses, OMP_CLAUSE_DEPEND);
4339 flags = build_int_cst (unsigned_type_node,
4340 (c ? 1 : 0) + (c2 ? 4 : 0) + (depend ? 8 : 0));
4342 c = find_omp_clause (clauses, OMP_CLAUSE_FINAL);
4343 if (c)
4345 c = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c));
4346 c = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, c,
4347 build_int_cst (unsigned_type_node, 2),
4348 build_int_cst (unsigned_type_node, 0));
4349 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, c);
4351 if (depend)
4352 depend = OMP_CLAUSE_DECL (depend);
4353 else
4354 depend = build_int_cst (ptr_type_node, 0);
4356 gsi = gsi_last_bb (bb);
4357 t = gimple_omp_task_data_arg (entry_stmt);
4358 if (t == NULL)
4359 t2 = null_pointer_node;
4360 else
4361 t2 = build_fold_addr_expr_loc (loc, t);
4362 t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt));
4363 t = gimple_omp_task_copy_fn (entry_stmt);
4364 if (t == NULL)
4365 t3 = null_pointer_node;
4366 else
4367 t3 = build_fold_addr_expr_loc (loc, t);
4369 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
4370 8, t1, t2, t3,
4371 gimple_omp_task_arg_size (entry_stmt),
4372 gimple_omp_task_arg_align (entry_stmt), cond, flags,
4373 depend);
4375 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4376 false, GSI_CONTINUE_LINKING);
4380 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4381 catch handler and return it. This prevents programs from violating the
4382 structured block semantics with throws. */
4384 static gimple_seq
4385 maybe_catch_exception (gimple_seq body)
4387 gimple g;
4388 tree decl;
4390 if (!flag_exceptions)
4391 return body;
4393 if (lang_hooks.eh_protect_cleanup_actions != NULL)
4394 decl = lang_hooks.eh_protect_cleanup_actions ();
4395 else
4396 decl = builtin_decl_explicit (BUILT_IN_TRAP);
4398 g = gimple_build_eh_must_not_throw (decl);
4399 g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
4400 GIMPLE_TRY_CATCH);
4402 return gimple_seq_alloc_with_stmt (g);
4405 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4407 static tree
4408 vec2chain (vec<tree, va_gc> *v)
4410 tree chain = NULL_TREE, t;
4411 unsigned ix;
4413 FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t)
4415 DECL_CHAIN (t) = chain;
4416 chain = t;
4419 return chain;
4423 /* Remove barriers in REGION->EXIT's block. Note that this is only
4424 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4425 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4426 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4427 removed. */
4429 static void
4430 remove_exit_barrier (struct omp_region *region)
4432 gimple_stmt_iterator gsi;
4433 basic_block exit_bb;
4434 edge_iterator ei;
4435 edge e;
4436 gimple stmt;
4437 int any_addressable_vars = -1;
4439 exit_bb = region->exit;
4441 /* If the parallel region doesn't return, we don't have REGION->EXIT
4442 block at all. */
4443 if (! exit_bb)
4444 return;
4446 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4447 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4448 statements that can appear in between are extremely limited -- no
4449 memory operations at all. Here, we allow nothing at all, so the
4450 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4451 gsi = gsi_last_bb (exit_bb);
4452 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
4453 gsi_prev (&gsi);
4454 if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
4455 return;
4457 FOR_EACH_EDGE (e, ei, exit_bb->preds)
4459 gsi = gsi_last_bb (e->src);
4460 if (gsi_end_p (gsi))
4461 continue;
4462 stmt = gsi_stmt (gsi);
4463 if (gimple_code (stmt) == GIMPLE_OMP_RETURN
4464 && !gimple_omp_return_nowait_p (stmt))
4466 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4467 in many cases. If there could be tasks queued, the barrier
4468 might be needed to let the tasks run before some local
4469 variable of the parallel that the task uses as shared
4470 runs out of scope. The task can be spawned either
4471 from within current function (this would be easy to check)
4472 or from some function it calls and gets passed an address
4473 of such a variable. */
4474 if (any_addressable_vars < 0)
4476 gimple parallel_stmt = last_stmt (region->entry);
4477 tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt);
4478 tree local_decls, block, decl;
4479 unsigned ix;
4481 any_addressable_vars = 0;
4482 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl)
4483 if (TREE_ADDRESSABLE (decl))
4485 any_addressable_vars = 1;
4486 break;
4488 for (block = gimple_block (stmt);
4489 !any_addressable_vars
4490 && block
4491 && TREE_CODE (block) == BLOCK;
4492 block = BLOCK_SUPERCONTEXT (block))
4494 for (local_decls = BLOCK_VARS (block);
4495 local_decls;
4496 local_decls = DECL_CHAIN (local_decls))
4497 if (TREE_ADDRESSABLE (local_decls))
4499 any_addressable_vars = 1;
4500 break;
4502 if (block == gimple_block (parallel_stmt))
4503 break;
4506 if (!any_addressable_vars)
4507 gimple_omp_return_set_nowait (stmt);
4512 static void
4513 remove_exit_barriers (struct omp_region *region)
4515 if (region->type == GIMPLE_OMP_PARALLEL)
4516 remove_exit_barrier (region);
4518 if (region->inner)
4520 region = region->inner;
4521 remove_exit_barriers (region);
4522 while (region->next)
4524 region = region->next;
4525 remove_exit_barriers (region);
4530 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4531 calls. These can't be declared as const functions, but
4532 within one parallel body they are constant, so they can be
4533 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4534 which are declared const. Similarly for task body, except
4535 that in untied task omp_get_thread_num () can change at any task
4536 scheduling point. */
4538 static void
4539 optimize_omp_library_calls (gimple entry_stmt)
4541 basic_block bb;
4542 gimple_stmt_iterator gsi;
4543 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
4544 tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree);
4545 tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
4546 tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree);
4547 bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
4548 && find_omp_clause (gimple_omp_task_clauses (entry_stmt),
4549 OMP_CLAUSE_UNTIED) != NULL);
4551 FOR_EACH_BB_FN (bb, cfun)
4552 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4554 gimple call = gsi_stmt (gsi);
4555 tree decl;
4557 if (is_gimple_call (call)
4558 && (decl = gimple_call_fndecl (call))
4559 && DECL_EXTERNAL (decl)
4560 && TREE_PUBLIC (decl)
4561 && DECL_INITIAL (decl) == NULL)
4563 tree built_in;
4565 if (DECL_NAME (decl) == thr_num_id)
4567 /* In #pragma omp task untied omp_get_thread_num () can change
4568 during the execution of the task region. */
4569 if (untied_task)
4570 continue;
4571 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
4573 else if (DECL_NAME (decl) == num_thr_id)
4574 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
4575 else
4576 continue;
4578 if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in)
4579 || gimple_call_num_args (call) != 0)
4580 continue;
4582 if (flag_exceptions && !TREE_NOTHROW (decl))
4583 continue;
4585 if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE
4586 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)),
4587 TREE_TYPE (TREE_TYPE (built_in))))
4588 continue;
4590 gimple_call_set_fndecl (call, built_in);
4595 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4596 regimplified. */
4598 static tree
4599 expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *)
4601 tree t = *tp;
4603 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4604 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t))
4605 return t;
4607 if (TREE_CODE (t) == ADDR_EXPR)
4608 recompute_tree_invariant_for_addr_expr (t);
4610 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
4611 return NULL_TREE;
4614 /* Prepend TO = FROM assignment before *GSI_P. */
4616 static void
4617 expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from)
4619 bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to);
4620 from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE,
4621 true, GSI_SAME_STMT);
4622 gimple stmt = gimple_build_assign (to, from);
4623 gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT);
4624 if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL)
4625 || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL))
4627 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
4628 gimple_regimplify_operands (stmt, &gsi);
4632 /* Expand the OpenMP parallel or task directive starting at REGION. */
4634 static void
4635 expand_omp_taskreg (struct omp_region *region)
4637 basic_block entry_bb, exit_bb, new_bb;
4638 struct function *child_cfun;
4639 tree child_fn, block, t;
4640 gimple_stmt_iterator gsi;
4641 gimple entry_stmt, stmt;
4642 edge e;
4643 vec<tree, va_gc> *ws_args;
4645 entry_stmt = last_stmt (region->entry);
4646 child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
4647 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
4649 entry_bb = region->entry;
4650 exit_bb = region->exit;
4652 if (is_combined_parallel (region))
4653 ws_args = region->ws_args;
4654 else
4655 ws_args = NULL;
4657 if (child_cfun->cfg)
4659 /* Due to inlining, it may happen that we have already outlined
4660 the region, in which case all we need to do is make the
4661 sub-graph unreachable and emit the parallel call. */
4662 edge entry_succ_e, exit_succ_e;
4663 gimple_stmt_iterator gsi;
4665 entry_succ_e = single_succ_edge (entry_bb);
4667 gsi = gsi_last_bb (entry_bb);
4668 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
4669 || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
4670 gsi_remove (&gsi, true);
4672 new_bb = entry_bb;
4673 if (exit_bb)
4675 exit_succ_e = single_succ_edge (exit_bb);
4676 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
4678 remove_edge_and_dominated_blocks (entry_succ_e);
4680 else
4682 unsigned srcidx, dstidx, num;
4684 /* If the parallel region needs data sent from the parent
4685 function, then the very first statement (except possible
4686 tree profile counter updates) of the parallel body
4687 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4688 &.OMP_DATA_O is passed as an argument to the child function,
4689 we need to replace it with the argument as seen by the child
4690 function.
4692 In most cases, this will end up being the identity assignment
4693 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4694 a function call that has been inlined, the original PARM_DECL
4695 .OMP_DATA_I may have been converted into a different local
4696 variable. In which case, we need to keep the assignment. */
4697 if (gimple_omp_taskreg_data_arg (entry_stmt))
4699 basic_block entry_succ_bb = single_succ (entry_bb);
4700 gimple_stmt_iterator gsi;
4701 tree arg, narg;
4702 gimple parcopy_stmt = NULL;
4704 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
4706 gimple stmt;
4708 gcc_assert (!gsi_end_p (gsi));
4709 stmt = gsi_stmt (gsi);
4710 if (gimple_code (stmt) != GIMPLE_ASSIGN)
4711 continue;
4713 if (gimple_num_ops (stmt) == 2)
4715 tree arg = gimple_assign_rhs1 (stmt);
4717 /* We're ignore the subcode because we're
4718 effectively doing a STRIP_NOPS. */
4720 if (TREE_CODE (arg) == ADDR_EXPR
4721 && TREE_OPERAND (arg, 0)
4722 == gimple_omp_taskreg_data_arg (entry_stmt))
4724 parcopy_stmt = stmt;
4725 break;
4730 gcc_assert (parcopy_stmt != NULL);
4731 arg = DECL_ARGUMENTS (child_fn);
4733 if (!gimple_in_ssa_p (cfun))
4735 if (gimple_assign_lhs (parcopy_stmt) == arg)
4736 gsi_remove (&gsi, true);
4737 else
4739 /* ?? Is setting the subcode really necessary ?? */
4740 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg));
4741 gimple_assign_set_rhs1 (parcopy_stmt, arg);
4744 else
4746 /* If we are in ssa form, we must load the value from the default
4747 definition of the argument. That should not be defined now,
4748 since the argument is not used uninitialized. */
4749 gcc_assert (ssa_default_def (cfun, arg) == NULL);
4750 narg = make_ssa_name (arg, gimple_build_nop ());
4751 set_ssa_default_def (cfun, arg, narg);
4752 /* ?? Is setting the subcode really necessary ?? */
4753 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (narg));
4754 gimple_assign_set_rhs1 (parcopy_stmt, narg);
4755 update_stmt (parcopy_stmt);
4759 /* Declare local variables needed in CHILD_CFUN. */
4760 block = DECL_INITIAL (child_fn);
4761 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
4762 /* The gimplifier could record temporaries in parallel/task block
4763 rather than in containing function's local_decls chain,
4764 which would mean cgraph missed finalizing them. Do it now. */
4765 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
4766 if (TREE_CODE (t) == VAR_DECL
4767 && TREE_STATIC (t)
4768 && !DECL_EXTERNAL (t))
4769 varpool_finalize_decl (t);
4770 DECL_SAVED_TREE (child_fn) = NULL;
4771 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4772 gimple_set_body (child_fn, NULL);
4773 TREE_USED (block) = 1;
4775 /* Reset DECL_CONTEXT on function arguments. */
4776 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
4777 DECL_CONTEXT (t) = child_fn;
4779 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4780 so that it can be moved to the child function. */
4781 gsi = gsi_last_bb (entry_bb);
4782 stmt = gsi_stmt (gsi);
4783 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
4784 || gimple_code (stmt) == GIMPLE_OMP_TASK));
4785 gsi_remove (&gsi, true);
4786 e = split_block (entry_bb, stmt);
4787 entry_bb = e->dest;
4788 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
4790 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4791 if (exit_bb)
4793 gsi = gsi_last_bb (exit_bb);
4794 gcc_assert (!gsi_end_p (gsi)
4795 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
4796 stmt = gimple_build_return (NULL);
4797 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
4798 gsi_remove (&gsi, true);
4801 /* Move the parallel region into CHILD_CFUN. */
4803 if (gimple_in_ssa_p (cfun))
4805 init_tree_ssa (child_cfun);
4806 init_ssa_operands (child_cfun);
4807 child_cfun->gimple_df->in_ssa_p = true;
4808 block = NULL_TREE;
4810 else
4811 block = gimple_block (entry_stmt);
4813 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
4814 if (exit_bb)
4815 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
4816 /* When the OMP expansion process cannot guarantee an up-to-date
4817 loop tree arrange for the child function to fixup loops. */
4818 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
4819 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
4821 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4822 num = vec_safe_length (child_cfun->local_decls);
4823 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
4825 t = (*child_cfun->local_decls)[srcidx];
4826 if (DECL_CONTEXT (t) == cfun->decl)
4827 continue;
4828 if (srcidx != dstidx)
4829 (*child_cfun->local_decls)[dstidx] = t;
4830 dstidx++;
4832 if (dstidx != num)
4833 vec_safe_truncate (child_cfun->local_decls, dstidx);
4835 /* Inform the callgraph about the new function. */
4836 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
4837 cgraph_add_new_function (child_fn, true);
4839 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4840 fixed in a following pass. */
4841 push_cfun (child_cfun);
4842 if (optimize)
4843 optimize_omp_library_calls (entry_stmt);
4844 rebuild_cgraph_edges ();
4846 /* Some EH regions might become dead, see PR34608. If
4847 pass_cleanup_cfg isn't the first pass to happen with the
4848 new child, these dead EH edges might cause problems.
4849 Clean them up now. */
4850 if (flag_exceptions)
4852 basic_block bb;
4853 bool changed = false;
4855 FOR_EACH_BB_FN (bb, cfun)
4856 changed |= gimple_purge_dead_eh_edges (bb);
4857 if (changed)
4858 cleanup_tree_cfg ();
4860 if (gimple_in_ssa_p (cfun))
4861 update_ssa (TODO_update_ssa);
4862 pop_cfun ();
4865 /* Emit a library call to launch the children threads. */
4866 if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
4867 expand_parallel_call (region, new_bb, entry_stmt, ws_args);
4868 else
4869 expand_task_call (new_bb, entry_stmt);
4870 if (gimple_in_ssa_p (cfun))
4871 update_ssa (TODO_update_ssa_only_virtuals);
4875 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4876 of the combined collapse > 1 loop constructs, generate code like:
4877 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4878 if (cond3 is <)
4879 adj = STEP3 - 1;
4880 else
4881 adj = STEP3 + 1;
4882 count3 = (adj + N32 - N31) / STEP3;
4883 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4884 if (cond2 is <)
4885 adj = STEP2 - 1;
4886 else
4887 adj = STEP2 + 1;
4888 count2 = (adj + N22 - N21) / STEP2;
4889 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4890 if (cond1 is <)
4891 adj = STEP1 - 1;
4892 else
4893 adj = STEP1 + 1;
4894 count1 = (adj + N12 - N11) / STEP1;
4895 count = count1 * count2 * count3;
4896 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4897 count = 0;
4898 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4899 of the combined loop constructs, just initialize COUNTS array
4900 from the _looptemp_ clauses. */
4902 /* NOTE: It *could* be better to moosh all of the BBs together,
4903 creating one larger BB with all the computation and the unexpected
4904 jump at the end. I.e.
4906 bool zero3, zero2, zero1, zero;
4908 zero3 = N32 c3 N31;
4909 count3 = (N32 - N31) /[cl] STEP3;
4910 zero2 = N22 c2 N21;
4911 count2 = (N22 - N21) /[cl] STEP2;
4912 zero1 = N12 c1 N11;
4913 count1 = (N12 - N11) /[cl] STEP1;
4914 zero = zero3 || zero2 || zero1;
4915 count = count1 * count2 * count3;
4916 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4918 After all, we expect the zero=false, and thus we expect to have to
4919 evaluate all of the comparison expressions, so short-circuiting
4920 oughtn't be a win. Since the condition isn't protecting a
4921 denominator, we're not concerned about divide-by-zero, so we can
4922 fully evaluate count even if a numerator turned out to be wrong.
4924 It seems like putting this all together would create much better
4925 scheduling opportunities, and less pressure on the chip's branch
4926 predictor. */
4928 static void
4929 expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
4930 basic_block &entry_bb, tree *counts,
4931 basic_block &zero_iter_bb, int &first_zero_iter,
4932 basic_block &l2_dom_bb)
4934 tree t, type = TREE_TYPE (fd->loop.v);
4935 gimple stmt;
4936 edge e, ne;
4937 int i;
4939 /* Collapsed loops need work for expansion into SSA form. */
4940 gcc_assert (!gimple_in_ssa_p (cfun));
4942 if (gimple_omp_for_combined_into_p (fd->for_stmt)
4943 && TREE_CODE (fd->loop.n2) != INTEGER_CST)
4945 /* First two _looptemp_ clauses are for istart/iend, counts[0]
4946 isn't supposed to be handled, as the inner loop doesn't
4947 use it. */
4948 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
4949 OMP_CLAUSE__LOOPTEMP_);
4950 gcc_assert (innerc);
4951 for (i = 0; i < fd->collapse; i++)
4953 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
4954 OMP_CLAUSE__LOOPTEMP_);
4955 gcc_assert (innerc);
4956 if (i)
4957 counts[i] = OMP_CLAUSE_DECL (innerc);
4958 else
4959 counts[0] = NULL_TREE;
4961 return;
4964 for (i = 0; i < fd->collapse; i++)
4966 tree itype = TREE_TYPE (fd->loops[i].v);
4968 if (SSA_VAR_P (fd->loop.n2)
4969 && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
4970 fold_convert (itype, fd->loops[i].n1),
4971 fold_convert (itype, fd->loops[i].n2)))
4972 == NULL_TREE || !integer_onep (t)))
4974 tree n1, n2;
4975 n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1));
4976 n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE,
4977 true, GSI_SAME_STMT);
4978 n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
4979 n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE,
4980 true, GSI_SAME_STMT);
4981 stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
4982 NULL_TREE, NULL_TREE);
4983 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
4984 if (walk_tree (gimple_cond_lhs_ptr (stmt),
4985 expand_omp_regimplify_p, NULL, NULL)
4986 || walk_tree (gimple_cond_rhs_ptr (stmt),
4987 expand_omp_regimplify_p, NULL, NULL))
4989 *gsi = gsi_for_stmt (stmt);
4990 gimple_regimplify_operands (stmt, gsi);
4992 e = split_block (entry_bb, stmt);
4993 if (zero_iter_bb == NULL)
4995 first_zero_iter = i;
4996 zero_iter_bb = create_empty_bb (entry_bb);
4997 if (current_loops)
4998 add_bb_to_loop (zero_iter_bb, entry_bb->loop_father);
4999 *gsi = gsi_after_labels (zero_iter_bb);
5000 stmt = gimple_build_assign (fd->loop.n2,
5001 build_zero_cst (type));
5002 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
5003 set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb,
5004 entry_bb);
5006 ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
5007 ne->probability = REG_BR_PROB_BASE / 2000 - 1;
5008 e->flags = EDGE_TRUE_VALUE;
5009 e->probability = REG_BR_PROB_BASE - ne->probability;
5010 if (l2_dom_bb == NULL)
5011 l2_dom_bb = entry_bb;
5012 entry_bb = e->dest;
5013 *gsi = gsi_last_bb (entry_bb);
5016 if (POINTER_TYPE_P (itype))
5017 itype = signed_type_for (itype);
5018 t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
5019 ? -1 : 1));
5020 t = fold_build2 (PLUS_EXPR, itype,
5021 fold_convert (itype, fd->loops[i].step), t);
5022 t = fold_build2 (PLUS_EXPR, itype, t,
5023 fold_convert (itype, fd->loops[i].n2));
5024 t = fold_build2 (MINUS_EXPR, itype, t,
5025 fold_convert (itype, fd->loops[i].n1));
5026 /* ?? We could probably use CEIL_DIV_EXPR instead of
5027 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5028 generate the same code in the end because generically we
5029 don't know that the values involved must be negative for
5030 GT?? */
5031 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
5032 t = fold_build2 (TRUNC_DIV_EXPR, itype,
5033 fold_build1 (NEGATE_EXPR, itype, t),
5034 fold_build1 (NEGATE_EXPR, itype,
5035 fold_convert (itype,
5036 fd->loops[i].step)));
5037 else
5038 t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
5039 fold_convert (itype, fd->loops[i].step));
5040 t = fold_convert (type, t);
5041 if (TREE_CODE (t) == INTEGER_CST)
5042 counts[i] = t;
5043 else
5045 counts[i] = create_tmp_reg (type, ".count");
5046 expand_omp_build_assign (gsi, counts[i], t);
5048 if (SSA_VAR_P (fd->loop.n2))
5050 if (i == 0)
5051 t = counts[0];
5052 else
5053 t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]);
5054 expand_omp_build_assign (gsi, fd->loop.n2, t);
5060 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5061 T = V;
5062 V3 = N31 + (T % count3) * STEP3;
5063 T = T / count3;
5064 V2 = N21 + (T % count2) * STEP2;
5065 T = T / count2;
5066 V1 = N11 + T * STEP1;
5067 if this loop doesn't have an inner loop construct combined with it.
5068 If it does have an inner loop construct combined with it and the
5069 iteration count isn't known constant, store values from counts array
5070 into its _looptemp_ temporaries instead. */
5072 static void
5073 expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
5074 tree *counts, gimple inner_stmt, tree startvar)
5076 int i;
5077 if (gimple_omp_for_combined_p (fd->for_stmt))
5079 /* If fd->loop.n2 is constant, then no propagation of the counts
5080 is needed, they are constant. */
5081 if (TREE_CODE (fd->loop.n2) == INTEGER_CST)
5082 return;
5084 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
5085 ? gimple_omp_parallel_clauses (inner_stmt)
5086 : gimple_omp_for_clauses (inner_stmt);
5087 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5088 isn't supposed to be handled, as the inner loop doesn't
5089 use it. */
5090 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
5091 gcc_assert (innerc);
5092 for (i = 0; i < fd->collapse; i++)
5094 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5095 OMP_CLAUSE__LOOPTEMP_);
5096 gcc_assert (innerc);
5097 if (i)
5099 tree tem = OMP_CLAUSE_DECL (innerc);
5100 tree t = fold_convert (TREE_TYPE (tem), counts[i]);
5101 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
5102 false, GSI_CONTINUE_LINKING);
5103 gimple stmt = gimple_build_assign (tem, t);
5104 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5107 return;
5110 tree type = TREE_TYPE (fd->loop.v);
5111 tree tem = create_tmp_reg (type, ".tem");
5112 gimple stmt = gimple_build_assign (tem, startvar);
5113 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5115 for (i = fd->collapse - 1; i >= 0; i--)
5117 tree vtype = TREE_TYPE (fd->loops[i].v), itype, t;
5118 itype = vtype;
5119 if (POINTER_TYPE_P (vtype))
5120 itype = signed_type_for (vtype);
5121 if (i != 0)
5122 t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]);
5123 else
5124 t = tem;
5125 t = fold_convert (itype, t);
5126 t = fold_build2 (MULT_EXPR, itype, t,
5127 fold_convert (itype, fd->loops[i].step));
5128 if (POINTER_TYPE_P (vtype))
5129 t = fold_build_pointer_plus (fd->loops[i].n1, t);
5130 else
5131 t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
5132 t = force_gimple_operand_gsi (gsi, t,
5133 DECL_P (fd->loops[i].v)
5134 && TREE_ADDRESSABLE (fd->loops[i].v),
5135 NULL_TREE, false,
5136 GSI_CONTINUE_LINKING);
5137 stmt = gimple_build_assign (fd->loops[i].v, t);
5138 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5139 if (i != 0)
5141 t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]);
5142 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
5143 false, GSI_CONTINUE_LINKING);
5144 stmt = gimple_build_assign (tem, t);
5145 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5151 /* Helper function for expand_omp_for_*. Generate code like:
5152 L10:
5153 V3 += STEP3;
5154 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5155 L11:
5156 V3 = N31;
5157 V2 += STEP2;
5158 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5159 L12:
5160 V2 = N21;
5161 V1 += STEP1;
5162 goto BODY_BB; */
5164 static basic_block
5165 extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb,
5166 basic_block body_bb)
5168 basic_block last_bb, bb, collapse_bb = NULL;
5169 int i;
5170 gimple_stmt_iterator gsi;
5171 edge e;
5172 tree t;
5173 gimple stmt;
5175 last_bb = cont_bb;
5176 for (i = fd->collapse - 1; i >= 0; i--)
5178 tree vtype = TREE_TYPE (fd->loops[i].v);
5180 bb = create_empty_bb (last_bb);
5181 if (current_loops)
5182 add_bb_to_loop (bb, last_bb->loop_father);
5183 gsi = gsi_start_bb (bb);
5185 if (i < fd->collapse - 1)
5187 e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
5188 e->probability = REG_BR_PROB_BASE / 8;
5190 t = fd->loops[i + 1].n1;
5191 t = force_gimple_operand_gsi (&gsi, t,
5192 DECL_P (fd->loops[i + 1].v)
5193 && TREE_ADDRESSABLE (fd->loops[i
5194 + 1].v),
5195 NULL_TREE, false,
5196 GSI_CONTINUE_LINKING);
5197 stmt = gimple_build_assign (fd->loops[i + 1].v, t);
5198 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5200 else
5201 collapse_bb = bb;
5203 set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
5205 if (POINTER_TYPE_P (vtype))
5206 t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
5207 else
5208 t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step);
5209 t = force_gimple_operand_gsi (&gsi, t,
5210 DECL_P (fd->loops[i].v)
5211 && TREE_ADDRESSABLE (fd->loops[i].v),
5212 NULL_TREE, false, GSI_CONTINUE_LINKING);
5213 stmt = gimple_build_assign (fd->loops[i].v, t);
5214 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5216 if (i > 0)
5218 t = fd->loops[i].n2;
5219 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5220 false, GSI_CONTINUE_LINKING);
5221 tree v = fd->loops[i].v;
5222 if (DECL_P (v) && TREE_ADDRESSABLE (v))
5223 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
5224 false, GSI_CONTINUE_LINKING);
5225 t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
5226 stmt = gimple_build_cond_empty (t);
5227 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5228 e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
5229 e->probability = REG_BR_PROB_BASE * 7 / 8;
5231 else
5232 make_edge (bb, body_bb, EDGE_FALLTHRU);
5233 last_bb = bb;
5236 return collapse_bb;
5240 /* A subroutine of expand_omp_for. Generate code for a parallel
5241 loop with any schedule. Given parameters:
5243 for (V = N1; V cond N2; V += STEP) BODY;
5245 where COND is "<" or ">", we generate pseudocode
5247 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5248 if (more) goto L0; else goto L3;
5250 V = istart0;
5251 iend = iend0;
5253 BODY;
5254 V += STEP;
5255 if (V cond iend) goto L1; else goto L2;
5257 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5260 If this is a combined omp parallel loop, instead of the call to
5261 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5262 If this is gimple_omp_for_combined_p loop, then instead of assigning
5263 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5264 inner GIMPLE_OMP_FOR and V += STEP; and
5265 if (V cond iend) goto L1; else goto L2; are removed.
5267 For collapsed loops, given parameters:
5268 collapse(3)
5269 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5270 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5271 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5272 BODY;
5274 we generate pseudocode
5276 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5277 if (cond3 is <)
5278 adj = STEP3 - 1;
5279 else
5280 adj = STEP3 + 1;
5281 count3 = (adj + N32 - N31) / STEP3;
5282 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5283 if (cond2 is <)
5284 adj = STEP2 - 1;
5285 else
5286 adj = STEP2 + 1;
5287 count2 = (adj + N22 - N21) / STEP2;
5288 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5289 if (cond1 is <)
5290 adj = STEP1 - 1;
5291 else
5292 adj = STEP1 + 1;
5293 count1 = (adj + N12 - N11) / STEP1;
5294 count = count1 * count2 * count3;
5295 goto Z1;
5297 count = 0;
5299 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5300 if (more) goto L0; else goto L3;
5302 V = istart0;
5303 T = V;
5304 V3 = N31 + (T % count3) * STEP3;
5305 T = T / count3;
5306 V2 = N21 + (T % count2) * STEP2;
5307 T = T / count2;
5308 V1 = N11 + T * STEP1;
5309 iend = iend0;
5311 BODY;
5312 V += 1;
5313 if (V < iend) goto L10; else goto L2;
5314 L10:
5315 V3 += STEP3;
5316 if (V3 cond3 N32) goto L1; else goto L11;
5317 L11:
5318 V3 = N31;
5319 V2 += STEP2;
5320 if (V2 cond2 N22) goto L1; else goto L12;
5321 L12:
5322 V2 = N21;
5323 V1 += STEP1;
5324 goto L1;
5326 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5331 static void
5332 expand_omp_for_generic (struct omp_region *region,
5333 struct omp_for_data *fd,
5334 enum built_in_function start_fn,
5335 enum built_in_function next_fn,
5336 gimple inner_stmt)
5338 tree type, istart0, iend0, iend;
5339 tree t, vmain, vback, bias = NULL_TREE;
5340 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
5341 basic_block l2_bb = NULL, l3_bb = NULL;
5342 gimple_stmt_iterator gsi;
5343 gimple stmt;
5344 bool in_combined_parallel = is_combined_parallel (region);
5345 bool broken_loop = region->cont == NULL;
5346 edge e, ne;
5347 tree *counts = NULL;
5348 int i;
5350 gcc_assert (!broken_loop || !in_combined_parallel);
5351 gcc_assert (fd->iter_type == long_integer_type_node
5352 || !in_combined_parallel);
5354 type = TREE_TYPE (fd->loop.v);
5355 istart0 = create_tmp_var (fd->iter_type, ".istart0");
5356 iend0 = create_tmp_var (fd->iter_type, ".iend0");
5357 TREE_ADDRESSABLE (istart0) = 1;
5358 TREE_ADDRESSABLE (iend0) = 1;
5360 /* See if we need to bias by LLONG_MIN. */
5361 if (fd->iter_type == long_long_unsigned_type_node
5362 && TREE_CODE (type) == INTEGER_TYPE
5363 && !TYPE_UNSIGNED (type))
5365 tree n1, n2;
5367 if (fd->loop.cond_code == LT_EXPR)
5369 n1 = fd->loop.n1;
5370 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
5372 else
5374 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
5375 n2 = fd->loop.n1;
5377 if (TREE_CODE (n1) != INTEGER_CST
5378 || TREE_CODE (n2) != INTEGER_CST
5379 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
5380 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
5383 entry_bb = region->entry;
5384 cont_bb = region->cont;
5385 collapse_bb = NULL;
5386 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
5387 gcc_assert (broken_loop
5388 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
5389 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
5390 l1_bb = single_succ (l0_bb);
5391 if (!broken_loop)
5393 l2_bb = create_empty_bb (cont_bb);
5394 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb);
5395 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
5397 else
5398 l2_bb = NULL;
5399 l3_bb = BRANCH_EDGE (entry_bb)->dest;
5400 exit_bb = region->exit;
5402 gsi = gsi_last_bb (entry_bb);
5404 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5405 if (fd->collapse > 1)
5407 int first_zero_iter = -1;
5408 basic_block zero_iter_bb = NULL, l2_dom_bb = NULL;
5410 counts = XALLOCAVEC (tree, fd->collapse);
5411 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
5412 zero_iter_bb, first_zero_iter,
5413 l2_dom_bb);
5415 if (zero_iter_bb)
5417 /* Some counts[i] vars might be uninitialized if
5418 some loop has zero iterations. But the body shouldn't
5419 be executed in that case, so just avoid uninit warnings. */
5420 for (i = first_zero_iter; i < fd->collapse; i++)
5421 if (SSA_VAR_P (counts[i]))
5422 TREE_NO_WARNING (counts[i]) = 1;
5423 gsi_prev (&gsi);
5424 e = split_block (entry_bb, gsi_stmt (gsi));
5425 entry_bb = e->dest;
5426 make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU);
5427 gsi = gsi_last_bb (entry_bb);
5428 set_immediate_dominator (CDI_DOMINATORS, entry_bb,
5429 get_immediate_dominator (CDI_DOMINATORS,
5430 zero_iter_bb));
5433 if (in_combined_parallel)
5435 /* In a combined parallel loop, emit a call to
5436 GOMP_loop_foo_next. */
5437 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
5438 build_fold_addr_expr (istart0),
5439 build_fold_addr_expr (iend0));
5441 else
5443 tree t0, t1, t2, t3, t4;
5444 /* If this is not a combined parallel loop, emit a call to
5445 GOMP_loop_foo_start in ENTRY_BB. */
5446 t4 = build_fold_addr_expr (iend0);
5447 t3 = build_fold_addr_expr (istart0);
5448 t2 = fold_convert (fd->iter_type, fd->loop.step);
5449 t1 = fd->loop.n2;
5450 t0 = fd->loop.n1;
5451 if (gimple_omp_for_combined_into_p (fd->for_stmt))
5453 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
5454 OMP_CLAUSE__LOOPTEMP_);
5455 gcc_assert (innerc);
5456 t0 = OMP_CLAUSE_DECL (innerc);
5457 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5458 OMP_CLAUSE__LOOPTEMP_);
5459 gcc_assert (innerc);
5460 t1 = OMP_CLAUSE_DECL (innerc);
5462 if (POINTER_TYPE_P (TREE_TYPE (t0))
5463 && TYPE_PRECISION (TREE_TYPE (t0))
5464 != TYPE_PRECISION (fd->iter_type))
5466 /* Avoid casting pointers to integer of a different size. */
5467 tree itype = signed_type_for (type);
5468 t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
5469 t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
5471 else
5473 t1 = fold_convert (fd->iter_type, t1);
5474 t0 = fold_convert (fd->iter_type, t0);
5476 if (bias)
5478 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
5479 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
5481 if (fd->iter_type == long_integer_type_node)
5483 if (fd->chunk_size)
5485 t = fold_convert (fd->iter_type, fd->chunk_size);
5486 t = build_call_expr (builtin_decl_explicit (start_fn),
5487 6, t0, t1, t2, t, t3, t4);
5489 else
5490 t = build_call_expr (builtin_decl_explicit (start_fn),
5491 5, t0, t1, t2, t3, t4);
5493 else
5495 tree t5;
5496 tree c_bool_type;
5497 tree bfn_decl;
5499 /* The GOMP_loop_ull_*start functions have additional boolean
5500 argument, true for < loops and false for > loops.
5501 In Fortran, the C bool type can be different from
5502 boolean_type_node. */
5503 bfn_decl = builtin_decl_explicit (start_fn);
5504 c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl));
5505 t5 = build_int_cst (c_bool_type,
5506 fd->loop.cond_code == LT_EXPR ? 1 : 0);
5507 if (fd->chunk_size)
5509 tree bfn_decl = builtin_decl_explicit (start_fn);
5510 t = fold_convert (fd->iter_type, fd->chunk_size);
5511 t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
5513 else
5514 t = build_call_expr (builtin_decl_explicit (start_fn),
5515 6, t5, t0, t1, t2, t3, t4);
5518 if (TREE_TYPE (t) != boolean_type_node)
5519 t = fold_build2 (NE_EXPR, boolean_type_node,
5520 t, build_int_cst (TREE_TYPE (t), 0));
5521 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5522 true, GSI_SAME_STMT);
5523 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5525 /* Remove the GIMPLE_OMP_FOR statement. */
5526 gsi_remove (&gsi, true);
5528 /* Iteration setup for sequential loop goes in L0_BB. */
5529 tree startvar = fd->loop.v;
5530 tree endvar = NULL_TREE;
5532 if (gimple_omp_for_combined_p (fd->for_stmt))
5534 gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR
5535 && gimple_omp_for_kind (inner_stmt)
5536 == GF_OMP_FOR_KIND_SIMD);
5537 tree innerc = find_omp_clause (gimple_omp_for_clauses (inner_stmt),
5538 OMP_CLAUSE__LOOPTEMP_);
5539 gcc_assert (innerc);
5540 startvar = OMP_CLAUSE_DECL (innerc);
5541 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5542 OMP_CLAUSE__LOOPTEMP_);
5543 gcc_assert (innerc);
5544 endvar = OMP_CLAUSE_DECL (innerc);
5547 gsi = gsi_start_bb (l0_bb);
5548 t = istart0;
5549 if (bias)
5550 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
5551 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
5552 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
5553 t = fold_convert (TREE_TYPE (startvar), t);
5554 t = force_gimple_operand_gsi (&gsi, t,
5555 DECL_P (startvar)
5556 && TREE_ADDRESSABLE (startvar),
5557 NULL_TREE, false, GSI_CONTINUE_LINKING);
5558 stmt = gimple_build_assign (startvar, t);
5559 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5561 t = iend0;
5562 if (bias)
5563 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
5564 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
5565 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
5566 t = fold_convert (TREE_TYPE (startvar), t);
5567 iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5568 false, GSI_CONTINUE_LINKING);
5569 if (endvar)
5571 stmt = gimple_build_assign (endvar, iend);
5572 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5574 if (fd->collapse > 1)
5575 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
5577 if (!broken_loop)
5579 /* Code to control the increment and predicate for the sequential
5580 loop goes in the CONT_BB. */
5581 gsi = gsi_last_bb (cont_bb);
5582 stmt = gsi_stmt (gsi);
5583 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
5584 vmain = gimple_omp_continue_control_use (stmt);
5585 vback = gimple_omp_continue_control_def (stmt);
5587 if (!gimple_omp_for_combined_p (fd->for_stmt))
5589 if (POINTER_TYPE_P (type))
5590 t = fold_build_pointer_plus (vmain, fd->loop.step);
5591 else
5592 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
5593 t = force_gimple_operand_gsi (&gsi, t,
5594 DECL_P (vback)
5595 && TREE_ADDRESSABLE (vback),
5596 NULL_TREE, true, GSI_SAME_STMT);
5597 stmt = gimple_build_assign (vback, t);
5598 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5600 t = build2 (fd->loop.cond_code, boolean_type_node,
5601 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
5602 iend);
5603 stmt = gimple_build_cond_empty (t);
5604 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5607 /* Remove GIMPLE_OMP_CONTINUE. */
5608 gsi_remove (&gsi, true);
5610 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
5611 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, l1_bb);
5613 /* Emit code to get the next parallel iteration in L2_BB. */
5614 gsi = gsi_start_bb (l2_bb);
5616 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
5617 build_fold_addr_expr (istart0),
5618 build_fold_addr_expr (iend0));
5619 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5620 false, GSI_CONTINUE_LINKING);
5621 if (TREE_TYPE (t) != boolean_type_node)
5622 t = fold_build2 (NE_EXPR, boolean_type_node,
5623 t, build_int_cst (TREE_TYPE (t), 0));
5624 stmt = gimple_build_cond_empty (t);
5625 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5628 /* Add the loop cleanup function. */
5629 gsi = gsi_last_bb (exit_bb);
5630 if (gimple_omp_return_nowait_p (gsi_stmt (gsi)))
5631 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
5632 else if (gimple_omp_return_lhs (gsi_stmt (gsi)))
5633 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
5634 else
5635 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
5636 stmt = gimple_build_call (t, 0);
5637 if (gimple_omp_return_lhs (gsi_stmt (gsi)))
5638 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
5639 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
5640 gsi_remove (&gsi, true);
5642 /* Connect the new blocks. */
5643 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
5644 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
5646 if (!broken_loop)
5648 gimple_seq phis;
5650 e = find_edge (cont_bb, l3_bb);
5651 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
5653 phis = phi_nodes (l3_bb);
5654 for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi))
5656 gimple phi = gsi_stmt (gsi);
5657 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
5658 PHI_ARG_DEF_FROM_EDGE (phi, e));
5660 remove_edge (e);
5662 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
5663 if (current_loops)
5664 add_bb_to_loop (l2_bb, cont_bb->loop_father);
5665 e = find_edge (cont_bb, l1_bb);
5666 if (gimple_omp_for_combined_p (fd->for_stmt))
5668 remove_edge (e);
5669 e = NULL;
5671 else if (fd->collapse > 1)
5673 remove_edge (e);
5674 e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
5676 else
5677 e->flags = EDGE_TRUE_VALUE;
5678 if (e)
5680 e->probability = REG_BR_PROB_BASE * 7 / 8;
5681 find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8;
5683 else
5685 e = find_edge (cont_bb, l2_bb);
5686 e->flags = EDGE_FALLTHRU;
5688 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
5690 set_immediate_dominator (CDI_DOMINATORS, l2_bb,
5691 recompute_dominator (CDI_DOMINATORS, l2_bb));
5692 set_immediate_dominator (CDI_DOMINATORS, l3_bb,
5693 recompute_dominator (CDI_DOMINATORS, l3_bb));
5694 set_immediate_dominator (CDI_DOMINATORS, l0_bb,
5695 recompute_dominator (CDI_DOMINATORS, l0_bb));
5696 set_immediate_dominator (CDI_DOMINATORS, l1_bb,
5697 recompute_dominator (CDI_DOMINATORS, l1_bb));
5699 struct loop *outer_loop = alloc_loop ();
5700 outer_loop->header = l0_bb;
5701 outer_loop->latch = l2_bb;
5702 add_loop (outer_loop, l0_bb->loop_father);
5704 if (!gimple_omp_for_combined_p (fd->for_stmt))
5706 struct loop *loop = alloc_loop ();
5707 loop->header = l1_bb;
5708 /* The loop may have multiple latches. */
5709 add_loop (loop, outer_loop);
5715 /* A subroutine of expand_omp_for. Generate code for a parallel
5716 loop with static schedule and no specified chunk size. Given
5717 parameters:
5719 for (V = N1; V cond N2; V += STEP) BODY;
5721 where COND is "<" or ">", we generate pseudocode
5723 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5724 if (cond is <)
5725 adj = STEP - 1;
5726 else
5727 adj = STEP + 1;
5728 if ((__typeof (V)) -1 > 0 && cond is >)
5729 n = -(adj + N2 - N1) / -STEP;
5730 else
5731 n = (adj + N2 - N1) / STEP;
5732 q = n / nthreads;
5733 tt = n % nthreads;
5734 if (threadid < tt) goto L3; else goto L4;
5736 tt = 0;
5737 q = q + 1;
5739 s0 = q * threadid + tt;
5740 e0 = s0 + q;
5741 V = s0 * STEP + N1;
5742 if (s0 >= e0) goto L2; else goto L0;
5744 e = e0 * STEP + N1;
5746 BODY;
5747 V += STEP;
5748 if (V cond e) goto L1;
5752 static void
5753 expand_omp_for_static_nochunk (struct omp_region *region,
5754 struct omp_for_data *fd,
5755 gimple inner_stmt)
5757 tree n, q, s0, e0, e, t, tt, nthreads, threadid;
5758 tree type, itype, vmain, vback;
5759 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
5760 basic_block body_bb, cont_bb, collapse_bb = NULL;
5761 basic_block fin_bb;
5762 gimple_stmt_iterator gsi;
5763 gimple stmt;
5764 edge ep;
5765 enum built_in_function get_num_threads = BUILT_IN_OMP_GET_NUM_THREADS;
5766 enum built_in_function get_thread_num = BUILT_IN_OMP_GET_THREAD_NUM;
5767 bool broken_loop = region->cont == NULL;
5768 tree *counts = NULL;
5769 tree n1, n2, step;
5771 itype = type = TREE_TYPE (fd->loop.v);
5772 if (POINTER_TYPE_P (type))
5773 itype = signed_type_for (type);
5775 entry_bb = region->entry;
5776 cont_bb = region->cont;
5777 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
5778 fin_bb = BRANCH_EDGE (entry_bb)->dest;
5779 gcc_assert (broken_loop
5780 || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
5781 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
5782 body_bb = single_succ (seq_start_bb);
5783 if (!broken_loop)
5785 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
5786 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
5788 exit_bb = region->exit;
5790 /* Iteration space partitioning goes in ENTRY_BB. */
5791 gsi = gsi_last_bb (entry_bb);
5792 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5794 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
5796 get_num_threads = BUILT_IN_OMP_GET_NUM_TEAMS;
5797 get_thread_num = BUILT_IN_OMP_GET_TEAM_NUM;
5800 if (fd->collapse > 1)
5802 int first_zero_iter = -1;
5803 basic_block l2_dom_bb = NULL;
5805 counts = XALLOCAVEC (tree, fd->collapse);
5806 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
5807 fin_bb, first_zero_iter,
5808 l2_dom_bb);
5809 t = NULL_TREE;
5811 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
5812 t = integer_one_node;
5813 else
5814 t = fold_binary (fd->loop.cond_code, boolean_type_node,
5815 fold_convert (type, fd->loop.n1),
5816 fold_convert (type, fd->loop.n2));
5817 if (fd->collapse == 1
5818 && TYPE_UNSIGNED (type)
5819 && (t == NULL_TREE || !integer_onep (t)))
5821 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
5822 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
5823 true, GSI_SAME_STMT);
5824 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
5825 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
5826 true, GSI_SAME_STMT);
5827 stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
5828 NULL_TREE, NULL_TREE);
5829 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5830 if (walk_tree (gimple_cond_lhs_ptr (stmt),
5831 expand_omp_regimplify_p, NULL, NULL)
5832 || walk_tree (gimple_cond_rhs_ptr (stmt),
5833 expand_omp_regimplify_p, NULL, NULL))
5835 gsi = gsi_for_stmt (stmt);
5836 gimple_regimplify_operands (stmt, &gsi);
5838 ep = split_block (entry_bb, stmt);
5839 ep->flags = EDGE_TRUE_VALUE;
5840 entry_bb = ep->dest;
5841 ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
5842 ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
5843 ep->probability = REG_BR_PROB_BASE / 2000 - 1;
5844 if (gimple_in_ssa_p (cfun))
5846 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
5847 for (gsi = gsi_start_phis (fin_bb);
5848 !gsi_end_p (gsi); gsi_next (&gsi))
5850 gimple phi = gsi_stmt (gsi);
5851 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
5852 ep, UNKNOWN_LOCATION);
5855 gsi = gsi_last_bb (entry_bb);
5858 t = build_call_expr (builtin_decl_explicit (get_num_threads), 0);
5859 t = fold_convert (itype, t);
5860 nthreads = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5861 true, GSI_SAME_STMT);
5863 t = build_call_expr (builtin_decl_explicit (get_thread_num), 0);
5864 t = fold_convert (itype, t);
5865 threadid = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5866 true, GSI_SAME_STMT);
5868 n1 = fd->loop.n1;
5869 n2 = fd->loop.n2;
5870 step = fd->loop.step;
5871 if (gimple_omp_for_combined_into_p (fd->for_stmt))
5873 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
5874 OMP_CLAUSE__LOOPTEMP_);
5875 gcc_assert (innerc);
5876 n1 = OMP_CLAUSE_DECL (innerc);
5877 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5878 OMP_CLAUSE__LOOPTEMP_);
5879 gcc_assert (innerc);
5880 n2 = OMP_CLAUSE_DECL (innerc);
5882 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
5883 true, NULL_TREE, true, GSI_SAME_STMT);
5884 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
5885 true, NULL_TREE, true, GSI_SAME_STMT);
5886 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
5887 true, NULL_TREE, true, GSI_SAME_STMT);
5889 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
5890 t = fold_build2 (PLUS_EXPR, itype, step, t);
5891 t = fold_build2 (PLUS_EXPR, itype, t, n2);
5892 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
5893 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
5894 t = fold_build2 (TRUNC_DIV_EXPR, itype,
5895 fold_build1 (NEGATE_EXPR, itype, t),
5896 fold_build1 (NEGATE_EXPR, itype, step));
5897 else
5898 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
5899 t = fold_convert (itype, t);
5900 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5902 q = create_tmp_reg (itype, "q");
5903 t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads);
5904 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
5905 gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT);
5907 tt = create_tmp_reg (itype, "tt");
5908 t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads);
5909 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
5910 gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT);
5912 t = build2 (LT_EXPR, boolean_type_node, threadid, tt);
5913 stmt = gimple_build_cond_empty (t);
5914 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5916 second_bb = split_block (entry_bb, stmt)->dest;
5917 gsi = gsi_last_bb (second_bb);
5918 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5920 gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)),
5921 GSI_SAME_STMT);
5922 stmt = gimple_build_assign_with_ops (PLUS_EXPR, q, q,
5923 build_int_cst (itype, 1));
5924 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5926 third_bb = split_block (second_bb, stmt)->dest;
5927 gsi = gsi_last_bb (third_bb);
5928 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5930 t = build2 (MULT_EXPR, itype, q, threadid);
5931 t = build2 (PLUS_EXPR, itype, t, tt);
5932 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5934 t = fold_build2 (PLUS_EXPR, itype, s0, q);
5935 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5937 t = build2 (GE_EXPR, boolean_type_node, s0, e0);
5938 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5940 /* Remove the GIMPLE_OMP_FOR statement. */
5941 gsi_remove (&gsi, true);
5943 /* Setup code for sequential iteration goes in SEQ_START_BB. */
5944 gsi = gsi_start_bb (seq_start_bb);
5946 tree startvar = fd->loop.v;
5947 tree endvar = NULL_TREE;
5949 if (gimple_omp_for_combined_p (fd->for_stmt))
5951 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
5952 ? gimple_omp_parallel_clauses (inner_stmt)
5953 : gimple_omp_for_clauses (inner_stmt);
5954 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
5955 gcc_assert (innerc);
5956 startvar = OMP_CLAUSE_DECL (innerc);
5957 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5958 OMP_CLAUSE__LOOPTEMP_);
5959 gcc_assert (innerc);
5960 endvar = OMP_CLAUSE_DECL (innerc);
5962 t = fold_convert (itype, s0);
5963 t = fold_build2 (MULT_EXPR, itype, t, step);
5964 if (POINTER_TYPE_P (type))
5965 t = fold_build_pointer_plus (n1, t);
5966 else
5967 t = fold_build2 (PLUS_EXPR, type, t, n1);
5968 t = fold_convert (TREE_TYPE (startvar), t);
5969 t = force_gimple_operand_gsi (&gsi, t,
5970 DECL_P (startvar)
5971 && TREE_ADDRESSABLE (startvar),
5972 NULL_TREE, false, GSI_CONTINUE_LINKING);
5973 stmt = gimple_build_assign (startvar, t);
5974 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5976 t = fold_convert (itype, e0);
5977 t = fold_build2 (MULT_EXPR, itype, t, step);
5978 if (POINTER_TYPE_P (type))
5979 t = fold_build_pointer_plus (n1, t);
5980 else
5981 t = fold_build2 (PLUS_EXPR, type, t, n1);
5982 t = fold_convert (TREE_TYPE (startvar), t);
5983 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5984 false, GSI_CONTINUE_LINKING);
5985 if (endvar)
5987 stmt = gimple_build_assign (endvar, e);
5988 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5990 if (fd->collapse > 1)
5991 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
5993 if (!broken_loop)
5995 /* The code controlling the sequential loop replaces the
5996 GIMPLE_OMP_CONTINUE. */
5997 gsi = gsi_last_bb (cont_bb);
5998 stmt = gsi_stmt (gsi);
5999 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
6000 vmain = gimple_omp_continue_control_use (stmt);
6001 vback = gimple_omp_continue_control_def (stmt);
6003 if (!gimple_omp_for_combined_p (fd->for_stmt))
6005 if (POINTER_TYPE_P (type))
6006 t = fold_build_pointer_plus (vmain, step);
6007 else
6008 t = fold_build2 (PLUS_EXPR, type, vmain, step);
6009 t = force_gimple_operand_gsi (&gsi, t,
6010 DECL_P (vback)
6011 && TREE_ADDRESSABLE (vback),
6012 NULL_TREE, true, GSI_SAME_STMT);
6013 stmt = gimple_build_assign (vback, t);
6014 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
6016 t = build2 (fd->loop.cond_code, boolean_type_node,
6017 DECL_P (vback) && TREE_ADDRESSABLE (vback)
6018 ? t : vback, e);
6019 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6022 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6023 gsi_remove (&gsi, true);
6025 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6026 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
6029 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6030 gsi = gsi_last_bb (exit_bb);
6031 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
6033 t = gimple_omp_return_lhs (gsi_stmt (gsi));
6034 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
6036 gsi_remove (&gsi, true);
6038 /* Connect all the blocks. */
6039 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
6040 ep->probability = REG_BR_PROB_BASE / 4 * 3;
6041 ep = find_edge (entry_bb, second_bb);
6042 ep->flags = EDGE_TRUE_VALUE;
6043 ep->probability = REG_BR_PROB_BASE / 4;
6044 find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
6045 find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
6047 if (!broken_loop)
6049 ep = find_edge (cont_bb, body_bb);
6050 if (gimple_omp_for_combined_p (fd->for_stmt))
6052 remove_edge (ep);
6053 ep = NULL;
6055 else if (fd->collapse > 1)
6057 remove_edge (ep);
6058 ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6060 else
6061 ep->flags = EDGE_TRUE_VALUE;
6062 find_edge (cont_bb, fin_bb)->flags
6063 = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
6066 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
6067 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
6068 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb);
6070 set_immediate_dominator (CDI_DOMINATORS, body_bb,
6071 recompute_dominator (CDI_DOMINATORS, body_bb));
6072 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
6073 recompute_dominator (CDI_DOMINATORS, fin_bb));
6075 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
6077 struct loop *loop = alloc_loop ();
6078 loop->header = body_bb;
6079 if (collapse_bb == NULL)
6080 loop->latch = cont_bb;
6081 add_loop (loop, body_bb->loop_father);
6086 /* A subroutine of expand_omp_for. Generate code for a parallel
6087 loop with static schedule and a specified chunk size. Given
6088 parameters:
6090 for (V = N1; V cond N2; V += STEP) BODY;
6092 where COND is "<" or ">", we generate pseudocode
6094 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6095 if (cond is <)
6096 adj = STEP - 1;
6097 else
6098 adj = STEP + 1;
6099 if ((__typeof (V)) -1 > 0 && cond is >)
6100 n = -(adj + N2 - N1) / -STEP;
6101 else
6102 n = (adj + N2 - N1) / STEP;
6103 trip = 0;
6104 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6105 here so that V is defined
6106 if the loop is not entered
6108 s0 = (trip * nthreads + threadid) * CHUNK;
6109 e0 = min(s0 + CHUNK, n);
6110 if (s0 < n) goto L1; else goto L4;
6112 V = s0 * STEP + N1;
6113 e = e0 * STEP + N1;
6115 BODY;
6116 V += STEP;
6117 if (V cond e) goto L2; else goto L3;
6119 trip += 1;
6120 goto L0;
6124 static void
6125 expand_omp_for_static_chunk (struct omp_region *region,
6126 struct omp_for_data *fd, gimple inner_stmt)
6128 tree n, s0, e0, e, t;
6129 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
6130 tree type, itype, v_main, v_back, v_extra;
6131 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
6132 basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb;
6133 gimple_stmt_iterator si;
6134 gimple stmt;
6135 edge se;
6136 enum built_in_function get_num_threads = BUILT_IN_OMP_GET_NUM_THREADS;
6137 enum built_in_function get_thread_num = BUILT_IN_OMP_GET_THREAD_NUM;
6138 bool broken_loop = region->cont == NULL;
6139 tree *counts = NULL;
6140 tree n1, n2, step;
6142 itype = type = TREE_TYPE (fd->loop.v);
6143 if (POINTER_TYPE_P (type))
6144 itype = signed_type_for (type);
6146 entry_bb = region->entry;
6147 se = split_block (entry_bb, last_stmt (entry_bb));
6148 entry_bb = se->src;
6149 iter_part_bb = se->dest;
6150 cont_bb = region->cont;
6151 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
6152 fin_bb = BRANCH_EDGE (iter_part_bb)->dest;
6153 gcc_assert (broken_loop
6154 || fin_bb == FALLTHRU_EDGE (cont_bb)->dest);
6155 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
6156 body_bb = single_succ (seq_start_bb);
6157 if (!broken_loop)
6159 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
6160 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6161 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
6163 exit_bb = region->exit;
6165 /* Trip and adjustment setup goes in ENTRY_BB. */
6166 si = gsi_last_bb (entry_bb);
6167 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_FOR);
6169 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
6171 get_num_threads = BUILT_IN_OMP_GET_NUM_TEAMS;
6172 get_thread_num = BUILT_IN_OMP_GET_TEAM_NUM;
6175 if (fd->collapse > 1)
6177 int first_zero_iter = -1;
6178 basic_block l2_dom_bb = NULL;
6180 counts = XALLOCAVEC (tree, fd->collapse);
6181 expand_omp_for_init_counts (fd, &si, entry_bb, counts,
6182 fin_bb, first_zero_iter,
6183 l2_dom_bb);
6184 t = NULL_TREE;
6186 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
6187 t = integer_one_node;
6188 else
6189 t = fold_binary (fd->loop.cond_code, boolean_type_node,
6190 fold_convert (type, fd->loop.n1),
6191 fold_convert (type, fd->loop.n2));
6192 if (fd->collapse == 1
6193 && TYPE_UNSIGNED (type)
6194 && (t == NULL_TREE || !integer_onep (t)))
6196 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
6197 n1 = force_gimple_operand_gsi (&si, n1, true, NULL_TREE,
6198 true, GSI_SAME_STMT);
6199 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
6200 n2 = force_gimple_operand_gsi (&si, n2, true, NULL_TREE,
6201 true, GSI_SAME_STMT);
6202 stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
6203 NULL_TREE, NULL_TREE);
6204 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
6205 if (walk_tree (gimple_cond_lhs_ptr (stmt),
6206 expand_omp_regimplify_p, NULL, NULL)
6207 || walk_tree (gimple_cond_rhs_ptr (stmt),
6208 expand_omp_regimplify_p, NULL, NULL))
6210 si = gsi_for_stmt (stmt);
6211 gimple_regimplify_operands (stmt, &si);
6213 se = split_block (entry_bb, stmt);
6214 se->flags = EDGE_TRUE_VALUE;
6215 entry_bb = se->dest;
6216 se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
6217 se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
6218 se->probability = REG_BR_PROB_BASE / 2000 - 1;
6219 if (gimple_in_ssa_p (cfun))
6221 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
6222 for (si = gsi_start_phis (fin_bb);
6223 !gsi_end_p (si); gsi_next (&si))
6225 gimple phi = gsi_stmt (si);
6226 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
6227 se, UNKNOWN_LOCATION);
6230 si = gsi_last_bb (entry_bb);
6233 t = build_call_expr (builtin_decl_explicit (get_num_threads), 0);
6234 t = fold_convert (itype, t);
6235 nthreads = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6236 true, GSI_SAME_STMT);
6238 t = build_call_expr (builtin_decl_explicit (get_thread_num), 0);
6239 t = fold_convert (itype, t);
6240 threadid = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6241 true, GSI_SAME_STMT);
6243 n1 = fd->loop.n1;
6244 n2 = fd->loop.n2;
6245 step = fd->loop.step;
6246 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6248 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6249 OMP_CLAUSE__LOOPTEMP_);
6250 gcc_assert (innerc);
6251 n1 = OMP_CLAUSE_DECL (innerc);
6252 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6253 OMP_CLAUSE__LOOPTEMP_);
6254 gcc_assert (innerc);
6255 n2 = OMP_CLAUSE_DECL (innerc);
6257 n1 = force_gimple_operand_gsi (&si, fold_convert (type, n1),
6258 true, NULL_TREE, true, GSI_SAME_STMT);
6259 n2 = force_gimple_operand_gsi (&si, fold_convert (itype, n2),
6260 true, NULL_TREE, true, GSI_SAME_STMT);
6261 step = force_gimple_operand_gsi (&si, fold_convert (itype, step),
6262 true, NULL_TREE, true, GSI_SAME_STMT);
6263 fd->chunk_size
6264 = force_gimple_operand_gsi (&si, fold_convert (itype, fd->chunk_size),
6265 true, NULL_TREE, true, GSI_SAME_STMT);
6267 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
6268 t = fold_build2 (PLUS_EXPR, itype, step, t);
6269 t = fold_build2 (PLUS_EXPR, itype, t, n2);
6270 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
6271 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
6272 t = fold_build2 (TRUNC_DIV_EXPR, itype,
6273 fold_build1 (NEGATE_EXPR, itype, t),
6274 fold_build1 (NEGATE_EXPR, itype, step));
6275 else
6276 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
6277 t = fold_convert (itype, t);
6278 n = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6279 true, GSI_SAME_STMT);
6281 trip_var = create_tmp_reg (itype, ".trip");
6282 if (gimple_in_ssa_p (cfun))
6284 trip_init = make_ssa_name (trip_var, NULL);
6285 trip_main = make_ssa_name (trip_var, NULL);
6286 trip_back = make_ssa_name (trip_var, NULL);
6288 else
6290 trip_init = trip_var;
6291 trip_main = trip_var;
6292 trip_back = trip_var;
6295 stmt = gimple_build_assign (trip_init, build_int_cst (itype, 0));
6296 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
6298 t = fold_build2 (MULT_EXPR, itype, threadid, fd->chunk_size);
6299 t = fold_build2 (MULT_EXPR, itype, t, step);
6300 if (POINTER_TYPE_P (type))
6301 t = fold_build_pointer_plus (n1, t);
6302 else
6303 t = fold_build2 (PLUS_EXPR, type, t, n1);
6304 v_extra = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6305 true, GSI_SAME_STMT);
6307 /* Remove the GIMPLE_OMP_FOR. */
6308 gsi_remove (&si, true);
6310 /* Iteration space partitioning goes in ITER_PART_BB. */
6311 si = gsi_last_bb (iter_part_bb);
6313 t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads);
6314 t = fold_build2 (PLUS_EXPR, itype, t, threadid);
6315 t = fold_build2 (MULT_EXPR, itype, t, fd->chunk_size);
6316 s0 = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6317 false, GSI_CONTINUE_LINKING);
6319 t = fold_build2 (PLUS_EXPR, itype, s0, fd->chunk_size);
6320 t = fold_build2 (MIN_EXPR, itype, t, n);
6321 e0 = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6322 false, GSI_CONTINUE_LINKING);
6324 t = build2 (LT_EXPR, boolean_type_node, s0, n);
6325 gsi_insert_after (&si, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING);
6327 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6328 si = gsi_start_bb (seq_start_bb);
6330 tree startvar = fd->loop.v;
6331 tree endvar = NULL_TREE;
6333 if (gimple_omp_for_combined_p (fd->for_stmt))
6335 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
6336 ? gimple_omp_parallel_clauses (inner_stmt)
6337 : gimple_omp_for_clauses (inner_stmt);
6338 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
6339 gcc_assert (innerc);
6340 startvar = OMP_CLAUSE_DECL (innerc);
6341 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6342 OMP_CLAUSE__LOOPTEMP_);
6343 gcc_assert (innerc);
6344 endvar = OMP_CLAUSE_DECL (innerc);
6347 t = fold_convert (itype, s0);
6348 t = fold_build2 (MULT_EXPR, itype, t, step);
6349 if (POINTER_TYPE_P (type))
6350 t = fold_build_pointer_plus (n1, t);
6351 else
6352 t = fold_build2 (PLUS_EXPR, type, t, n1);
6353 t = fold_convert (TREE_TYPE (startvar), t);
6354 t = force_gimple_operand_gsi (&si, t,
6355 DECL_P (startvar)
6356 && TREE_ADDRESSABLE (startvar),
6357 NULL_TREE, false, GSI_CONTINUE_LINKING);
6358 stmt = gimple_build_assign (startvar, t);
6359 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
6361 t = fold_convert (itype, e0);
6362 t = fold_build2 (MULT_EXPR, itype, t, step);
6363 if (POINTER_TYPE_P (type))
6364 t = fold_build_pointer_plus (n1, t);
6365 else
6366 t = fold_build2 (PLUS_EXPR, type, t, n1);
6367 t = fold_convert (TREE_TYPE (startvar), t);
6368 e = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6369 false, GSI_CONTINUE_LINKING);
6370 if (endvar)
6372 stmt = gimple_build_assign (endvar, e);
6373 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
6375 if (fd->collapse > 1)
6376 expand_omp_for_init_vars (fd, &si, counts, inner_stmt, startvar);
6378 if (!broken_loop)
6380 /* The code controlling the sequential loop goes in CONT_BB,
6381 replacing the GIMPLE_OMP_CONTINUE. */
6382 si = gsi_last_bb (cont_bb);
6383 stmt = gsi_stmt (si);
6384 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
6385 v_main = gimple_omp_continue_control_use (stmt);
6386 v_back = gimple_omp_continue_control_def (stmt);
6388 if (!gimple_omp_for_combined_p (fd->for_stmt))
6390 if (POINTER_TYPE_P (type))
6391 t = fold_build_pointer_plus (v_main, step);
6392 else
6393 t = fold_build2 (PLUS_EXPR, type, v_main, step);
6394 if (DECL_P (v_back) && TREE_ADDRESSABLE (v_back))
6395 t = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6396 true, GSI_SAME_STMT);
6397 stmt = gimple_build_assign (v_back, t);
6398 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
6400 t = build2 (fd->loop.cond_code, boolean_type_node,
6401 DECL_P (v_back) && TREE_ADDRESSABLE (v_back)
6402 ? t : v_back, e);
6403 gsi_insert_before (&si, gimple_build_cond_empty (t), GSI_SAME_STMT);
6406 /* Remove GIMPLE_OMP_CONTINUE. */
6407 gsi_remove (&si, true);
6409 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6410 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
6412 /* Trip update code goes into TRIP_UPDATE_BB. */
6413 si = gsi_start_bb (trip_update_bb);
6415 t = build_int_cst (itype, 1);
6416 t = build2 (PLUS_EXPR, itype, trip_main, t);
6417 stmt = gimple_build_assign (trip_back, t);
6418 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
6421 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6422 si = gsi_last_bb (exit_bb);
6423 if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
6425 t = gimple_omp_return_lhs (gsi_stmt (si));
6426 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT);
6428 gsi_remove (&si, true);
6430 /* Connect the new blocks. */
6431 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
6432 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
6434 if (!broken_loop)
6436 se = find_edge (cont_bb, body_bb);
6437 if (gimple_omp_for_combined_p (fd->for_stmt))
6439 remove_edge (se);
6440 se = NULL;
6442 else if (fd->collapse > 1)
6444 remove_edge (se);
6445 se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6447 else
6448 se->flags = EDGE_TRUE_VALUE;
6449 find_edge (cont_bb, trip_update_bb)->flags
6450 = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
6452 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb);
6455 if (gimple_in_ssa_p (cfun))
6457 gimple_stmt_iterator psi;
6458 gimple phi;
6459 edge re, ene;
6460 edge_var_map_vector *head;
6461 edge_var_map *vm;
6462 size_t i;
6464 gcc_assert (fd->collapse == 1 && !broken_loop);
6466 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6467 remove arguments of the phi nodes in fin_bb. We need to create
6468 appropriate phi nodes in iter_part_bb instead. */
6469 se = single_pred_edge (fin_bb);
6470 re = single_succ_edge (trip_update_bb);
6471 head = redirect_edge_var_map_vector (re);
6472 ene = single_succ_edge (entry_bb);
6474 psi = gsi_start_phis (fin_bb);
6475 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
6476 gsi_next (&psi), ++i)
6478 gimple nphi;
6479 source_location locus;
6481 phi = gsi_stmt (psi);
6482 t = gimple_phi_result (phi);
6483 gcc_assert (t == redirect_edge_var_map_result (vm));
6484 nphi = create_phi_node (t, iter_part_bb);
6486 t = PHI_ARG_DEF_FROM_EDGE (phi, se);
6487 locus = gimple_phi_arg_location_from_edge (phi, se);
6489 /* A special case -- fd->loop.v is not yet computed in
6490 iter_part_bb, we need to use v_extra instead. */
6491 if (t == fd->loop.v)
6492 t = v_extra;
6493 add_phi_arg (nphi, t, ene, locus);
6494 locus = redirect_edge_var_map_location (vm);
6495 add_phi_arg (nphi, redirect_edge_var_map_def (vm), re, locus);
6497 gcc_assert (!gsi_end_p (psi) && i == head->length ());
6498 redirect_edge_var_map_clear (re);
6499 while (1)
6501 psi = gsi_start_phis (fin_bb);
6502 if (gsi_end_p (psi))
6503 break;
6504 remove_phi_node (&psi, false);
6507 /* Make phi node for trip. */
6508 phi = create_phi_node (trip_main, iter_part_bb);
6509 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb),
6510 UNKNOWN_LOCATION);
6511 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb),
6512 UNKNOWN_LOCATION);
6515 if (!broken_loop)
6516 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
6517 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
6518 recompute_dominator (CDI_DOMINATORS, iter_part_bb));
6519 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
6520 recompute_dominator (CDI_DOMINATORS, fin_bb));
6521 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
6522 recompute_dominator (CDI_DOMINATORS, seq_start_bb));
6523 set_immediate_dominator (CDI_DOMINATORS, body_bb,
6524 recompute_dominator (CDI_DOMINATORS, body_bb));
6526 if (!broken_loop)
6528 struct loop *trip_loop = alloc_loop ();
6529 trip_loop->header = iter_part_bb;
6530 trip_loop->latch = trip_update_bb;
6531 add_loop (trip_loop, iter_part_bb->loop_father);
6533 if (!gimple_omp_for_combined_p (fd->for_stmt))
6535 struct loop *loop = alloc_loop ();
6536 loop->header = body_bb;
6537 if (collapse_bb == NULL)
6538 loop->latch = cont_bb;
6539 add_loop (loop, trip_loop);
6545 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6546 loop. Given parameters:
6548 for (V = N1; V cond N2; V += STEP) BODY;
6550 where COND is "<" or ">", we generate pseudocode
6552 V = N1;
6553 goto L1;
6555 BODY;
6556 V += STEP;
6558 if (V cond N2) goto L0; else goto L2;
6561 For collapsed loops, given parameters:
6562 collapse(3)
6563 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6564 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6565 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6566 BODY;
6568 we generate pseudocode
6570 if (cond3 is <)
6571 adj = STEP3 - 1;
6572 else
6573 adj = STEP3 + 1;
6574 count3 = (adj + N32 - N31) / STEP3;
6575 if (cond2 is <)
6576 adj = STEP2 - 1;
6577 else
6578 adj = STEP2 + 1;
6579 count2 = (adj + N22 - N21) / STEP2;
6580 if (cond1 is <)
6581 adj = STEP1 - 1;
6582 else
6583 adj = STEP1 + 1;
6584 count1 = (adj + N12 - N11) / STEP1;
6585 count = count1 * count2 * count3;
6586 V = 0;
6587 V1 = N11;
6588 V2 = N21;
6589 V3 = N31;
6590 goto L1;
6592 BODY;
6593 V += 1;
6594 V3 += STEP3;
6595 V2 += (V3 cond3 N32) ? 0 : STEP2;
6596 V3 = (V3 cond3 N32) ? V3 : N31;
6597 V1 += (V2 cond2 N22) ? 0 : STEP1;
6598 V2 = (V2 cond2 N22) ? V2 : N21;
6600 if (V < count) goto L0; else goto L2;
6605 static void
6606 expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
6608 tree type, t;
6609 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb;
6610 gimple_stmt_iterator gsi;
6611 gimple stmt;
6612 bool broken_loop = region->cont == NULL;
6613 edge e, ne;
6614 tree *counts = NULL;
6615 int i;
6616 tree safelen = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6617 OMP_CLAUSE_SAFELEN);
6618 tree simduid = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6619 OMP_CLAUSE__SIMDUID_);
6620 tree n1, n2;
6622 type = TREE_TYPE (fd->loop.v);
6623 entry_bb = region->entry;
6624 cont_bb = region->cont;
6625 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6626 gcc_assert (broken_loop
6627 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
6628 l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
6629 if (!broken_loop)
6631 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
6632 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6633 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
6634 l2_bb = BRANCH_EDGE (entry_bb)->dest;
6636 else
6638 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
6639 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
6640 l2_bb = single_succ (l1_bb);
6642 exit_bb = region->exit;
6643 l2_dom_bb = NULL;
6645 gsi = gsi_last_bb (entry_bb);
6647 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6648 /* Not needed in SSA form right now. */
6649 gcc_assert (!gimple_in_ssa_p (cfun));
6650 if (fd->collapse > 1)
6652 int first_zero_iter = -1;
6653 basic_block zero_iter_bb = l2_bb;
6655 counts = XALLOCAVEC (tree, fd->collapse);
6656 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6657 zero_iter_bb, first_zero_iter,
6658 l2_dom_bb);
6660 if (l2_dom_bb == NULL)
6661 l2_dom_bb = l1_bb;
6663 n1 = fd->loop.n1;
6664 n2 = fd->loop.n2;
6665 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6667 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6668 OMP_CLAUSE__LOOPTEMP_);
6669 gcc_assert (innerc);
6670 n1 = OMP_CLAUSE_DECL (innerc);
6671 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6672 OMP_CLAUSE__LOOPTEMP_);
6673 gcc_assert (innerc);
6674 n2 = OMP_CLAUSE_DECL (innerc);
6675 expand_omp_build_assign (&gsi, fd->loop.v,
6676 fold_convert (type, n1));
6677 if (fd->collapse > 1)
6679 gsi_prev (&gsi);
6680 expand_omp_for_init_vars (fd, &gsi, counts, NULL, n1);
6681 gsi_next (&gsi);
6684 else
6686 expand_omp_build_assign (&gsi, fd->loop.v,
6687 fold_convert (type, fd->loop.n1));
6688 if (fd->collapse > 1)
6689 for (i = 0; i < fd->collapse; i++)
6691 tree itype = TREE_TYPE (fd->loops[i].v);
6692 if (POINTER_TYPE_P (itype))
6693 itype = signed_type_for (itype);
6694 t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1);
6695 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6699 /* Remove the GIMPLE_OMP_FOR statement. */
6700 gsi_remove (&gsi, true);
6702 if (!broken_loop)
6704 /* Code to control the increment goes in the CONT_BB. */
6705 gsi = gsi_last_bb (cont_bb);
6706 stmt = gsi_stmt (gsi);
6707 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
6709 if (POINTER_TYPE_P (type))
6710 t = fold_build_pointer_plus (fd->loop.v, fd->loop.step);
6711 else
6712 t = fold_build2 (PLUS_EXPR, type, fd->loop.v, fd->loop.step);
6713 expand_omp_build_assign (&gsi, fd->loop.v, t);
6715 if (fd->collapse > 1)
6717 i = fd->collapse - 1;
6718 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
6720 t = fold_convert (sizetype, fd->loops[i].step);
6721 t = fold_build_pointer_plus (fd->loops[i].v, t);
6723 else
6725 t = fold_convert (TREE_TYPE (fd->loops[i].v),
6726 fd->loops[i].step);
6727 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
6728 fd->loops[i].v, t);
6730 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6732 for (i = fd->collapse - 1; i > 0; i--)
6734 tree itype = TREE_TYPE (fd->loops[i].v);
6735 tree itype2 = TREE_TYPE (fd->loops[i - 1].v);
6736 if (POINTER_TYPE_P (itype2))
6737 itype2 = signed_type_for (itype2);
6738 t = build3 (COND_EXPR, itype2,
6739 build2 (fd->loops[i].cond_code, boolean_type_node,
6740 fd->loops[i].v,
6741 fold_convert (itype, fd->loops[i].n2)),
6742 build_int_cst (itype2, 0),
6743 fold_convert (itype2, fd->loops[i - 1].step));
6744 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i - 1].v)))
6745 t = fold_build_pointer_plus (fd->loops[i - 1].v, t);
6746 else
6747 t = fold_build2 (PLUS_EXPR, itype2, fd->loops[i - 1].v, t);
6748 expand_omp_build_assign (&gsi, fd->loops[i - 1].v, t);
6750 t = build3 (COND_EXPR, itype,
6751 build2 (fd->loops[i].cond_code, boolean_type_node,
6752 fd->loops[i].v,
6753 fold_convert (itype, fd->loops[i].n2)),
6754 fd->loops[i].v,
6755 fold_convert (itype, fd->loops[i].n1));
6756 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6760 /* Remove GIMPLE_OMP_CONTINUE. */
6761 gsi_remove (&gsi, true);
6764 /* Emit the condition in L1_BB. */
6765 gsi = gsi_start_bb (l1_bb);
6767 t = fold_convert (type, n2);
6768 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6769 false, GSI_CONTINUE_LINKING);
6770 t = build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t);
6771 stmt = gimple_build_cond_empty (t);
6772 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6773 if (walk_tree (gimple_cond_lhs_ptr (stmt), expand_omp_regimplify_p,
6774 NULL, NULL)
6775 || walk_tree (gimple_cond_rhs_ptr (stmt), expand_omp_regimplify_p,
6776 NULL, NULL))
6778 gsi = gsi_for_stmt (stmt);
6779 gimple_regimplify_operands (stmt, &gsi);
6782 /* Remove GIMPLE_OMP_RETURN. */
6783 gsi = gsi_last_bb (exit_bb);
6784 gsi_remove (&gsi, true);
6786 /* Connect the new blocks. */
6787 remove_edge (FALLTHRU_EDGE (entry_bb));
6789 if (!broken_loop)
6791 remove_edge (BRANCH_EDGE (entry_bb));
6792 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
6794 e = BRANCH_EDGE (l1_bb);
6795 ne = FALLTHRU_EDGE (l1_bb);
6796 e->flags = EDGE_TRUE_VALUE;
6798 else
6800 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
6802 ne = single_succ_edge (l1_bb);
6803 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
6806 ne->flags = EDGE_FALSE_VALUE;
6807 e->probability = REG_BR_PROB_BASE * 7 / 8;
6808 ne->probability = REG_BR_PROB_BASE / 8;
6810 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
6811 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
6812 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
6814 if (!broken_loop)
6816 struct loop *loop = alloc_loop ();
6817 loop->header = l1_bb;
6818 loop->latch = cont_bb;
6819 add_loop (loop, l1_bb->loop_father);
6820 if (safelen == NULL_TREE)
6821 loop->safelen = INT_MAX;
6822 else
6824 safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen);
6825 if (!tree_fits_uhwi_p (safelen)
6826 || tree_to_uhwi (safelen) > INT_MAX)
6827 loop->safelen = INT_MAX;
6828 else
6829 loop->safelen = tree_to_uhwi (safelen);
6830 if (loop->safelen == 1)
6831 loop->safelen = 0;
6833 if (simduid)
6835 loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
6836 cfun->has_simduid_loops = true;
6838 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6839 the loop. */
6840 if ((flag_tree_loop_vectorize
6841 || (!global_options_set.x_flag_tree_loop_vectorize
6842 && !global_options_set.x_flag_tree_vectorize))
6843 && flag_tree_loop_optimize
6844 && loop->safelen > 1)
6846 loop->force_vect = true;
6847 cfun->has_force_vect_loops = true;
6853 /* Expand the OpenMP loop defined by REGION. */
6855 static void
6856 expand_omp_for (struct omp_region *region, gimple inner_stmt)
6858 struct omp_for_data fd;
6859 struct omp_for_data_loop *loops;
6861 loops
6862 = (struct omp_for_data_loop *)
6863 alloca (gimple_omp_for_collapse (last_stmt (region->entry))
6864 * sizeof (struct omp_for_data_loop));
6865 extract_omp_for_data (last_stmt (region->entry), &fd, loops);
6866 region->sched_kind = fd.sched_kind;
6868 gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
6869 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
6870 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
6871 if (region->cont)
6873 gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
6874 BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
6875 FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
6877 else
6878 /* If there isn't a continue then this is a degerate case where
6879 the introduction of abnormal edges during lowering will prevent
6880 original loops from being detected. Fix that up. */
6881 loops_state_set (LOOPS_NEED_FIXUP);
6883 if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_KIND_SIMD)
6884 expand_omp_simd (region, &fd);
6885 else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
6886 && !fd.have_ordered)
6888 if (fd.chunk_size == NULL)
6889 expand_omp_for_static_nochunk (region, &fd, inner_stmt);
6890 else
6891 expand_omp_for_static_chunk (region, &fd, inner_stmt);
6893 else
6895 int fn_index, start_ix, next_ix;
6897 gcc_assert (gimple_omp_for_kind (fd.for_stmt)
6898 == GF_OMP_FOR_KIND_FOR);
6899 if (fd.chunk_size == NULL
6900 && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
6901 fd.chunk_size = integer_zero_node;
6902 gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
6903 fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
6904 ? 3 : fd.sched_kind;
6905 fn_index += fd.have_ordered * 4;
6906 start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
6907 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
6908 if (fd.iter_type == long_long_unsigned_type_node)
6910 start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6911 - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
6912 next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6913 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
6915 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
6916 (enum built_in_function) next_ix, inner_stmt);
6919 if (gimple_in_ssa_p (cfun))
6920 update_ssa (TODO_update_ssa_only_virtuals);
6924 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
6926 v = GOMP_sections_start (n);
6928 switch (v)
6930 case 0:
6931 goto L2;
6932 case 1:
6933 section 1;
6934 goto L1;
6935 case 2:
6937 case n:
6939 default:
6940 abort ();
6943 v = GOMP_sections_next ();
6944 goto L0;
6946 reduction;
6948 If this is a combined parallel sections, replace the call to
6949 GOMP_sections_start with call to GOMP_sections_next. */
6951 static void
6952 expand_omp_sections (struct omp_region *region)
6954 tree t, u, vin = NULL, vmain, vnext, l2;
6955 unsigned len;
6956 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
6957 gimple_stmt_iterator si, switch_si;
6958 gimple sections_stmt, stmt, cont;
6959 edge_iterator ei;
6960 edge e;
6961 struct omp_region *inner;
6962 unsigned i, casei;
6963 bool exit_reachable = region->cont != NULL;
6965 gcc_assert (region->exit != NULL);
6966 entry_bb = region->entry;
6967 l0_bb = single_succ (entry_bb);
6968 l1_bb = region->cont;
6969 l2_bb = region->exit;
6970 if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb)
6971 l2 = gimple_block_label (l2_bb);
6972 else
6974 /* This can happen if there are reductions. */
6975 len = EDGE_COUNT (l0_bb->succs);
6976 gcc_assert (len > 0);
6977 e = EDGE_SUCC (l0_bb, len - 1);
6978 si = gsi_last_bb (e->dest);
6979 l2 = NULL_TREE;
6980 if (gsi_end_p (si)
6981 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
6982 l2 = gimple_block_label (e->dest);
6983 else
6984 FOR_EACH_EDGE (e, ei, l0_bb->succs)
6986 si = gsi_last_bb (e->dest);
6987 if (gsi_end_p (si)
6988 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
6990 l2 = gimple_block_label (e->dest);
6991 break;
6995 if (exit_reachable)
6996 default_bb = create_empty_bb (l1_bb->prev_bb);
6997 else
6998 default_bb = create_empty_bb (l0_bb);
7000 /* We will build a switch() with enough cases for all the
7001 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7002 and a default case to abort if something goes wrong. */
7003 len = EDGE_COUNT (l0_bb->succs);
7005 /* Use vec::quick_push on label_vec throughout, since we know the size
7006 in advance. */
7007 auto_vec<tree> label_vec (len);
7009 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7010 GIMPLE_OMP_SECTIONS statement. */
7011 si = gsi_last_bb (entry_bb);
7012 sections_stmt = gsi_stmt (si);
7013 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
7014 vin = gimple_omp_sections_control (sections_stmt);
7015 if (!is_combined_parallel (region))
7017 /* If we are not inside a combined parallel+sections region,
7018 call GOMP_sections_start. */
7019 t = build_int_cst (unsigned_type_node, len - 1);
7020 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
7021 stmt = gimple_build_call (u, 1, t);
7023 else
7025 /* Otherwise, call GOMP_sections_next. */
7026 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
7027 stmt = gimple_build_call (u, 0);
7029 gimple_call_set_lhs (stmt, vin);
7030 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
7031 gsi_remove (&si, true);
7033 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7034 L0_BB. */
7035 switch_si = gsi_last_bb (l0_bb);
7036 gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH);
7037 if (exit_reachable)
7039 cont = last_stmt (l1_bb);
7040 gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE);
7041 vmain = gimple_omp_continue_control_use (cont);
7042 vnext = gimple_omp_continue_control_def (cont);
7044 else
7046 vmain = vin;
7047 vnext = NULL_TREE;
7050 t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2);
7051 label_vec.quick_push (t);
7052 i = 1;
7054 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7055 for (inner = region->inner, casei = 1;
7056 inner;
7057 inner = inner->next, i++, casei++)
7059 basic_block s_entry_bb, s_exit_bb;
7061 /* Skip optional reduction region. */
7062 if (inner->type == GIMPLE_OMP_ATOMIC_LOAD)
7064 --i;
7065 --casei;
7066 continue;
7069 s_entry_bb = inner->entry;
7070 s_exit_bb = inner->exit;
7072 t = gimple_block_label (s_entry_bb);
7073 u = build_int_cst (unsigned_type_node, casei);
7074 u = build_case_label (u, NULL, t);
7075 label_vec.quick_push (u);
7077 si = gsi_last_bb (s_entry_bb);
7078 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION);
7079 gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si)));
7080 gsi_remove (&si, true);
7081 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
7083 if (s_exit_bb == NULL)
7084 continue;
7086 si = gsi_last_bb (s_exit_bb);
7087 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
7088 gsi_remove (&si, true);
7090 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
7093 /* Error handling code goes in DEFAULT_BB. */
7094 t = gimple_block_label (default_bb);
7095 u = build_case_label (NULL, NULL, t);
7096 make_edge (l0_bb, default_bb, 0);
7097 if (current_loops)
7098 add_bb_to_loop (default_bb, current_loops->tree_root);
7100 stmt = gimple_build_switch (vmain, u, label_vec);
7101 gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT);
7102 gsi_remove (&switch_si, true);
7104 si = gsi_start_bb (default_bb);
7105 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
7106 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
7108 if (exit_reachable)
7110 tree bfn_decl;
7112 /* Code to get the next section goes in L1_BB. */
7113 si = gsi_last_bb (l1_bb);
7114 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
7116 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
7117 stmt = gimple_build_call (bfn_decl, 0);
7118 gimple_call_set_lhs (stmt, vnext);
7119 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
7120 gsi_remove (&si, true);
7122 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
7125 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7126 si = gsi_last_bb (l2_bb);
7127 if (gimple_omp_return_nowait_p (gsi_stmt (si)))
7128 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT);
7129 else if (gimple_omp_return_lhs (gsi_stmt (si)))
7130 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL);
7131 else
7132 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END);
7133 stmt = gimple_build_call (t, 0);
7134 if (gimple_omp_return_lhs (gsi_stmt (si)))
7135 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si)));
7136 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
7137 gsi_remove (&si, true);
7139 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
7143 /* Expand code for an OpenMP single directive. We've already expanded
7144 much of the code, here we simply place the GOMP_barrier call. */
7146 static void
7147 expand_omp_single (struct omp_region *region)
7149 basic_block entry_bb, exit_bb;
7150 gimple_stmt_iterator si;
7152 entry_bb = region->entry;
7153 exit_bb = region->exit;
7155 si = gsi_last_bb (entry_bb);
7156 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE);
7157 gsi_remove (&si, true);
7158 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7160 si = gsi_last_bb (exit_bb);
7161 if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
7163 tree t = gimple_omp_return_lhs (gsi_stmt (si));
7164 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT);
7166 gsi_remove (&si, true);
7167 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
7171 /* Generic expansion for OpenMP synchronization directives: master,
7172 ordered and critical. All we need to do here is remove the entry
7173 and exit markers for REGION. */
7175 static void
7176 expand_omp_synch (struct omp_region *region)
7178 basic_block entry_bb, exit_bb;
7179 gimple_stmt_iterator si;
7181 entry_bb = region->entry;
7182 exit_bb = region->exit;
7184 si = gsi_last_bb (entry_bb);
7185 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
7186 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
7187 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
7188 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
7189 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
7190 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS);
7191 gsi_remove (&si, true);
7192 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7194 if (exit_bb)
7196 si = gsi_last_bb (exit_bb);
7197 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
7198 gsi_remove (&si, true);
7199 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
7203 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7204 operation as a normal volatile load. */
7206 static bool
7207 expand_omp_atomic_load (basic_block load_bb, tree addr,
7208 tree loaded_val, int index)
7210 enum built_in_function tmpbase;
7211 gimple_stmt_iterator gsi;
7212 basic_block store_bb;
7213 location_t loc;
7214 gimple stmt;
7215 tree decl, call, type, itype;
7217 gsi = gsi_last_bb (load_bb);
7218 stmt = gsi_stmt (gsi);
7219 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
7220 loc = gimple_location (stmt);
7222 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7223 is smaller than word size, then expand_atomic_load assumes that the load
7224 is atomic. We could avoid the builtin entirely in this case. */
7226 tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
7227 decl = builtin_decl_explicit (tmpbase);
7228 if (decl == NULL_TREE)
7229 return false;
7231 type = TREE_TYPE (loaded_val);
7232 itype = TREE_TYPE (TREE_TYPE (decl));
7234 call = build_call_expr_loc (loc, decl, 2, addr,
7235 build_int_cst (NULL,
7236 gimple_omp_atomic_seq_cst_p (stmt)
7237 ? MEMMODEL_SEQ_CST
7238 : MEMMODEL_RELAXED));
7239 if (!useless_type_conversion_p (type, itype))
7240 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
7241 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
7243 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
7244 gsi_remove (&gsi, true);
7246 store_bb = single_succ (load_bb);
7247 gsi = gsi_last_bb (store_bb);
7248 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
7249 gsi_remove (&gsi, true);
7251 if (gimple_in_ssa_p (cfun))
7252 update_ssa (TODO_update_ssa_no_phi);
7254 return true;
7257 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7258 operation as a normal volatile store. */
7260 static bool
7261 expand_omp_atomic_store (basic_block load_bb, tree addr,
7262 tree loaded_val, tree stored_val, int index)
7264 enum built_in_function tmpbase;
7265 gimple_stmt_iterator gsi;
7266 basic_block store_bb = single_succ (load_bb);
7267 location_t loc;
7268 gimple stmt;
7269 tree decl, call, type, itype;
7270 enum machine_mode imode;
7271 bool exchange;
7273 gsi = gsi_last_bb (load_bb);
7274 stmt = gsi_stmt (gsi);
7275 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
7277 /* If the load value is needed, then this isn't a store but an exchange. */
7278 exchange = gimple_omp_atomic_need_value_p (stmt);
7280 gsi = gsi_last_bb (store_bb);
7281 stmt = gsi_stmt (gsi);
7282 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE);
7283 loc = gimple_location (stmt);
7285 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7286 is smaller than word size, then expand_atomic_store assumes that the store
7287 is atomic. We could avoid the builtin entirely in this case. */
7289 tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N);
7290 tmpbase = (enum built_in_function) ((int) tmpbase + index + 1);
7291 decl = builtin_decl_explicit (tmpbase);
7292 if (decl == NULL_TREE)
7293 return false;
7295 type = TREE_TYPE (stored_val);
7297 /* Dig out the type of the function's second argument. */
7298 itype = TREE_TYPE (decl);
7299 itype = TYPE_ARG_TYPES (itype);
7300 itype = TREE_CHAIN (itype);
7301 itype = TREE_VALUE (itype);
7302 imode = TYPE_MODE (itype);
7304 if (exchange && !can_atomic_exchange_p (imode, true))
7305 return false;
7307 if (!useless_type_conversion_p (itype, type))
7308 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
7309 call = build_call_expr_loc (loc, decl, 3, addr, stored_val,
7310 build_int_cst (NULL,
7311 gimple_omp_atomic_seq_cst_p (stmt)
7312 ? MEMMODEL_SEQ_CST
7313 : MEMMODEL_RELAXED));
7314 if (exchange)
7316 if (!useless_type_conversion_p (type, itype))
7317 call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
7318 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
7321 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
7322 gsi_remove (&gsi, true);
7324 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7325 gsi = gsi_last_bb (load_bb);
7326 gsi_remove (&gsi, true);
7328 if (gimple_in_ssa_p (cfun))
7329 update_ssa (TODO_update_ssa_no_phi);
7331 return true;
7334 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7335 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7336 size of the data type, and thus usable to find the index of the builtin
7337 decl. Returns false if the expression is not of the proper form. */
7339 static bool
7340 expand_omp_atomic_fetch_op (basic_block load_bb,
7341 tree addr, tree loaded_val,
7342 tree stored_val, int index)
7344 enum built_in_function oldbase, newbase, tmpbase;
7345 tree decl, itype, call;
7346 tree lhs, rhs;
7347 basic_block store_bb = single_succ (load_bb);
7348 gimple_stmt_iterator gsi;
7349 gimple stmt;
7350 location_t loc;
7351 enum tree_code code;
7352 bool need_old, need_new;
7353 enum machine_mode imode;
7354 bool seq_cst;
7356 /* We expect to find the following sequences:
7358 load_bb:
7359 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7361 store_bb:
7362 val = tmp OP something; (or: something OP tmp)
7363 GIMPLE_OMP_STORE (val)
7365 ???FIXME: Allow a more flexible sequence.
7366 Perhaps use data flow to pick the statements.
7370 gsi = gsi_after_labels (store_bb);
7371 stmt = gsi_stmt (gsi);
7372 loc = gimple_location (stmt);
7373 if (!is_gimple_assign (stmt))
7374 return false;
7375 gsi_next (&gsi);
7376 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
7377 return false;
7378 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
7379 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb));
7380 seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb));
7381 gcc_checking_assert (!need_old || !need_new);
7383 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
7384 return false;
7386 /* Check for one of the supported fetch-op operations. */
7387 code = gimple_assign_rhs_code (stmt);
7388 switch (code)
7390 case PLUS_EXPR:
7391 case POINTER_PLUS_EXPR:
7392 oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N;
7393 newbase = BUILT_IN_ATOMIC_ADD_FETCH_N;
7394 break;
7395 case MINUS_EXPR:
7396 oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N;
7397 newbase = BUILT_IN_ATOMIC_SUB_FETCH_N;
7398 break;
7399 case BIT_AND_EXPR:
7400 oldbase = BUILT_IN_ATOMIC_FETCH_AND_N;
7401 newbase = BUILT_IN_ATOMIC_AND_FETCH_N;
7402 break;
7403 case BIT_IOR_EXPR:
7404 oldbase = BUILT_IN_ATOMIC_FETCH_OR_N;
7405 newbase = BUILT_IN_ATOMIC_OR_FETCH_N;
7406 break;
7407 case BIT_XOR_EXPR:
7408 oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N;
7409 newbase = BUILT_IN_ATOMIC_XOR_FETCH_N;
7410 break;
7411 default:
7412 return false;
7415 /* Make sure the expression is of the proper form. */
7416 if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
7417 rhs = gimple_assign_rhs2 (stmt);
7418 else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
7419 && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
7420 rhs = gimple_assign_rhs1 (stmt);
7421 else
7422 return false;
7424 tmpbase = ((enum built_in_function)
7425 ((need_new ? newbase : oldbase) + index + 1));
7426 decl = builtin_decl_explicit (tmpbase);
7427 if (decl == NULL_TREE)
7428 return false;
7429 itype = TREE_TYPE (TREE_TYPE (decl));
7430 imode = TYPE_MODE (itype);
7432 /* We could test all of the various optabs involved, but the fact of the
7433 matter is that (with the exception of i486 vs i586 and xadd) all targets
7434 that support any atomic operaton optab also implements compare-and-swap.
7435 Let optabs.c take care of expanding any compare-and-swap loop. */
7436 if (!can_compare_and_swap_p (imode, true))
7437 return false;
7439 gsi = gsi_last_bb (load_bb);
7440 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
7442 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7443 It only requires that the operation happen atomically. Thus we can
7444 use the RELAXED memory model. */
7445 call = build_call_expr_loc (loc, decl, 3, addr,
7446 fold_convert_loc (loc, itype, rhs),
7447 build_int_cst (NULL,
7448 seq_cst ? MEMMODEL_SEQ_CST
7449 : MEMMODEL_RELAXED));
7451 if (need_old || need_new)
7453 lhs = need_old ? loaded_val : stored_val;
7454 call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
7455 call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call);
7457 else
7458 call = fold_convert_loc (loc, void_type_node, call);
7459 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
7460 gsi_remove (&gsi, true);
7462 gsi = gsi_last_bb (store_bb);
7463 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
7464 gsi_remove (&gsi, true);
7465 gsi = gsi_last_bb (store_bb);
7466 gsi_remove (&gsi, true);
7468 if (gimple_in_ssa_p (cfun))
7469 update_ssa (TODO_update_ssa_no_phi);
7471 return true;
7474 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7476 oldval = *addr;
7477 repeat:
7478 newval = rhs; // with oldval replacing *addr in rhs
7479 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7480 if (oldval != newval)
7481 goto repeat;
7483 INDEX is log2 of the size of the data type, and thus usable to find the
7484 index of the builtin decl. */
7486 static bool
7487 expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
7488 tree addr, tree loaded_val, tree stored_val,
7489 int index)
7491 tree loadedi, storedi, initial, new_storedi, old_vali;
7492 tree type, itype, cmpxchg, iaddr;
7493 gimple_stmt_iterator si;
7494 basic_block loop_header = single_succ (load_bb);
7495 gimple phi, stmt;
7496 edge e;
7497 enum built_in_function fncode;
7499 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7500 order to use the RELAXED memory model effectively. */
7501 fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7502 + index + 1);
7503 cmpxchg = builtin_decl_explicit (fncode);
7504 if (cmpxchg == NULL_TREE)
7505 return false;
7506 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
7507 itype = TREE_TYPE (TREE_TYPE (cmpxchg));
7509 if (!can_compare_and_swap_p (TYPE_MODE (itype), true))
7510 return false;
7512 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7513 si = gsi_last_bb (load_bb);
7514 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
7516 /* For floating-point values, we'll need to view-convert them to integers
7517 so that we can perform the atomic compare and swap. Simplify the
7518 following code by always setting up the "i"ntegral variables. */
7519 if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
7521 tree iaddr_val;
7523 iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode,
7524 true), NULL);
7525 iaddr_val
7526 = force_gimple_operand_gsi (&si,
7527 fold_convert (TREE_TYPE (iaddr), addr),
7528 false, NULL_TREE, true, GSI_SAME_STMT);
7529 stmt = gimple_build_assign (iaddr, iaddr_val);
7530 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7531 loadedi = create_tmp_var (itype, NULL);
7532 if (gimple_in_ssa_p (cfun))
7533 loadedi = make_ssa_name (loadedi, NULL);
7535 else
7537 iaddr = addr;
7538 loadedi = loaded_val;
7541 fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
7542 tree loaddecl = builtin_decl_explicit (fncode);
7543 if (loaddecl)
7544 initial
7545 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr)),
7546 build_call_expr (loaddecl, 2, iaddr,
7547 build_int_cst (NULL_TREE,
7548 MEMMODEL_RELAXED)));
7549 else
7550 initial = build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)), iaddr,
7551 build_int_cst (TREE_TYPE (iaddr), 0));
7553 initial
7554 = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true,
7555 GSI_SAME_STMT);
7557 /* Move the value to the LOADEDI temporary. */
7558 if (gimple_in_ssa_p (cfun))
7560 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
7561 phi = create_phi_node (loadedi, loop_header);
7562 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
7563 initial);
7565 else
7566 gsi_insert_before (&si,
7567 gimple_build_assign (loadedi, initial),
7568 GSI_SAME_STMT);
7569 if (loadedi != loaded_val)
7571 gimple_stmt_iterator gsi2;
7572 tree x;
7574 x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
7575 gsi2 = gsi_start_bb (loop_header);
7576 if (gimple_in_ssa_p (cfun))
7578 gimple stmt;
7579 x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
7580 true, GSI_SAME_STMT);
7581 stmt = gimple_build_assign (loaded_val, x);
7582 gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
7584 else
7586 x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
7587 force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
7588 true, GSI_SAME_STMT);
7591 gsi_remove (&si, true);
7593 si = gsi_last_bb (store_bb);
7594 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
7596 if (iaddr == addr)
7597 storedi = stored_val;
7598 else
7599 storedi =
7600 force_gimple_operand_gsi (&si,
7601 build1 (VIEW_CONVERT_EXPR, itype,
7602 stored_val), true, NULL_TREE, true,
7603 GSI_SAME_STMT);
7605 /* Build the compare&swap statement. */
7606 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
7607 new_storedi = force_gimple_operand_gsi (&si,
7608 fold_convert (TREE_TYPE (loadedi),
7609 new_storedi),
7610 true, NULL_TREE,
7611 true, GSI_SAME_STMT);
7613 if (gimple_in_ssa_p (cfun))
7614 old_vali = loadedi;
7615 else
7617 old_vali = create_tmp_var (TREE_TYPE (loadedi), NULL);
7618 stmt = gimple_build_assign (old_vali, loadedi);
7619 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7621 stmt = gimple_build_assign (loadedi, new_storedi);
7622 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7625 /* Note that we always perform the comparison as an integer, even for
7626 floating point. This allows the atomic operation to properly
7627 succeed even with NaNs and -0.0. */
7628 stmt = gimple_build_cond_empty
7629 (build2 (NE_EXPR, boolean_type_node,
7630 new_storedi, old_vali));
7631 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7633 /* Update cfg. */
7634 e = single_succ_edge (store_bb);
7635 e->flags &= ~EDGE_FALLTHRU;
7636 e->flags |= EDGE_FALSE_VALUE;
7638 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
7640 /* Copy the new value to loadedi (we already did that before the condition
7641 if we are not in SSA). */
7642 if (gimple_in_ssa_p (cfun))
7644 phi = gimple_seq_first_stmt (phi_nodes (loop_header));
7645 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
7648 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7649 gsi_remove (&si, true);
7651 struct loop *loop = alloc_loop ();
7652 loop->header = loop_header;
7653 loop->latch = store_bb;
7654 add_loop (loop, loop_header->loop_father);
7656 if (gimple_in_ssa_p (cfun))
7657 update_ssa (TODO_update_ssa_no_phi);
7659 return true;
7662 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7664 GOMP_atomic_start ();
7665 *addr = rhs;
7666 GOMP_atomic_end ();
7668 The result is not globally atomic, but works so long as all parallel
7669 references are within #pragma omp atomic directives. According to
7670 responses received from omp@openmp.org, appears to be within spec.
7671 Which makes sense, since that's how several other compilers handle
7672 this situation as well.
7673 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7674 expanding. STORED_VAL is the operand of the matching
7675 GIMPLE_OMP_ATOMIC_STORE.
7677 We replace
7678 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7679 loaded_val = *addr;
7681 and replace
7682 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7683 *addr = stored_val;
7686 static bool
7687 expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
7688 tree addr, tree loaded_val, tree stored_val)
7690 gimple_stmt_iterator si;
7691 gimple stmt;
7692 tree t;
7694 si = gsi_last_bb (load_bb);
7695 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
7697 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
7698 t = build_call_expr (t, 0);
7699 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
7701 stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr));
7702 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7703 gsi_remove (&si, true);
7705 si = gsi_last_bb (store_bb);
7706 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
7708 stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)),
7709 stored_val);
7710 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7712 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
7713 t = build_call_expr (t, 0);
7714 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
7715 gsi_remove (&si, true);
7717 if (gimple_in_ssa_p (cfun))
7718 update_ssa (TODO_update_ssa_no_phi);
7719 return true;
7722 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7723 using expand_omp_atomic_fetch_op. If it failed, we try to
7724 call expand_omp_atomic_pipeline, and if it fails too, the
7725 ultimate fallback is wrapping the operation in a mutex
7726 (expand_omp_atomic_mutex). REGION is the atomic region built
7727 by build_omp_regions_1(). */
7729 static void
7730 expand_omp_atomic (struct omp_region *region)
7732 basic_block load_bb = region->entry, store_bb = region->exit;
7733 gimple load = last_stmt (load_bb), store = last_stmt (store_bb);
7734 tree loaded_val = gimple_omp_atomic_load_lhs (load);
7735 tree addr = gimple_omp_atomic_load_rhs (load);
7736 tree stored_val = gimple_omp_atomic_store_val (store);
7737 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
7738 HOST_WIDE_INT index;
7740 /* Make sure the type is one of the supported sizes. */
7741 index = tree_to_uhwi (TYPE_SIZE_UNIT (type));
7742 index = exact_log2 (index);
7743 if (index >= 0 && index <= 4)
7745 unsigned int align = TYPE_ALIGN_UNIT (type);
7747 /* __sync builtins require strict data alignment. */
7748 if (exact_log2 (align) >= index)
7750 /* Atomic load. */
7751 if (loaded_val == stored_val
7752 && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
7753 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
7754 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
7755 && expand_omp_atomic_load (load_bb, addr, loaded_val, index))
7756 return;
7758 /* Atomic store. */
7759 if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
7760 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
7761 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
7762 && store_bb == single_succ (load_bb)
7763 && first_stmt (store_bb) == store
7764 && expand_omp_atomic_store (load_bb, addr, loaded_val,
7765 stored_val, index))
7766 return;
7768 /* When possible, use specialized atomic update functions. */
7769 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
7770 && store_bb == single_succ (load_bb)
7771 && expand_omp_atomic_fetch_op (load_bb, addr,
7772 loaded_val, stored_val, index))
7773 return;
7775 /* If we don't have specialized __sync builtins, try and implement
7776 as a compare and swap loop. */
7777 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
7778 loaded_val, stored_val, index))
7779 return;
7783 /* The ultimate fallback is wrapping the operation in a mutex. */
7784 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
7788 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7790 static void
7791 expand_omp_target (struct omp_region *region)
7793 basic_block entry_bb, exit_bb, new_bb;
7794 struct function *child_cfun = NULL;
7795 tree child_fn = NULL_TREE, block, t;
7796 gimple_stmt_iterator gsi;
7797 gimple entry_stmt, stmt;
7798 edge e;
7800 entry_stmt = last_stmt (region->entry);
7801 new_bb = region->entry;
7802 int kind = gimple_omp_target_kind (entry_stmt);
7803 if (kind == GF_OMP_TARGET_KIND_REGION)
7805 child_fn = gimple_omp_target_child_fn (entry_stmt);
7806 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
7809 entry_bb = region->entry;
7810 exit_bb = region->exit;
7812 if (kind == GF_OMP_TARGET_KIND_REGION)
7814 unsigned srcidx, dstidx, num;
7816 /* If the target region needs data sent from the parent
7817 function, then the very first statement (except possible
7818 tree profile counter updates) of the parallel body
7819 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7820 &.OMP_DATA_O is passed as an argument to the child function,
7821 we need to replace it with the argument as seen by the child
7822 function.
7824 In most cases, this will end up being the identity assignment
7825 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7826 a function call that has been inlined, the original PARM_DECL
7827 .OMP_DATA_I may have been converted into a different local
7828 variable. In which case, we need to keep the assignment. */
7829 if (gimple_omp_target_data_arg (entry_stmt))
7831 basic_block entry_succ_bb = single_succ (entry_bb);
7832 gimple_stmt_iterator gsi;
7833 tree arg;
7834 gimple tgtcopy_stmt = NULL;
7835 tree sender
7836 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt), 0);
7838 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
7840 gcc_assert (!gsi_end_p (gsi));
7841 stmt = gsi_stmt (gsi);
7842 if (gimple_code (stmt) != GIMPLE_ASSIGN)
7843 continue;
7845 if (gimple_num_ops (stmt) == 2)
7847 tree arg = gimple_assign_rhs1 (stmt);
7849 /* We're ignoring the subcode because we're
7850 effectively doing a STRIP_NOPS. */
7852 if (TREE_CODE (arg) == ADDR_EXPR
7853 && TREE_OPERAND (arg, 0) == sender)
7855 tgtcopy_stmt = stmt;
7856 break;
7861 gcc_assert (tgtcopy_stmt != NULL);
7862 arg = DECL_ARGUMENTS (child_fn);
7864 gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg);
7865 gsi_remove (&gsi, true);
7868 /* Declare local variables needed in CHILD_CFUN. */
7869 block = DECL_INITIAL (child_fn);
7870 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
7871 /* The gimplifier could record temporaries in target block
7872 rather than in containing function's local_decls chain,
7873 which would mean cgraph missed finalizing them. Do it now. */
7874 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
7875 if (TREE_CODE (t) == VAR_DECL
7876 && TREE_STATIC (t)
7877 && !DECL_EXTERNAL (t))
7878 varpool_finalize_decl (t);
7879 DECL_SAVED_TREE (child_fn) = NULL;
7880 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7881 gimple_set_body (child_fn, NULL);
7882 TREE_USED (block) = 1;
7884 /* Reset DECL_CONTEXT on function arguments. */
7885 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
7886 DECL_CONTEXT (t) = child_fn;
7888 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7889 so that it can be moved to the child function. */
7890 gsi = gsi_last_bb (entry_bb);
7891 stmt = gsi_stmt (gsi);
7892 gcc_assert (stmt && gimple_code (stmt) == GIMPLE_OMP_TARGET
7893 && gimple_omp_target_kind (stmt)
7894 == GF_OMP_TARGET_KIND_REGION);
7895 gsi_remove (&gsi, true);
7896 e = split_block (entry_bb, stmt);
7897 entry_bb = e->dest;
7898 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7900 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7901 if (exit_bb)
7903 gsi = gsi_last_bb (exit_bb);
7904 gcc_assert (!gsi_end_p (gsi)
7905 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
7906 stmt = gimple_build_return (NULL);
7907 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
7908 gsi_remove (&gsi, true);
7911 /* Move the target region into CHILD_CFUN. */
7913 block = gimple_block (entry_stmt);
7915 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
7916 if (exit_bb)
7917 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
7918 /* When the OMP expansion process cannot guarantee an up-to-date
7919 loop tree arrange for the child function to fixup loops. */
7920 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
7921 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
7923 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7924 num = vec_safe_length (child_cfun->local_decls);
7925 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
7927 t = (*child_cfun->local_decls)[srcidx];
7928 if (DECL_CONTEXT (t) == cfun->decl)
7929 continue;
7930 if (srcidx != dstidx)
7931 (*child_cfun->local_decls)[dstidx] = t;
7932 dstidx++;
7934 if (dstidx != num)
7935 vec_safe_truncate (child_cfun->local_decls, dstidx);
7937 /* Inform the callgraph about the new function. */
7938 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
7939 cgraph_add_new_function (child_fn, true);
7941 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7942 fixed in a following pass. */
7943 push_cfun (child_cfun);
7944 rebuild_cgraph_edges ();
7946 /* Some EH regions might become dead, see PR34608. If
7947 pass_cleanup_cfg isn't the first pass to happen with the
7948 new child, these dead EH edges might cause problems.
7949 Clean them up now. */
7950 if (flag_exceptions)
7952 basic_block bb;
7953 bool changed = false;
7955 FOR_EACH_BB_FN (bb, cfun)
7956 changed |= gimple_purge_dead_eh_edges (bb);
7957 if (changed)
7958 cleanup_tree_cfg ();
7960 pop_cfun ();
7963 /* Emit a library call to launch the target region, or do data
7964 transfers. */
7965 tree t1, t2, t3, t4, device, cond, c, clauses;
7966 enum built_in_function start_ix;
7967 location_t clause_loc;
7969 clauses = gimple_omp_target_clauses (entry_stmt);
7971 if (kind == GF_OMP_TARGET_KIND_REGION)
7972 start_ix = BUILT_IN_GOMP_TARGET;
7973 else if (kind == GF_OMP_TARGET_KIND_DATA)
7974 start_ix = BUILT_IN_GOMP_TARGET_DATA;
7975 else
7976 start_ix = BUILT_IN_GOMP_TARGET_UPDATE;
7978 /* By default, the value of DEVICE is -1 (let runtime library choose)
7979 and there is no conditional. */
7980 cond = NULL_TREE;
7981 device = build_int_cst (integer_type_node, -1);
7983 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
7984 if (c)
7985 cond = OMP_CLAUSE_IF_EXPR (c);
7987 c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE);
7988 if (c)
7990 device = OMP_CLAUSE_DEVICE_ID (c);
7991 clause_loc = OMP_CLAUSE_LOCATION (c);
7993 else
7994 clause_loc = gimple_location (entry_stmt);
7996 /* Ensure 'device' is of the correct type. */
7997 device = fold_convert_loc (clause_loc, integer_type_node, device);
7999 /* If we found the clause 'if (cond)', build
8000 (cond ? device : -2). */
8001 if (cond)
8003 cond = gimple_boolify (cond);
8005 basic_block cond_bb, then_bb, else_bb;
8006 edge e;
8007 tree tmp_var;
8009 tmp_var = create_tmp_var (TREE_TYPE (device), NULL);
8010 if (kind != GF_OMP_TARGET_KIND_REGION)
8012 gsi = gsi_last_bb (new_bb);
8013 gsi_prev (&gsi);
8014 e = split_block (new_bb, gsi_stmt (gsi));
8016 else
8017 e = split_block (new_bb, NULL);
8018 cond_bb = e->src;
8019 new_bb = e->dest;
8020 remove_edge (e);
8022 then_bb = create_empty_bb (cond_bb);
8023 else_bb = create_empty_bb (then_bb);
8024 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
8025 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
8027 stmt = gimple_build_cond_empty (cond);
8028 gsi = gsi_last_bb (cond_bb);
8029 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
8031 gsi = gsi_start_bb (then_bb);
8032 stmt = gimple_build_assign (tmp_var, device);
8033 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
8035 gsi = gsi_start_bb (else_bb);
8036 stmt = gimple_build_assign (tmp_var,
8037 build_int_cst (integer_type_node, -2));
8038 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
8040 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
8041 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
8042 if (current_loops)
8044 add_bb_to_loop (then_bb, cond_bb->loop_father);
8045 add_bb_to_loop (else_bb, cond_bb->loop_father);
8047 make_edge (then_bb, new_bb, EDGE_FALLTHRU);
8048 make_edge (else_bb, new_bb, EDGE_FALLTHRU);
8050 device = tmp_var;
8053 gsi = gsi_last_bb (new_bb);
8054 t = gimple_omp_target_data_arg (entry_stmt);
8055 if (t == NULL)
8057 t1 = size_zero_node;
8058 t2 = build_zero_cst (ptr_type_node);
8059 t3 = t2;
8060 t4 = t2;
8062 else
8064 t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1))));
8065 t1 = size_binop (PLUS_EXPR, t1, size_int (1));
8066 t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0));
8067 t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1));
8068 t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2));
8071 gimple g;
8072 /* FIXME: This will be address of
8073 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8074 symbol, as soon as the linker plugin is able to create it for us. */
8075 tree openmp_target = build_zero_cst (ptr_type_node);
8076 if (kind == GF_OMP_TARGET_KIND_REGION)
8078 tree fnaddr = build_fold_addr_expr (child_fn);
8079 g = gimple_build_call (builtin_decl_explicit (start_ix), 7,
8080 device, fnaddr, openmp_target, t1, t2, t3, t4);
8082 else
8083 g = gimple_build_call (builtin_decl_explicit (start_ix), 6,
8084 device, openmp_target, t1, t2, t3, t4);
8085 gimple_set_location (g, gimple_location (entry_stmt));
8086 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
8087 if (kind != GF_OMP_TARGET_KIND_REGION)
8089 g = gsi_stmt (gsi);
8090 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET);
8091 gsi_remove (&gsi, true);
8093 if (kind == GF_OMP_TARGET_KIND_DATA && region->exit)
8095 gsi = gsi_last_bb (region->exit);
8096 g = gsi_stmt (gsi);
8097 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN);
8098 gsi_remove (&gsi, true);
8103 /* Expand the parallel region tree rooted at REGION. Expansion
8104 proceeds in depth-first order. Innermost regions are expanded
8105 first. This way, parallel regions that require a new function to
8106 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8107 internal dependencies in their body. */
8109 static void
8110 expand_omp (struct omp_region *region)
8112 while (region)
8114 location_t saved_location;
8115 gimple inner_stmt = NULL;
8117 /* First, determine whether this is a combined parallel+workshare
8118 region. */
8119 if (region->type == GIMPLE_OMP_PARALLEL)
8120 determine_parallel_type (region);
8122 if (region->type == GIMPLE_OMP_FOR
8123 && gimple_omp_for_combined_p (last_stmt (region->entry)))
8124 inner_stmt = last_stmt (region->inner->entry);
8126 if (region->inner)
8127 expand_omp (region->inner);
8129 saved_location = input_location;
8130 if (gimple_has_location (last_stmt (region->entry)))
8131 input_location = gimple_location (last_stmt (region->entry));
8133 switch (region->type)
8135 case GIMPLE_OMP_PARALLEL:
8136 case GIMPLE_OMP_TASK:
8137 expand_omp_taskreg (region);
8138 break;
8140 case GIMPLE_OMP_FOR:
8141 expand_omp_for (region, inner_stmt);
8142 break;
8144 case GIMPLE_OMP_SECTIONS:
8145 expand_omp_sections (region);
8146 break;
8148 case GIMPLE_OMP_SECTION:
8149 /* Individual omp sections are handled together with their
8150 parent GIMPLE_OMP_SECTIONS region. */
8151 break;
8153 case GIMPLE_OMP_SINGLE:
8154 expand_omp_single (region);
8155 break;
8157 case GIMPLE_OMP_MASTER:
8158 case GIMPLE_OMP_TASKGROUP:
8159 case GIMPLE_OMP_ORDERED:
8160 case GIMPLE_OMP_CRITICAL:
8161 case GIMPLE_OMP_TEAMS:
8162 expand_omp_synch (region);
8163 break;
8165 case GIMPLE_OMP_ATOMIC_LOAD:
8166 expand_omp_atomic (region);
8167 break;
8169 case GIMPLE_OMP_TARGET:
8170 expand_omp_target (region);
8171 break;
8173 default:
8174 gcc_unreachable ();
8177 input_location = saved_location;
8178 region = region->next;
8183 /* Helper for build_omp_regions. Scan the dominator tree starting at
8184 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8185 true, the function ends once a single tree is built (otherwise, whole
8186 forest of OMP constructs may be built). */
8188 static void
8189 build_omp_regions_1 (basic_block bb, struct omp_region *parent,
8190 bool single_tree)
8192 gimple_stmt_iterator gsi;
8193 gimple stmt;
8194 basic_block son;
8196 gsi = gsi_last_bb (bb);
8197 if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi)))
8199 struct omp_region *region;
8200 enum gimple_code code;
8202 stmt = gsi_stmt (gsi);
8203 code = gimple_code (stmt);
8204 if (code == GIMPLE_OMP_RETURN)
8206 /* STMT is the return point out of region PARENT. Mark it
8207 as the exit point and make PARENT the immediately
8208 enclosing region. */
8209 gcc_assert (parent);
8210 region = parent;
8211 region->exit = bb;
8212 parent = parent->outer;
8214 else if (code == GIMPLE_OMP_ATOMIC_STORE)
8216 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8217 GIMPLE_OMP_RETURN, but matches with
8218 GIMPLE_OMP_ATOMIC_LOAD. */
8219 gcc_assert (parent);
8220 gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD);
8221 region = parent;
8222 region->exit = bb;
8223 parent = parent->outer;
8226 else if (code == GIMPLE_OMP_CONTINUE)
8228 gcc_assert (parent);
8229 parent->cont = bb;
8231 else if (code == GIMPLE_OMP_SECTIONS_SWITCH)
8233 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8234 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8237 else if (code == GIMPLE_OMP_TARGET
8238 && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_UPDATE)
8239 new_omp_region (bb, code, parent);
8240 else
8242 /* Otherwise, this directive becomes the parent for a new
8243 region. */
8244 region = new_omp_region (bb, code, parent);
8245 parent = region;
8249 if (single_tree && !parent)
8250 return;
8252 for (son = first_dom_son (CDI_DOMINATORS, bb);
8253 son;
8254 son = next_dom_son (CDI_DOMINATORS, son))
8255 build_omp_regions_1 (son, parent, single_tree);
8258 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8259 root_omp_region. */
8261 static void
8262 build_omp_regions_root (basic_block root)
8264 gcc_assert (root_omp_region == NULL);
8265 build_omp_regions_1 (root, NULL, true);
8266 gcc_assert (root_omp_region != NULL);
8269 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8271 void
8272 omp_expand_local (basic_block head)
8274 build_omp_regions_root (head);
8275 if (dump_file && (dump_flags & TDF_DETAILS))
8277 fprintf (dump_file, "\nOMP region tree\n\n");
8278 dump_omp_region (dump_file, root_omp_region, 0);
8279 fprintf (dump_file, "\n");
8282 remove_exit_barriers (root_omp_region);
8283 expand_omp (root_omp_region);
8285 free_omp_regions ();
8288 /* Scan the CFG and build a tree of OMP regions. Return the root of
8289 the OMP region tree. */
8291 static void
8292 build_omp_regions (void)
8294 gcc_assert (root_omp_region == NULL);
8295 calculate_dominance_info (CDI_DOMINATORS);
8296 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false);
8299 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8301 static unsigned int
8302 execute_expand_omp (void)
8304 build_omp_regions ();
8306 if (!root_omp_region)
8307 return 0;
8309 if (dump_file)
8311 fprintf (dump_file, "\nOMP region tree\n\n");
8312 dump_omp_region (dump_file, root_omp_region, 0);
8313 fprintf (dump_file, "\n");
8316 remove_exit_barriers (root_omp_region);
8318 expand_omp (root_omp_region);
8320 cleanup_tree_cfg ();
8322 free_omp_regions ();
8324 return 0;
8327 /* OMP expansion -- the default pass, run before creation of SSA form. */
8329 static bool
8330 gate_expand_omp (void)
8332 return ((flag_openmp != 0 || flag_openmp_simd != 0
8333 || flag_cilkplus != 0) && !seen_error ());
8336 namespace {
8338 const pass_data pass_data_expand_omp =
8340 GIMPLE_PASS, /* type */
8341 "ompexp", /* name */
8342 OPTGROUP_NONE, /* optinfo_flags */
8343 true, /* has_gate */
8344 true, /* has_execute */
8345 TV_NONE, /* tv_id */
8346 PROP_gimple_any, /* properties_required */
8347 0, /* properties_provided */
8348 0, /* properties_destroyed */
8349 0, /* todo_flags_start */
8350 0, /* todo_flags_finish */
8353 class pass_expand_omp : public gimple_opt_pass
8355 public:
8356 pass_expand_omp (gcc::context *ctxt)
8357 : gimple_opt_pass (pass_data_expand_omp, ctxt)
8360 /* opt_pass methods: */
8361 bool gate () { return gate_expand_omp (); }
8362 unsigned int execute () { return execute_expand_omp (); }
8364 }; // class pass_expand_omp
8366 } // anon namespace
8368 gimple_opt_pass *
8369 make_pass_expand_omp (gcc::context *ctxt)
8371 return new pass_expand_omp (ctxt);
8374 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8376 /* If ctx is a worksharing context inside of a cancellable parallel
8377 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8378 and conditional branch to parallel's cancel_label to handle
8379 cancellation in the implicit barrier. */
8381 static void
8382 maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body)
8384 gimple omp_return = gimple_seq_last_stmt (*body);
8385 gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
8386 if (gimple_omp_return_nowait_p (omp_return))
8387 return;
8388 if (ctx->outer
8389 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL
8390 && ctx->outer->cancellable)
8392 tree lhs = create_tmp_var (boolean_type_node, NULL);
8393 gimple_omp_return_set_lhs (omp_return, lhs);
8394 tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
8395 gimple g = gimple_build_cond (NE_EXPR, lhs, boolean_false_node,
8396 ctx->outer->cancel_label, fallthru_label);
8397 gimple_seq_add_stmt (body, g);
8398 gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
8402 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8403 CTX is the enclosing OMP context for the current statement. */
8405 static void
8406 lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8408 tree block, control;
8409 gimple_stmt_iterator tgsi;
8410 gimple stmt, new_stmt, bind, t;
8411 gimple_seq ilist, dlist, olist, new_body;
8413 stmt = gsi_stmt (*gsi_p);
8415 push_gimplify_context ();
8417 dlist = NULL;
8418 ilist = NULL;
8419 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
8420 &ilist, &dlist, ctx, NULL);
8422 new_body = gimple_omp_body (stmt);
8423 gimple_omp_set_body (stmt, NULL);
8424 tgsi = gsi_start (new_body);
8425 for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
8427 omp_context *sctx;
8428 gimple sec_start;
8430 sec_start = gsi_stmt (tgsi);
8431 sctx = maybe_lookup_ctx (sec_start);
8432 gcc_assert (sctx);
8434 lower_omp (gimple_omp_body_ptr (sec_start), sctx);
8435 gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
8436 GSI_CONTINUE_LINKING);
8437 gimple_omp_set_body (sec_start, NULL);
8439 if (gsi_one_before_end_p (tgsi))
8441 gimple_seq l = NULL;
8442 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
8443 &l, ctx);
8444 gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
8445 gimple_omp_section_set_last (sec_start);
8448 gsi_insert_after (&tgsi, gimple_build_omp_return (false),
8449 GSI_CONTINUE_LINKING);
8452 block = make_node (BLOCK);
8453 bind = gimple_build_bind (NULL, new_body, block);
8455 olist = NULL;
8456 lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx);
8458 block = make_node (BLOCK);
8459 new_stmt = gimple_build_bind (NULL, NULL, block);
8460 gsi_replace (gsi_p, new_stmt, true);
8462 pop_gimplify_context (new_stmt);
8463 gimple_bind_append_vars (new_stmt, ctx->block_vars);
8464 BLOCK_VARS (block) = gimple_bind_vars (bind);
8465 if (BLOCK_VARS (block))
8466 TREE_USED (block) = 1;
8468 new_body = NULL;
8469 gimple_seq_add_seq (&new_body, ilist);
8470 gimple_seq_add_stmt (&new_body, stmt);
8471 gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
8472 gimple_seq_add_stmt (&new_body, bind);
8474 control = create_tmp_var (unsigned_type_node, ".section");
8475 t = gimple_build_omp_continue (control, control);
8476 gimple_omp_sections_set_control (stmt, control);
8477 gimple_seq_add_stmt (&new_body, t);
8479 gimple_seq_add_seq (&new_body, olist);
8480 if (ctx->cancellable)
8481 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
8482 gimple_seq_add_seq (&new_body, dlist);
8484 new_body = maybe_catch_exception (new_body);
8486 t = gimple_build_omp_return
8487 (!!find_omp_clause (gimple_omp_sections_clauses (stmt),
8488 OMP_CLAUSE_NOWAIT));
8489 gimple_seq_add_stmt (&new_body, t);
8490 maybe_add_implicit_barrier_cancel (ctx, &new_body);
8492 gimple_bind_set_body (new_stmt, new_body);
8496 /* A subroutine of lower_omp_single. Expand the simple form of
8497 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8499 if (GOMP_single_start ())
8500 BODY;
8501 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8503 FIXME. It may be better to delay expanding the logic of this until
8504 pass_expand_omp. The expanded logic may make the job more difficult
8505 to a synchronization analysis pass. */
8507 static void
8508 lower_omp_single_simple (gimple single_stmt, gimple_seq *pre_p)
8510 location_t loc = gimple_location (single_stmt);
8511 tree tlabel = create_artificial_label (loc);
8512 tree flabel = create_artificial_label (loc);
8513 gimple call, cond;
8514 tree lhs, decl;
8516 decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START);
8517 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)), NULL);
8518 call = gimple_build_call (decl, 0);
8519 gimple_call_set_lhs (call, lhs);
8520 gimple_seq_add_stmt (pre_p, call);
8522 cond = gimple_build_cond (EQ_EXPR, lhs,
8523 fold_convert_loc (loc, TREE_TYPE (lhs),
8524 boolean_true_node),
8525 tlabel, flabel);
8526 gimple_seq_add_stmt (pre_p, cond);
8527 gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
8528 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8529 gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
8533 /* A subroutine of lower_omp_single. Expand the simple form of
8534 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8536 #pragma omp single copyprivate (a, b, c)
8538 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8541 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8543 BODY;
8544 copyout.a = a;
8545 copyout.b = b;
8546 copyout.c = c;
8547 GOMP_single_copy_end (&copyout);
8549 else
8551 a = copyout_p->a;
8552 b = copyout_p->b;
8553 c = copyout_p->c;
8555 GOMP_barrier ();
8558 FIXME. It may be better to delay expanding the logic of this until
8559 pass_expand_omp. The expanded logic may make the job more difficult
8560 to a synchronization analysis pass. */
8562 static void
8563 lower_omp_single_copy (gimple single_stmt, gimple_seq *pre_p, omp_context *ctx)
8565 tree ptr_type, t, l0, l1, l2, bfn_decl;
8566 gimple_seq copyin_seq;
8567 location_t loc = gimple_location (single_stmt);
8569 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
8571 ptr_type = build_pointer_type (ctx->record_type);
8572 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
8574 l0 = create_artificial_label (loc);
8575 l1 = create_artificial_label (loc);
8576 l2 = create_artificial_label (loc);
8578 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
8579 t = build_call_expr_loc (loc, bfn_decl, 0);
8580 t = fold_convert_loc (loc, ptr_type, t);
8581 gimplify_assign (ctx->receiver_decl, t, pre_p);
8583 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
8584 build_int_cst (ptr_type, 0));
8585 t = build3 (COND_EXPR, void_type_node, t,
8586 build_and_jump (&l0), build_and_jump (&l1));
8587 gimplify_and_add (t, pre_p);
8589 gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
8591 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8593 copyin_seq = NULL;
8594 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
8595 &copyin_seq, ctx);
8597 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
8598 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
8599 t = build_call_expr_loc (loc, bfn_decl, 1, t);
8600 gimplify_and_add (t, pre_p);
8602 t = build_and_jump (&l2);
8603 gimplify_and_add (t, pre_p);
8605 gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
8607 gimple_seq_add_seq (pre_p, copyin_seq);
8609 gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
8613 /* Expand code for an OpenMP single directive. */
8615 static void
8616 lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8618 tree block;
8619 gimple t, bind, single_stmt = gsi_stmt (*gsi_p);
8620 gimple_seq bind_body, bind_body_tail = NULL, dlist;
8622 push_gimplify_context ();
8624 block = make_node (BLOCK);
8625 bind = gimple_build_bind (NULL, NULL, block);
8626 gsi_replace (gsi_p, bind, true);
8627 bind_body = NULL;
8628 dlist = NULL;
8629 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
8630 &bind_body, &dlist, ctx, NULL);
8631 lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
8633 gimple_seq_add_stmt (&bind_body, single_stmt);
8635 if (ctx->record_type)
8636 lower_omp_single_copy (single_stmt, &bind_body, ctx);
8637 else
8638 lower_omp_single_simple (single_stmt, &bind_body);
8640 gimple_omp_set_body (single_stmt, NULL);
8642 gimple_seq_add_seq (&bind_body, dlist);
8644 bind_body = maybe_catch_exception (bind_body);
8646 t = gimple_build_omp_return
8647 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt),
8648 OMP_CLAUSE_NOWAIT));
8649 gimple_seq_add_stmt (&bind_body_tail, t);
8650 maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail);
8651 if (ctx->record_type)
8653 gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
8654 tree clobber = build_constructor (ctx->record_type, NULL);
8655 TREE_THIS_VOLATILE (clobber) = 1;
8656 gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
8657 clobber), GSI_SAME_STMT);
8659 gimple_seq_add_seq (&bind_body, bind_body_tail);
8660 gimple_bind_set_body (bind, bind_body);
8662 pop_gimplify_context (bind);
8664 gimple_bind_append_vars (bind, ctx->block_vars);
8665 BLOCK_VARS (block) = ctx->block_vars;
8666 if (BLOCK_VARS (block))
8667 TREE_USED (block) = 1;
8671 /* Expand code for an OpenMP master directive. */
8673 static void
8674 lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8676 tree block, lab = NULL, x, bfn_decl;
8677 gimple stmt = gsi_stmt (*gsi_p), bind;
8678 location_t loc = gimple_location (stmt);
8679 gimple_seq tseq;
8681 push_gimplify_context ();
8683 block = make_node (BLOCK);
8684 bind = gimple_build_bind (NULL, NULL, block);
8685 gsi_replace (gsi_p, bind, true);
8686 gimple_bind_add_stmt (bind, stmt);
8688 bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
8689 x = build_call_expr_loc (loc, bfn_decl, 0);
8690 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node);
8691 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
8692 tseq = NULL;
8693 gimplify_and_add (x, &tseq);
8694 gimple_bind_add_seq (bind, tseq);
8696 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8697 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
8698 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8699 gimple_omp_set_body (stmt, NULL);
8701 gimple_bind_add_stmt (bind, gimple_build_label (lab));
8703 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8705 pop_gimplify_context (bind);
8707 gimple_bind_append_vars (bind, ctx->block_vars);
8708 BLOCK_VARS (block) = ctx->block_vars;
8712 /* Expand code for an OpenMP taskgroup directive. */
8714 static void
8715 lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8717 gimple stmt = gsi_stmt (*gsi_p), bind, x;
8718 tree block = make_node (BLOCK);
8720 bind = gimple_build_bind (NULL, NULL, block);
8721 gsi_replace (gsi_p, bind, true);
8722 gimple_bind_add_stmt (bind, stmt);
8724 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
8726 gimple_bind_add_stmt (bind, x);
8728 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8729 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8730 gimple_omp_set_body (stmt, NULL);
8732 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8734 gimple_bind_append_vars (bind, ctx->block_vars);
8735 BLOCK_VARS (block) = ctx->block_vars;
8739 /* Expand code for an OpenMP ordered directive. */
8741 static void
8742 lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8744 tree block;
8745 gimple stmt = gsi_stmt (*gsi_p), bind, x;
8747 push_gimplify_context ();
8749 block = make_node (BLOCK);
8750 bind = gimple_build_bind (NULL, NULL, block);
8751 gsi_replace (gsi_p, bind, true);
8752 gimple_bind_add_stmt (bind, stmt);
8754 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
8756 gimple_bind_add_stmt (bind, x);
8758 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8759 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
8760 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8761 gimple_omp_set_body (stmt, NULL);
8763 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END), 0);
8764 gimple_bind_add_stmt (bind, x);
8766 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8768 pop_gimplify_context (bind);
8770 gimple_bind_append_vars (bind, ctx->block_vars);
8771 BLOCK_VARS (block) = gimple_bind_vars (bind);
8775 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8776 substitution of a couple of function calls. But in the NAMED case,
8777 requires that languages coordinate a symbol name. It is therefore
8778 best put here in common code. */
8780 static GTY((param1_is (tree), param2_is (tree)))
8781 splay_tree critical_name_mutexes;
8783 static void
8784 lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8786 tree block;
8787 tree name, lock, unlock;
8788 gimple stmt = gsi_stmt (*gsi_p), bind;
8789 location_t loc = gimple_location (stmt);
8790 gimple_seq tbody;
8792 name = gimple_omp_critical_name (stmt);
8793 if (name)
8795 tree decl;
8796 splay_tree_node n;
8798 if (!critical_name_mutexes)
8799 critical_name_mutexes
8800 = splay_tree_new_ggc (splay_tree_compare_pointers,
8801 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s,
8802 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s);
8804 n = splay_tree_lookup (critical_name_mutexes, (splay_tree_key) name);
8805 if (n == NULL)
8807 char *new_str;
8809 decl = create_tmp_var_raw (ptr_type_node, NULL);
8811 new_str = ACONCAT ((".gomp_critical_user_",
8812 IDENTIFIER_POINTER (name), NULL));
8813 DECL_NAME (decl) = get_identifier (new_str);
8814 TREE_PUBLIC (decl) = 1;
8815 TREE_STATIC (decl) = 1;
8816 DECL_COMMON (decl) = 1;
8817 DECL_ARTIFICIAL (decl) = 1;
8818 DECL_IGNORED_P (decl) = 1;
8819 varpool_finalize_decl (decl);
8821 splay_tree_insert (critical_name_mutexes, (splay_tree_key) name,
8822 (splay_tree_value) decl);
8824 else
8825 decl = (tree) n->value;
8827 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
8828 lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl));
8830 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
8831 unlock = build_call_expr_loc (loc, unlock, 1,
8832 build_fold_addr_expr_loc (loc, decl));
8834 else
8836 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
8837 lock = build_call_expr_loc (loc, lock, 0);
8839 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
8840 unlock = build_call_expr_loc (loc, unlock, 0);
8843 push_gimplify_context ();
8845 block = make_node (BLOCK);
8846 bind = gimple_build_bind (NULL, NULL, block);
8847 gsi_replace (gsi_p, bind, true);
8848 gimple_bind_add_stmt (bind, stmt);
8850 tbody = gimple_bind_body (bind);
8851 gimplify_and_add (lock, &tbody);
8852 gimple_bind_set_body (bind, tbody);
8854 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8855 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
8856 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8857 gimple_omp_set_body (stmt, NULL);
8859 tbody = gimple_bind_body (bind);
8860 gimplify_and_add (unlock, &tbody);
8861 gimple_bind_set_body (bind, tbody);
8863 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8865 pop_gimplify_context (bind);
8866 gimple_bind_append_vars (bind, ctx->block_vars);
8867 BLOCK_VARS (block) = gimple_bind_vars (bind);
8871 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8872 for a lastprivate clause. Given a loop control predicate of (V
8873 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8874 is appended to *DLIST, iterator initialization is appended to
8875 *BODY_P. */
8877 static void
8878 lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
8879 gimple_seq *dlist, struct omp_context *ctx)
8881 tree clauses, cond, vinit;
8882 enum tree_code cond_code;
8883 gimple_seq stmts;
8885 cond_code = fd->loop.cond_code;
8886 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
8888 /* When possible, use a strict equality expression. This can let VRP
8889 type optimizations deduce the value and remove a copy. */
8890 if (tree_fits_shwi_p (fd->loop.step))
8892 HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
8893 if (step == 1 || step == -1)
8894 cond_code = EQ_EXPR;
8897 cond = build2 (cond_code, boolean_type_node, fd->loop.v, fd->loop.n2);
8899 clauses = gimple_omp_for_clauses (fd->for_stmt);
8900 stmts = NULL;
8901 lower_lastprivate_clauses (clauses, cond, &stmts, ctx);
8902 if (!gimple_seq_empty_p (stmts))
8904 gimple_seq_add_seq (&stmts, *dlist);
8905 *dlist = stmts;
8907 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8908 vinit = fd->loop.n1;
8909 if (cond_code == EQ_EXPR
8910 && tree_fits_shwi_p (fd->loop.n2)
8911 && ! integer_zerop (fd->loop.n2))
8912 vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
8913 else
8914 vinit = unshare_expr (vinit);
8916 /* Initialize the iterator variable, so that threads that don't execute
8917 any iterations don't execute the lastprivate clauses by accident. */
8918 gimplify_assign (fd->loop.v, vinit, body_p);
8923 /* Lower code for an OpenMP loop directive. */
8925 static void
8926 lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8928 tree *rhs_p, block;
8929 struct omp_for_data fd, *fdp = NULL;
8930 gimple stmt = gsi_stmt (*gsi_p), new_stmt;
8931 gimple_seq omp_for_body, body, dlist;
8932 size_t i;
8934 push_gimplify_context ();
8936 lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
8938 block = make_node (BLOCK);
8939 new_stmt = gimple_build_bind (NULL, NULL, block);
8940 /* Replace at gsi right away, so that 'stmt' is no member
8941 of a sequence anymore as we're going to add to to a different
8942 one below. */
8943 gsi_replace (gsi_p, new_stmt, true);
8945 /* Move declaration of temporaries in the loop body before we make
8946 it go away. */
8947 omp_for_body = gimple_omp_body (stmt);
8948 if (!gimple_seq_empty_p (omp_for_body)
8949 && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
8951 gimple inner_bind = gimple_seq_first_stmt (omp_for_body);
8952 tree vars = gimple_bind_vars (inner_bind);
8953 gimple_bind_append_vars (new_stmt, vars);
8954 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
8955 keep them on the inner_bind and it's block. */
8956 gimple_bind_set_vars (inner_bind, NULL_TREE);
8957 if (gimple_bind_block (inner_bind))
8958 BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
8961 if (gimple_omp_for_combined_into_p (stmt))
8963 extract_omp_for_data (stmt, &fd, NULL);
8964 fdp = &fd;
8966 /* We need two temporaries with fd.loop.v type (istart/iend)
8967 and then (fd.collapse - 1) temporaries with the same
8968 type for count2 ... countN-1 vars if not constant. */
8969 size_t count = 2;
8970 tree type = fd.iter_type;
8971 if (fd.collapse > 1
8972 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
8973 count += fd.collapse - 1;
8974 bool parallel_for = gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR;
8975 tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
8976 tree clauses = *pc;
8977 if (parallel_for)
8978 outerc
8979 = find_omp_clause (gimple_omp_parallel_clauses (ctx->outer->stmt),
8980 OMP_CLAUSE__LOOPTEMP_);
8981 for (i = 0; i < count; i++)
8983 tree temp;
8984 if (parallel_for)
8986 gcc_assert (outerc);
8987 temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
8988 outerc = find_omp_clause (OMP_CLAUSE_CHAIN (outerc),
8989 OMP_CLAUSE__LOOPTEMP_);
8991 else
8992 temp = create_tmp_var (type, NULL);
8993 *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
8994 OMP_CLAUSE_DECL (*pc) = temp;
8995 pc = &OMP_CLAUSE_CHAIN (*pc);
8997 *pc = clauses;
9000 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9001 dlist = NULL;
9002 body = NULL;
9003 lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
9004 fdp);
9005 gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt));
9007 lower_omp (gimple_omp_body_ptr (stmt), ctx);
9009 /* Lower the header expressions. At this point, we can assume that
9010 the header is of the form:
9012 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9014 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9015 using the .omp_data_s mapping, if needed. */
9016 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
9018 rhs_p = gimple_omp_for_initial_ptr (stmt, i);
9019 if (!is_gimple_min_invariant (*rhs_p))
9020 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
9022 rhs_p = gimple_omp_for_final_ptr (stmt, i);
9023 if (!is_gimple_min_invariant (*rhs_p))
9024 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
9026 rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
9027 if (!is_gimple_min_invariant (*rhs_p))
9028 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
9031 /* Once lowered, extract the bounds and clauses. */
9032 extract_omp_for_data (stmt, &fd, NULL);
9034 lower_omp_for_lastprivate (&fd, &body, &dlist, ctx);
9036 gimple_seq_add_stmt (&body, stmt);
9037 gimple_seq_add_seq (&body, gimple_omp_body (stmt));
9039 gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
9040 fd.loop.v));
9042 /* After the loop, add exit clauses. */
9043 lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx);
9045 if (ctx->cancellable)
9046 gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
9048 gimple_seq_add_seq (&body, dlist);
9050 body = maybe_catch_exception (body);
9052 /* Region exit marker goes at the end of the loop body. */
9053 gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait));
9054 maybe_add_implicit_barrier_cancel (ctx, &body);
9055 pop_gimplify_context (new_stmt);
9057 gimple_bind_append_vars (new_stmt, ctx->block_vars);
9058 BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
9059 if (BLOCK_VARS (block))
9060 TREE_USED (block) = 1;
9062 gimple_bind_set_body (new_stmt, body);
9063 gimple_omp_set_body (stmt, NULL);
9064 gimple_omp_for_set_pre_body (stmt, NULL);
9067 /* Callback for walk_stmts. Check if the current statement only contains
9068 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9070 static tree
9071 check_combined_parallel (gimple_stmt_iterator *gsi_p,
9072 bool *handled_ops_p,
9073 struct walk_stmt_info *wi)
9075 int *info = (int *) wi->info;
9076 gimple stmt = gsi_stmt (*gsi_p);
9078 *handled_ops_p = true;
9079 switch (gimple_code (stmt))
9081 WALK_SUBSTMTS;
9083 case GIMPLE_OMP_FOR:
9084 case GIMPLE_OMP_SECTIONS:
9085 *info = *info == 0 ? 1 : -1;
9086 break;
9087 default:
9088 *info = -1;
9089 break;
9091 return NULL;
9094 struct omp_taskcopy_context
9096 /* This field must be at the beginning, as we do "inheritance": Some
9097 callback functions for tree-inline.c (e.g., omp_copy_decl)
9098 receive a copy_body_data pointer that is up-casted to an
9099 omp_context pointer. */
9100 copy_body_data cb;
9101 omp_context *ctx;
9104 static tree
9105 task_copyfn_copy_decl (tree var, copy_body_data *cb)
9107 struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
9109 if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
9110 return create_tmp_var (TREE_TYPE (var), NULL);
9112 return var;
9115 static tree
9116 task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
9118 tree name, new_fields = NULL, type, f;
9120 type = lang_hooks.types.make_type (RECORD_TYPE);
9121 name = DECL_NAME (TYPE_NAME (orig_type));
9122 name = build_decl (gimple_location (tcctx->ctx->stmt),
9123 TYPE_DECL, name, type);
9124 TYPE_NAME (type) = name;
9126 for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
9128 tree new_f = copy_node (f);
9129 DECL_CONTEXT (new_f) = type;
9130 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
9131 TREE_CHAIN (new_f) = new_fields;
9132 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
9133 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
9134 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
9135 &tcctx->cb, NULL);
9136 new_fields = new_f;
9137 *pointer_map_insert (tcctx->cb.decl_map, f) = new_f;
9139 TYPE_FIELDS (type) = nreverse (new_fields);
9140 layout_type (type);
9141 return type;
9144 /* Create task copyfn. */
9146 static void
9147 create_task_copyfn (gimple task_stmt, omp_context *ctx)
9149 struct function *child_cfun;
9150 tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
9151 tree record_type, srecord_type, bind, list;
9152 bool record_needs_remap = false, srecord_needs_remap = false;
9153 splay_tree_node n;
9154 struct omp_taskcopy_context tcctx;
9155 location_t loc = gimple_location (task_stmt);
9157 child_fn = gimple_omp_task_copy_fn (task_stmt);
9158 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
9159 gcc_assert (child_cfun->cfg == NULL);
9160 DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
9162 /* Reset DECL_CONTEXT on function arguments. */
9163 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
9164 DECL_CONTEXT (t) = child_fn;
9166 /* Populate the function. */
9167 push_gimplify_context ();
9168 push_cfun (child_cfun);
9170 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
9171 TREE_SIDE_EFFECTS (bind) = 1;
9172 list = NULL;
9173 DECL_SAVED_TREE (child_fn) = bind;
9174 DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
9176 /* Remap src and dst argument types if needed. */
9177 record_type = ctx->record_type;
9178 srecord_type = ctx->srecord_type;
9179 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
9180 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
9182 record_needs_remap = true;
9183 break;
9185 for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
9186 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
9188 srecord_needs_remap = true;
9189 break;
9192 if (record_needs_remap || srecord_needs_remap)
9194 memset (&tcctx, '\0', sizeof (tcctx));
9195 tcctx.cb.src_fn = ctx->cb.src_fn;
9196 tcctx.cb.dst_fn = child_fn;
9197 tcctx.cb.src_node = cgraph_get_node (tcctx.cb.src_fn);
9198 gcc_checking_assert (tcctx.cb.src_node);
9199 tcctx.cb.dst_node = tcctx.cb.src_node;
9200 tcctx.cb.src_cfun = ctx->cb.src_cfun;
9201 tcctx.cb.copy_decl = task_copyfn_copy_decl;
9202 tcctx.cb.eh_lp_nr = 0;
9203 tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
9204 tcctx.cb.decl_map = pointer_map_create ();
9205 tcctx.ctx = ctx;
9207 if (record_needs_remap)
9208 record_type = task_copyfn_remap_type (&tcctx, record_type);
9209 if (srecord_needs_remap)
9210 srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
9212 else
9213 tcctx.cb.decl_map = NULL;
9215 arg = DECL_ARGUMENTS (child_fn);
9216 TREE_TYPE (arg) = build_pointer_type (record_type);
9217 sarg = DECL_CHAIN (arg);
9218 TREE_TYPE (sarg) = build_pointer_type (srecord_type);
9220 /* First pass: initialize temporaries used in record_type and srecord_type
9221 sizes and field offsets. */
9222 if (tcctx.cb.decl_map)
9223 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
9224 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
9226 tree *p;
9228 decl = OMP_CLAUSE_DECL (c);
9229 p = (tree *) pointer_map_contains (tcctx.cb.decl_map, decl);
9230 if (p == NULL)
9231 continue;
9232 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9233 sf = (tree) n->value;
9234 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9235 src = build_simple_mem_ref_loc (loc, sarg);
9236 src = omp_build_component_ref (src, sf);
9237 t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
9238 append_to_statement_list (t, &list);
9241 /* Second pass: copy shared var pointers and copy construct non-VLA
9242 firstprivate vars. */
9243 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
9244 switch (OMP_CLAUSE_CODE (c))
9246 case OMP_CLAUSE_SHARED:
9247 decl = OMP_CLAUSE_DECL (c);
9248 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9249 if (n == NULL)
9250 break;
9251 f = (tree) n->value;
9252 if (tcctx.cb.decl_map)
9253 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9254 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9255 sf = (tree) n->value;
9256 if (tcctx.cb.decl_map)
9257 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9258 src = build_simple_mem_ref_loc (loc, sarg);
9259 src = omp_build_component_ref (src, sf);
9260 dst = build_simple_mem_ref_loc (loc, arg);
9261 dst = omp_build_component_ref (dst, f);
9262 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
9263 append_to_statement_list (t, &list);
9264 break;
9265 case OMP_CLAUSE_FIRSTPRIVATE:
9266 decl = OMP_CLAUSE_DECL (c);
9267 if (is_variable_sized (decl))
9268 break;
9269 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9270 if (n == NULL)
9271 break;
9272 f = (tree) n->value;
9273 if (tcctx.cb.decl_map)
9274 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9275 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9276 if (n != NULL)
9278 sf = (tree) n->value;
9279 if (tcctx.cb.decl_map)
9280 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9281 src = build_simple_mem_ref_loc (loc, sarg);
9282 src = omp_build_component_ref (src, sf);
9283 if (use_pointer_for_field (decl, NULL) || is_reference (decl))
9284 src = build_simple_mem_ref_loc (loc, src);
9286 else
9287 src = decl;
9288 dst = build_simple_mem_ref_loc (loc, arg);
9289 dst = omp_build_component_ref (dst, f);
9290 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
9291 append_to_statement_list (t, &list);
9292 break;
9293 case OMP_CLAUSE_PRIVATE:
9294 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
9295 break;
9296 decl = OMP_CLAUSE_DECL (c);
9297 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9298 f = (tree) n->value;
9299 if (tcctx.cb.decl_map)
9300 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9301 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9302 if (n != NULL)
9304 sf = (tree) n->value;
9305 if (tcctx.cb.decl_map)
9306 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9307 src = build_simple_mem_ref_loc (loc, sarg);
9308 src = omp_build_component_ref (src, sf);
9309 if (use_pointer_for_field (decl, NULL))
9310 src = build_simple_mem_ref_loc (loc, src);
9312 else
9313 src = decl;
9314 dst = build_simple_mem_ref_loc (loc, arg);
9315 dst = omp_build_component_ref (dst, f);
9316 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
9317 append_to_statement_list (t, &list);
9318 break;
9319 default:
9320 break;
9323 /* Last pass: handle VLA firstprivates. */
9324 if (tcctx.cb.decl_map)
9325 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
9326 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
9328 tree ind, ptr, df;
9330 decl = OMP_CLAUSE_DECL (c);
9331 if (!is_variable_sized (decl))
9332 continue;
9333 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9334 if (n == NULL)
9335 continue;
9336 f = (tree) n->value;
9337 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9338 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
9339 ind = DECL_VALUE_EXPR (decl);
9340 gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
9341 gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
9342 n = splay_tree_lookup (ctx->sfield_map,
9343 (splay_tree_key) TREE_OPERAND (ind, 0));
9344 sf = (tree) n->value;
9345 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9346 src = build_simple_mem_ref_loc (loc, sarg);
9347 src = omp_build_component_ref (src, sf);
9348 src = build_simple_mem_ref_loc (loc, src);
9349 dst = build_simple_mem_ref_loc (loc, arg);
9350 dst = omp_build_component_ref (dst, f);
9351 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
9352 append_to_statement_list (t, &list);
9353 n = splay_tree_lookup (ctx->field_map,
9354 (splay_tree_key) TREE_OPERAND (ind, 0));
9355 df = (tree) n->value;
9356 df = *(tree *) pointer_map_contains (tcctx.cb.decl_map, df);
9357 ptr = build_simple_mem_ref_loc (loc, arg);
9358 ptr = omp_build_component_ref (ptr, df);
9359 t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
9360 build_fold_addr_expr_loc (loc, dst));
9361 append_to_statement_list (t, &list);
9364 t = build1 (RETURN_EXPR, void_type_node, NULL);
9365 append_to_statement_list (t, &list);
9367 if (tcctx.cb.decl_map)
9368 pointer_map_destroy (tcctx.cb.decl_map);
9369 pop_gimplify_context (NULL);
9370 BIND_EXPR_BODY (bind) = list;
9371 pop_cfun ();
9374 static void
9375 lower_depend_clauses (gimple stmt, gimple_seq *iseq, gimple_seq *oseq)
9377 tree c, clauses;
9378 gimple g;
9379 size_t n_in = 0, n_out = 0, idx = 2, i;
9381 clauses = find_omp_clause (gimple_omp_task_clauses (stmt),
9382 OMP_CLAUSE_DEPEND);
9383 gcc_assert (clauses);
9384 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9385 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
9386 switch (OMP_CLAUSE_DEPEND_KIND (c))
9388 case OMP_CLAUSE_DEPEND_IN:
9389 n_in++;
9390 break;
9391 case OMP_CLAUSE_DEPEND_OUT:
9392 case OMP_CLAUSE_DEPEND_INOUT:
9393 n_out++;
9394 break;
9395 default:
9396 gcc_unreachable ();
9398 tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2);
9399 tree array = create_tmp_var (type, NULL);
9400 tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
9401 NULL_TREE);
9402 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out));
9403 gimple_seq_add_stmt (iseq, g);
9404 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
9405 NULL_TREE);
9406 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out));
9407 gimple_seq_add_stmt (iseq, g);
9408 for (i = 0; i < 2; i++)
9410 if ((i ? n_in : n_out) == 0)
9411 continue;
9412 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9413 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
9414 && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i))
9416 tree t = OMP_CLAUSE_DECL (c);
9417 t = fold_convert (ptr_type_node, t);
9418 gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
9419 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
9420 NULL_TREE, NULL_TREE);
9421 g = gimple_build_assign (r, t);
9422 gimple_seq_add_stmt (iseq, g);
9425 tree *p = gimple_omp_task_clauses_ptr (stmt);
9426 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
9427 OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
9428 OMP_CLAUSE_CHAIN (c) = *p;
9429 *p = c;
9430 tree clobber = build_constructor (type, NULL);
9431 TREE_THIS_VOLATILE (clobber) = 1;
9432 g = gimple_build_assign (array, clobber);
9433 gimple_seq_add_stmt (oseq, g);
9436 /* Lower the OpenMP parallel or task directive in the current statement
9437 in GSI_P. CTX holds context information for the directive. */
9439 static void
9440 lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9442 tree clauses;
9443 tree child_fn, t;
9444 gimple stmt = gsi_stmt (*gsi_p);
9445 gimple par_bind, bind, dep_bind = NULL;
9446 gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body;
9447 location_t loc = gimple_location (stmt);
9449 clauses = gimple_omp_taskreg_clauses (stmt);
9450 par_bind = gimple_seq_first_stmt (gimple_omp_body (stmt));
9451 par_body = gimple_bind_body (par_bind);
9452 child_fn = ctx->cb.dst_fn;
9453 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
9454 && !gimple_omp_parallel_combined_p (stmt))
9456 struct walk_stmt_info wi;
9457 int ws_num = 0;
9459 memset (&wi, 0, sizeof (wi));
9460 wi.info = &ws_num;
9461 wi.val_only = true;
9462 walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
9463 if (ws_num == 1)
9464 gimple_omp_parallel_set_combined_p (stmt, true);
9466 gimple_seq dep_ilist = NULL;
9467 gimple_seq dep_olist = NULL;
9468 if (gimple_code (stmt) == GIMPLE_OMP_TASK
9469 && find_omp_clause (clauses, OMP_CLAUSE_DEPEND))
9471 push_gimplify_context ();
9472 dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
9473 lower_depend_clauses (stmt, &dep_ilist, &dep_olist);
9476 if (ctx->srecord_type)
9477 create_task_copyfn (stmt, ctx);
9479 push_gimplify_context ();
9481 par_olist = NULL;
9482 par_ilist = NULL;
9483 par_rlist = NULL;
9484 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
9485 lower_omp (&par_body, ctx);
9486 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL)
9487 lower_reduction_clauses (clauses, &par_rlist, ctx);
9489 /* Declare all the variables created by mapping and the variables
9490 declared in the scope of the parallel body. */
9491 record_vars_into (ctx->block_vars, child_fn);
9492 record_vars_into (gimple_bind_vars (par_bind), child_fn);
9494 if (ctx->record_type)
9496 ctx->sender_decl
9497 = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
9498 : ctx->record_type, ".omp_data_o");
9499 DECL_NAMELESS (ctx->sender_decl) = 1;
9500 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
9501 gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
9504 olist = NULL;
9505 ilist = NULL;
9506 lower_send_clauses (clauses, &ilist, &olist, ctx);
9507 lower_send_shared_vars (&ilist, &olist, ctx);
9509 if (ctx->record_type)
9511 tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL);
9512 TREE_THIS_VOLATILE (clobber) = 1;
9513 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
9514 clobber));
9517 /* Once all the expansions are done, sequence all the different
9518 fragments inside gimple_omp_body. */
9520 new_body = NULL;
9522 if (ctx->record_type)
9524 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
9525 /* fixup_child_record_type might have changed receiver_decl's type. */
9526 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
9527 gimple_seq_add_stmt (&new_body,
9528 gimple_build_assign (ctx->receiver_decl, t));
9531 gimple_seq_add_seq (&new_body, par_ilist);
9532 gimple_seq_add_seq (&new_body, par_body);
9533 gimple_seq_add_seq (&new_body, par_rlist);
9534 if (ctx->cancellable)
9535 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
9536 gimple_seq_add_seq (&new_body, par_olist);
9537 new_body = maybe_catch_exception (new_body);
9538 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
9539 gimple_omp_set_body (stmt, new_body);
9541 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
9542 gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
9543 gimple_bind_add_seq (bind, ilist);
9544 gimple_bind_add_stmt (bind, stmt);
9545 gimple_bind_add_seq (bind, olist);
9547 pop_gimplify_context (NULL);
9549 if (dep_bind)
9551 gimple_bind_add_seq (dep_bind, dep_ilist);
9552 gimple_bind_add_stmt (dep_bind, bind);
9553 gimple_bind_add_seq (dep_bind, dep_olist);
9554 pop_gimplify_context (dep_bind);
9558 /* Lower the OpenMP target directive in the current statement
9559 in GSI_P. CTX holds context information for the directive. */
9561 static void
9562 lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9564 tree clauses;
9565 tree child_fn, t, c;
9566 gimple stmt = gsi_stmt (*gsi_p);
9567 gimple tgt_bind = NULL, bind;
9568 gimple_seq tgt_body = NULL, olist, ilist, new_body;
9569 location_t loc = gimple_location (stmt);
9570 int kind = gimple_omp_target_kind (stmt);
9571 unsigned int map_cnt = 0;
9573 clauses = gimple_omp_target_clauses (stmt);
9574 if (kind == GF_OMP_TARGET_KIND_REGION)
9576 tgt_bind = gimple_seq_first_stmt (gimple_omp_body (stmt));
9577 tgt_body = gimple_bind_body (tgt_bind);
9579 else if (kind == GF_OMP_TARGET_KIND_DATA)
9580 tgt_body = gimple_omp_body (stmt);
9581 child_fn = ctx->cb.dst_fn;
9583 push_gimplify_context ();
9585 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
9586 switch (OMP_CLAUSE_CODE (c))
9588 tree var, x;
9590 default:
9591 break;
9592 case OMP_CLAUSE_MAP:
9593 case OMP_CLAUSE_TO:
9594 case OMP_CLAUSE_FROM:
9595 var = OMP_CLAUSE_DECL (c);
9596 if (!DECL_P (var))
9598 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
9599 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
9600 map_cnt++;
9601 continue;
9604 if (DECL_SIZE (var)
9605 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
9607 tree var2 = DECL_VALUE_EXPR (var);
9608 gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
9609 var2 = TREE_OPERAND (var2, 0);
9610 gcc_assert (DECL_P (var2));
9611 var = var2;
9614 if (!maybe_lookup_field (var, ctx))
9615 continue;
9617 if (kind == GF_OMP_TARGET_KIND_REGION)
9619 x = build_receiver_ref (var, true, ctx);
9620 tree new_var = lookup_decl (var, ctx);
9621 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
9622 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
9623 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
9624 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
9625 x = build_simple_mem_ref (x);
9626 SET_DECL_VALUE_EXPR (new_var, x);
9627 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
9629 map_cnt++;
9632 if (kind == GF_OMP_TARGET_KIND_REGION)
9634 target_nesting_level++;
9635 lower_omp (&tgt_body, ctx);
9636 target_nesting_level--;
9638 else if (kind == GF_OMP_TARGET_KIND_DATA)
9639 lower_omp (&tgt_body, ctx);
9641 if (kind == GF_OMP_TARGET_KIND_REGION)
9643 /* Declare all the variables created by mapping and the variables
9644 declared in the scope of the target body. */
9645 record_vars_into (ctx->block_vars, child_fn);
9646 record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
9649 olist = NULL;
9650 ilist = NULL;
9651 if (ctx->record_type)
9653 ctx->sender_decl
9654 = create_tmp_var (ctx->record_type, ".omp_data_arr");
9655 DECL_NAMELESS (ctx->sender_decl) = 1;
9656 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
9657 t = make_tree_vec (3);
9658 TREE_VEC_ELT (t, 0) = ctx->sender_decl;
9659 TREE_VEC_ELT (t, 1)
9660 = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt),
9661 ".omp_data_sizes");
9662 DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
9663 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
9664 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
9665 TREE_VEC_ELT (t, 2)
9666 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node,
9667 map_cnt),
9668 ".omp_data_kinds");
9669 DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
9670 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
9671 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
9672 gimple_omp_target_set_data_arg (stmt, t);
9674 vec<constructor_elt, va_gc> *vsize;
9675 vec<constructor_elt, va_gc> *vkind;
9676 vec_alloc (vsize, map_cnt);
9677 vec_alloc (vkind, map_cnt);
9678 unsigned int map_idx = 0;
9680 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
9681 switch (OMP_CLAUSE_CODE (c))
9683 tree ovar, nc;
9685 default:
9686 break;
9687 case OMP_CLAUSE_MAP:
9688 case OMP_CLAUSE_TO:
9689 case OMP_CLAUSE_FROM:
9690 nc = c;
9691 ovar = OMP_CLAUSE_DECL (c);
9692 if (!DECL_P (ovar))
9694 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
9695 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
9697 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c))
9698 == get_base_address (ovar));
9699 nc = OMP_CLAUSE_CHAIN (c);
9700 ovar = OMP_CLAUSE_DECL (nc);
9702 else
9704 tree x = build_sender_ref (ovar, ctx);
9705 tree v
9706 = build_fold_addr_expr_with_type (ovar, ptr_type_node);
9707 gimplify_assign (x, v, &ilist);
9708 nc = NULL_TREE;
9711 else
9713 if (DECL_SIZE (ovar)
9714 && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST)
9716 tree ovar2 = DECL_VALUE_EXPR (ovar);
9717 gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
9718 ovar2 = TREE_OPERAND (ovar2, 0);
9719 gcc_assert (DECL_P (ovar2));
9720 ovar = ovar2;
9722 if (!maybe_lookup_field (ovar, ctx))
9723 continue;
9726 if (nc)
9728 tree var = lookup_decl_in_outer_ctx (ovar, ctx);
9729 tree x = build_sender_ref (ovar, ctx);
9730 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
9731 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
9732 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
9733 && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
9735 gcc_assert (kind == GF_OMP_TARGET_KIND_REGION);
9736 tree avar
9737 = create_tmp_var (TREE_TYPE (TREE_TYPE (x)), NULL);
9738 mark_addressable (avar);
9739 gimplify_assign (avar, build_fold_addr_expr (var), &ilist);
9740 avar = build_fold_addr_expr (avar);
9741 gimplify_assign (x, avar, &ilist);
9743 else if (is_gimple_reg (var))
9745 gcc_assert (kind == GF_OMP_TARGET_KIND_REGION);
9746 tree avar = create_tmp_var (TREE_TYPE (var), NULL);
9747 mark_addressable (avar);
9748 if (OMP_CLAUSE_MAP_KIND (c) != OMP_CLAUSE_MAP_ALLOC
9749 && OMP_CLAUSE_MAP_KIND (c) != OMP_CLAUSE_MAP_FROM)
9750 gimplify_assign (avar, var, &ilist);
9751 avar = build_fold_addr_expr (avar);
9752 gimplify_assign (x, avar, &ilist);
9753 if ((OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_FROM
9754 || OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_TOFROM)
9755 && !TYPE_READONLY (TREE_TYPE (var)))
9757 x = build_sender_ref (ovar, ctx);
9758 x = build_simple_mem_ref (x);
9759 gimplify_assign (var, x, &olist);
9762 else
9764 var = build_fold_addr_expr (var);
9765 gimplify_assign (x, var, &ilist);
9768 tree s = OMP_CLAUSE_SIZE (c);
9769 if (s == NULL_TREE)
9770 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
9771 s = fold_convert (size_type_node, s);
9772 tree purpose = size_int (map_idx++);
9773 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
9774 if (TREE_CODE (s) != INTEGER_CST)
9775 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
9777 unsigned char tkind = 0;
9778 switch (OMP_CLAUSE_CODE (c))
9780 case OMP_CLAUSE_MAP:
9781 tkind = OMP_CLAUSE_MAP_KIND (c);
9782 break;
9783 case OMP_CLAUSE_TO:
9784 tkind = OMP_CLAUSE_MAP_TO;
9785 break;
9786 case OMP_CLAUSE_FROM:
9787 tkind = OMP_CLAUSE_MAP_FROM;
9788 break;
9789 default:
9790 gcc_unreachable ();
9792 unsigned int talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
9793 if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
9794 talign = DECL_ALIGN_UNIT (ovar);
9795 talign = ceil_log2 (talign);
9796 tkind |= talign << 3;
9797 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
9798 build_int_cst (unsigned_char_type_node,
9799 tkind));
9800 if (nc && nc != c)
9801 c = nc;
9804 gcc_assert (map_idx == map_cnt);
9806 DECL_INITIAL (TREE_VEC_ELT (t, 1))
9807 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
9808 DECL_INITIAL (TREE_VEC_ELT (t, 2))
9809 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
9810 if (!TREE_STATIC (TREE_VEC_ELT (t, 1)))
9812 gimple_seq initlist = NULL;
9813 force_gimple_operand (build1 (DECL_EXPR, void_type_node,
9814 TREE_VEC_ELT (t, 1)),
9815 &initlist, true, NULL_TREE);
9816 gimple_seq_add_seq (&ilist, initlist);
9818 tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)),
9819 NULL);
9820 TREE_THIS_VOLATILE (clobber) = 1;
9821 gimple_seq_add_stmt (&olist,
9822 gimple_build_assign (TREE_VEC_ELT (t, 1),
9823 clobber));
9826 tree clobber = build_constructor (ctx->record_type, NULL);
9827 TREE_THIS_VOLATILE (clobber) = 1;
9828 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
9829 clobber));
9832 /* Once all the expansions are done, sequence all the different
9833 fragments inside gimple_omp_body. */
9835 new_body = NULL;
9837 if (ctx->record_type && kind == GF_OMP_TARGET_KIND_REGION)
9839 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
9840 /* fixup_child_record_type might have changed receiver_decl's type. */
9841 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
9842 gimple_seq_add_stmt (&new_body,
9843 gimple_build_assign (ctx->receiver_decl, t));
9846 if (kind == GF_OMP_TARGET_KIND_REGION)
9848 gimple_seq_add_seq (&new_body, tgt_body);
9849 new_body = maybe_catch_exception (new_body);
9851 else if (kind == GF_OMP_TARGET_KIND_DATA)
9852 new_body = tgt_body;
9853 if (kind != GF_OMP_TARGET_KIND_UPDATE)
9855 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
9856 gimple_omp_set_body (stmt, new_body);
9859 bind = gimple_build_bind (NULL, NULL,
9860 tgt_bind ? gimple_bind_block (tgt_bind)
9861 : NULL_TREE);
9862 gsi_replace (gsi_p, bind, true);
9863 gimple_bind_add_seq (bind, ilist);
9864 gimple_bind_add_stmt (bind, stmt);
9865 gimple_bind_add_seq (bind, olist);
9867 pop_gimplify_context (NULL);
9870 /* Expand code for an OpenMP teams directive. */
9872 static void
9873 lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9875 gimple teams_stmt = gsi_stmt (*gsi_p);
9876 push_gimplify_context ();
9878 tree block = make_node (BLOCK);
9879 gimple bind = gimple_build_bind (NULL, NULL, block);
9880 gsi_replace (gsi_p, bind, true);
9881 gimple_seq bind_body = NULL;
9882 gimple_seq dlist = NULL;
9883 gimple_seq olist = NULL;
9885 tree num_teams = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
9886 OMP_CLAUSE_NUM_TEAMS);
9887 if (num_teams == NULL_TREE)
9888 num_teams = build_int_cst (unsigned_type_node, 0);
9889 else
9891 num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams);
9892 num_teams = fold_convert (unsigned_type_node, num_teams);
9893 gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
9895 tree thread_limit = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
9896 OMP_CLAUSE_THREAD_LIMIT);
9897 if (thread_limit == NULL_TREE)
9898 thread_limit = build_int_cst (unsigned_type_node, 0);
9899 else
9901 thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
9902 thread_limit = fold_convert (unsigned_type_node, thread_limit);
9903 gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
9904 fb_rvalue);
9907 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
9908 &bind_body, &dlist, ctx, NULL);
9909 lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
9910 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx);
9911 gimple_seq_add_stmt (&bind_body, teams_stmt);
9913 location_t loc = gimple_location (teams_stmt);
9914 tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS);
9915 gimple call = gimple_build_call (decl, 2, num_teams, thread_limit);
9916 gimple_set_location (call, loc);
9917 gimple_seq_add_stmt (&bind_body, call);
9919 gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
9920 gimple_omp_set_body (teams_stmt, NULL);
9921 gimple_seq_add_seq (&bind_body, olist);
9922 gimple_seq_add_seq (&bind_body, dlist);
9923 gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
9924 gimple_bind_set_body (bind, bind_body);
9926 pop_gimplify_context (bind);
9928 gimple_bind_append_vars (bind, ctx->block_vars);
9929 BLOCK_VARS (block) = ctx->block_vars;
9930 if (BLOCK_VARS (block))
9931 TREE_USED (block) = 1;
9935 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
9936 regimplified. If DATA is non-NULL, lower_omp_1 is outside
9937 of OpenMP context, but with task_shared_vars set. */
9939 static tree
9940 lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
9941 void *data)
9943 tree t = *tp;
9945 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
9946 if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t))
9947 return t;
9949 if (task_shared_vars
9950 && DECL_P (t)
9951 && bitmap_bit_p (task_shared_vars, DECL_UID (t)))
9952 return t;
9954 /* If a global variable has been privatized, TREE_CONSTANT on
9955 ADDR_EXPR might be wrong. */
9956 if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
9957 recompute_tree_invariant_for_addr_expr (t);
9959 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
9960 return NULL_TREE;
9963 static void
9964 lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9966 gimple stmt = gsi_stmt (*gsi_p);
9967 struct walk_stmt_info wi;
9969 if (gimple_has_location (stmt))
9970 input_location = gimple_location (stmt);
9972 if (task_shared_vars)
9973 memset (&wi, '\0', sizeof (wi));
9975 /* If we have issued syntax errors, avoid doing any heavy lifting.
9976 Just replace the OpenMP directives with a NOP to avoid
9977 confusing RTL expansion. */
9978 if (seen_error () && is_gimple_omp (stmt))
9980 gsi_replace (gsi_p, gimple_build_nop (), true);
9981 return;
9984 switch (gimple_code (stmt))
9986 case GIMPLE_COND:
9987 if ((ctx || task_shared_vars)
9988 && (walk_tree (gimple_cond_lhs_ptr (stmt), lower_omp_regimplify_p,
9989 ctx ? NULL : &wi, NULL)
9990 || walk_tree (gimple_cond_rhs_ptr (stmt), lower_omp_regimplify_p,
9991 ctx ? NULL : &wi, NULL)))
9992 gimple_regimplify_operands (stmt, gsi_p);
9993 break;
9994 case GIMPLE_CATCH:
9995 lower_omp (gimple_catch_handler_ptr (stmt), ctx);
9996 break;
9997 case GIMPLE_EH_FILTER:
9998 lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
9999 break;
10000 case GIMPLE_TRY:
10001 lower_omp (gimple_try_eval_ptr (stmt), ctx);
10002 lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
10003 break;
10004 case GIMPLE_TRANSACTION:
10005 lower_omp (gimple_transaction_body_ptr (stmt), ctx);
10006 break;
10007 case GIMPLE_BIND:
10008 lower_omp (gimple_bind_body_ptr (stmt), ctx);
10009 break;
10010 case GIMPLE_OMP_PARALLEL:
10011 case GIMPLE_OMP_TASK:
10012 ctx = maybe_lookup_ctx (stmt);
10013 gcc_assert (ctx);
10014 if (ctx->cancellable)
10015 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
10016 lower_omp_taskreg (gsi_p, ctx);
10017 break;
10018 case GIMPLE_OMP_FOR:
10019 ctx = maybe_lookup_ctx (stmt);
10020 gcc_assert (ctx);
10021 if (ctx->cancellable)
10022 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
10023 lower_omp_for (gsi_p, ctx);
10024 break;
10025 case GIMPLE_OMP_SECTIONS:
10026 ctx = maybe_lookup_ctx (stmt);
10027 gcc_assert (ctx);
10028 if (ctx->cancellable)
10029 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
10030 lower_omp_sections (gsi_p, ctx);
10031 break;
10032 case GIMPLE_OMP_SINGLE:
10033 ctx = maybe_lookup_ctx (stmt);
10034 gcc_assert (ctx);
10035 lower_omp_single (gsi_p, ctx);
10036 break;
10037 case GIMPLE_OMP_MASTER:
10038 ctx = maybe_lookup_ctx (stmt);
10039 gcc_assert (ctx);
10040 lower_omp_master (gsi_p, ctx);
10041 break;
10042 case GIMPLE_OMP_TASKGROUP:
10043 ctx = maybe_lookup_ctx (stmt);
10044 gcc_assert (ctx);
10045 lower_omp_taskgroup (gsi_p, ctx);
10046 break;
10047 case GIMPLE_OMP_ORDERED:
10048 ctx = maybe_lookup_ctx (stmt);
10049 gcc_assert (ctx);
10050 lower_omp_ordered (gsi_p, ctx);
10051 break;
10052 case GIMPLE_OMP_CRITICAL:
10053 ctx = maybe_lookup_ctx (stmt);
10054 gcc_assert (ctx);
10055 lower_omp_critical (gsi_p, ctx);
10056 break;
10057 case GIMPLE_OMP_ATOMIC_LOAD:
10058 if ((ctx || task_shared_vars)
10059 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt),
10060 lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
10061 gimple_regimplify_operands (stmt, gsi_p);
10062 break;
10063 case GIMPLE_OMP_TARGET:
10064 ctx = maybe_lookup_ctx (stmt);
10065 gcc_assert (ctx);
10066 lower_omp_target (gsi_p, ctx);
10067 break;
10068 case GIMPLE_OMP_TEAMS:
10069 ctx = maybe_lookup_ctx (stmt);
10070 gcc_assert (ctx);
10071 lower_omp_teams (gsi_p, ctx);
10072 break;
10073 case GIMPLE_CALL:
10074 tree fndecl;
10075 fndecl = gimple_call_fndecl (stmt);
10076 if (fndecl
10077 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
10078 switch (DECL_FUNCTION_CODE (fndecl))
10080 case BUILT_IN_GOMP_BARRIER:
10081 if (ctx == NULL)
10082 break;
10083 /* FALLTHRU */
10084 case BUILT_IN_GOMP_CANCEL:
10085 case BUILT_IN_GOMP_CANCELLATION_POINT:
10086 omp_context *cctx;
10087 cctx = ctx;
10088 if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
10089 cctx = cctx->outer;
10090 gcc_assert (gimple_call_lhs (stmt) == NULL_TREE);
10091 if (!cctx->cancellable)
10093 if (DECL_FUNCTION_CODE (fndecl)
10094 == BUILT_IN_GOMP_CANCELLATION_POINT)
10096 stmt = gimple_build_nop ();
10097 gsi_replace (gsi_p, stmt, false);
10099 break;
10101 tree lhs;
10102 lhs = create_tmp_var (boolean_type_node, NULL);
10103 if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
10105 fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
10106 gimple_call_set_fndecl (stmt, fndecl);
10107 gimple_call_set_fntype (stmt, TREE_TYPE (fndecl));
10109 gimple_call_set_lhs (stmt, lhs);
10110 tree fallthru_label;
10111 fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
10112 gimple g;
10113 g = gimple_build_label (fallthru_label);
10114 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
10115 g = gimple_build_cond (NE_EXPR, lhs, boolean_false_node,
10116 cctx->cancel_label, fallthru_label);
10117 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
10118 break;
10119 default:
10120 break;
10122 /* FALLTHRU */
10123 default:
10124 if ((ctx || task_shared_vars)
10125 && walk_gimple_op (stmt, lower_omp_regimplify_p,
10126 ctx ? NULL : &wi))
10128 /* Just remove clobbers, this should happen only if we have
10129 "privatized" local addressable variables in SIMD regions,
10130 the clobber isn't needed in that case and gimplifying address
10131 of the ARRAY_REF into a pointer and creating MEM_REF based
10132 clobber would create worse code than we get with the clobber
10133 dropped. */
10134 if (gimple_clobber_p (stmt))
10136 gsi_replace (gsi_p, gimple_build_nop (), true);
10137 break;
10139 gimple_regimplify_operands (stmt, gsi_p);
10141 break;
10145 static void
10146 lower_omp (gimple_seq *body, omp_context *ctx)
10148 location_t saved_location = input_location;
10149 gimple_stmt_iterator gsi;
10150 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
10151 lower_omp_1 (&gsi, ctx);
10152 /* During gimplification, we have not always invoked fold_stmt
10153 (gimplify.c:maybe_fold_stmt); call it now. */
10154 if (target_nesting_level)
10155 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
10156 fold_stmt (&gsi);
10157 input_location = saved_location;
10160 /* Main entry point. */
10162 static unsigned int
10163 execute_lower_omp (void)
10165 gimple_seq body;
10167 /* This pass always runs, to provide PROP_gimple_lomp.
10168 But there is nothing to do unless -fopenmp is given. */
10169 if (flag_openmp == 0 && flag_openmp_simd == 0 && flag_cilkplus == 0)
10170 return 0;
10172 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
10173 delete_omp_context);
10175 body = gimple_body (current_function_decl);
10176 scan_omp (&body, NULL);
10177 gcc_assert (taskreg_nesting_level == 0);
10179 if (all_contexts->root)
10181 if (task_shared_vars)
10182 push_gimplify_context ();
10183 lower_omp (&body, NULL);
10184 if (task_shared_vars)
10185 pop_gimplify_context (NULL);
10188 if (all_contexts)
10190 splay_tree_delete (all_contexts);
10191 all_contexts = NULL;
10193 BITMAP_FREE (task_shared_vars);
10194 return 0;
10197 namespace {
10199 const pass_data pass_data_lower_omp =
10201 GIMPLE_PASS, /* type */
10202 "omplower", /* name */
10203 OPTGROUP_NONE, /* optinfo_flags */
10204 false, /* has_gate */
10205 true, /* has_execute */
10206 TV_NONE, /* tv_id */
10207 PROP_gimple_any, /* properties_required */
10208 PROP_gimple_lomp, /* properties_provided */
10209 0, /* properties_destroyed */
10210 0, /* todo_flags_start */
10211 0, /* todo_flags_finish */
10214 class pass_lower_omp : public gimple_opt_pass
10216 public:
10217 pass_lower_omp (gcc::context *ctxt)
10218 : gimple_opt_pass (pass_data_lower_omp, ctxt)
10221 /* opt_pass methods: */
10222 unsigned int execute () { return execute_lower_omp (); }
10224 }; // class pass_lower_omp
10226 } // anon namespace
10228 gimple_opt_pass *
10229 make_pass_lower_omp (gcc::context *ctxt)
10231 return new pass_lower_omp (ctxt);
10234 /* The following is a utility to diagnose OpenMP structured block violations.
10235 It is not part of the "omplower" pass, as that's invoked too late. It
10236 should be invoked by the respective front ends after gimplification. */
10238 static splay_tree all_labels;
10240 /* Check for mismatched contexts and generate an error if needed. Return
10241 true if an error is detected. */
10243 static bool
10244 diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
10245 gimple branch_ctx, gimple label_ctx)
10247 if (label_ctx == branch_ctx)
10248 return false;
10252 Previously we kept track of the label's entire context in diagnose_sb_[12]
10253 so we could traverse it and issue a correct "exit" or "enter" error
10254 message upon a structured block violation.
10256 We built the context by building a list with tree_cons'ing, but there is
10257 no easy counterpart in gimple tuples. It seems like far too much work
10258 for issuing exit/enter error messages. If someone really misses the
10259 distinct error message... patches welcome.
10262 #if 0
10263 /* Try to avoid confusing the user by producing and error message
10264 with correct "exit" or "enter" verbiage. We prefer "exit"
10265 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10266 if (branch_ctx == NULL)
10267 exit_p = false;
10268 else
10270 while (label_ctx)
10272 if (TREE_VALUE (label_ctx) == branch_ctx)
10274 exit_p = false;
10275 break;
10277 label_ctx = TREE_CHAIN (label_ctx);
10281 if (exit_p)
10282 error ("invalid exit from OpenMP structured block");
10283 else
10284 error ("invalid entry to OpenMP structured block");
10285 #endif
10287 bool cilkplus_block = false;
10288 if (flag_cilkplus)
10290 if ((branch_ctx
10291 && gimple_code (branch_ctx) == GIMPLE_OMP_FOR
10292 && gimple_omp_for_kind (branch_ctx) == GF_OMP_FOR_KIND_CILKSIMD)
10293 || (label_ctx
10294 && gimple_code (label_ctx) == GIMPLE_OMP_FOR
10295 && gimple_omp_for_kind (label_ctx) == GF_OMP_FOR_KIND_CILKSIMD))
10296 cilkplus_block = true;
10299 /* If it's obvious we have an invalid entry, be specific about the error. */
10300 if (branch_ctx == NULL)
10302 if (cilkplus_block)
10303 error ("invalid entry to Cilk Plus structured block");
10304 else
10305 error ("invalid entry to OpenMP structured block");
10307 else
10309 /* Otherwise, be vague and lazy, but efficient. */
10310 if (cilkplus_block)
10311 error ("invalid branch to/from a Cilk Plus structured block");
10312 else
10313 error ("invalid branch to/from an OpenMP structured block");
10316 gsi_replace (gsi_p, gimple_build_nop (), false);
10317 return true;
10320 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10321 where each label is found. */
10323 static tree
10324 diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10325 struct walk_stmt_info *wi)
10327 gimple context = (gimple) wi->info;
10328 gimple inner_context;
10329 gimple stmt = gsi_stmt (*gsi_p);
10331 *handled_ops_p = true;
10333 switch (gimple_code (stmt))
10335 WALK_SUBSTMTS;
10337 case GIMPLE_OMP_PARALLEL:
10338 case GIMPLE_OMP_TASK:
10339 case GIMPLE_OMP_SECTIONS:
10340 case GIMPLE_OMP_SINGLE:
10341 case GIMPLE_OMP_SECTION:
10342 case GIMPLE_OMP_MASTER:
10343 case GIMPLE_OMP_ORDERED:
10344 case GIMPLE_OMP_CRITICAL:
10345 case GIMPLE_OMP_TARGET:
10346 case GIMPLE_OMP_TEAMS:
10347 case GIMPLE_OMP_TASKGROUP:
10348 /* The minimal context here is just the current OMP construct. */
10349 inner_context = stmt;
10350 wi->info = inner_context;
10351 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
10352 wi->info = context;
10353 break;
10355 case GIMPLE_OMP_FOR:
10356 inner_context = stmt;
10357 wi->info = inner_context;
10358 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10359 walk them. */
10360 walk_gimple_seq (gimple_omp_for_pre_body (stmt),
10361 diagnose_sb_1, NULL, wi);
10362 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
10363 wi->info = context;
10364 break;
10366 case GIMPLE_LABEL:
10367 splay_tree_insert (all_labels, (splay_tree_key) gimple_label_label (stmt),
10368 (splay_tree_value) context);
10369 break;
10371 default:
10372 break;
10375 return NULL_TREE;
10378 /* Pass 2: Check each branch and see if its context differs from that of
10379 the destination label's context. */
10381 static tree
10382 diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10383 struct walk_stmt_info *wi)
10385 gimple context = (gimple) wi->info;
10386 splay_tree_node n;
10387 gimple stmt = gsi_stmt (*gsi_p);
10389 *handled_ops_p = true;
10391 switch (gimple_code (stmt))
10393 WALK_SUBSTMTS;
10395 case GIMPLE_OMP_PARALLEL:
10396 case GIMPLE_OMP_TASK:
10397 case GIMPLE_OMP_SECTIONS:
10398 case GIMPLE_OMP_SINGLE:
10399 case GIMPLE_OMP_SECTION:
10400 case GIMPLE_OMP_MASTER:
10401 case GIMPLE_OMP_ORDERED:
10402 case GIMPLE_OMP_CRITICAL:
10403 case GIMPLE_OMP_TARGET:
10404 case GIMPLE_OMP_TEAMS:
10405 case GIMPLE_OMP_TASKGROUP:
10406 wi->info = stmt;
10407 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
10408 wi->info = context;
10409 break;
10411 case GIMPLE_OMP_FOR:
10412 wi->info = stmt;
10413 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10414 walk them. */
10415 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
10416 diagnose_sb_2, NULL, wi);
10417 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
10418 wi->info = context;
10419 break;
10421 case GIMPLE_COND:
10423 tree lab = gimple_cond_true_label (stmt);
10424 if (lab)
10426 n = splay_tree_lookup (all_labels,
10427 (splay_tree_key) lab);
10428 diagnose_sb_0 (gsi_p, context,
10429 n ? (gimple) n->value : NULL);
10431 lab = gimple_cond_false_label (stmt);
10432 if (lab)
10434 n = splay_tree_lookup (all_labels,
10435 (splay_tree_key) lab);
10436 diagnose_sb_0 (gsi_p, context,
10437 n ? (gimple) n->value : NULL);
10440 break;
10442 case GIMPLE_GOTO:
10444 tree lab = gimple_goto_dest (stmt);
10445 if (TREE_CODE (lab) != LABEL_DECL)
10446 break;
10448 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
10449 diagnose_sb_0 (gsi_p, context, n ? (gimple) n->value : NULL);
10451 break;
10453 case GIMPLE_SWITCH:
10455 unsigned int i;
10456 for (i = 0; i < gimple_switch_num_labels (stmt); ++i)
10458 tree lab = CASE_LABEL (gimple_switch_label (stmt, i));
10459 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
10460 if (n && diagnose_sb_0 (gsi_p, context, (gimple) n->value))
10461 break;
10464 break;
10466 case GIMPLE_RETURN:
10467 diagnose_sb_0 (gsi_p, context, NULL);
10468 break;
10470 default:
10471 break;
10474 return NULL_TREE;
10477 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10478 codes. */
10479 bool
10480 make_gimple_omp_edges (basic_block bb, struct omp_region **region,
10481 int *region_idx)
10483 gimple last = last_stmt (bb);
10484 enum gimple_code code = gimple_code (last);
10485 struct omp_region *cur_region = *region;
10486 bool fallthru = false;
10488 switch (code)
10490 case GIMPLE_OMP_PARALLEL:
10491 case GIMPLE_OMP_TASK:
10492 case GIMPLE_OMP_FOR:
10493 case GIMPLE_OMP_SINGLE:
10494 case GIMPLE_OMP_TEAMS:
10495 case GIMPLE_OMP_MASTER:
10496 case GIMPLE_OMP_TASKGROUP:
10497 case GIMPLE_OMP_ORDERED:
10498 case GIMPLE_OMP_CRITICAL:
10499 case GIMPLE_OMP_SECTION:
10500 cur_region = new_omp_region (bb, code, cur_region);
10501 fallthru = true;
10502 break;
10504 case GIMPLE_OMP_TARGET:
10505 cur_region = new_omp_region (bb, code, cur_region);
10506 fallthru = true;
10507 if (gimple_omp_target_kind (last) == GF_OMP_TARGET_KIND_UPDATE)
10508 cur_region = cur_region->outer;
10509 break;
10511 case GIMPLE_OMP_SECTIONS:
10512 cur_region = new_omp_region (bb, code, cur_region);
10513 fallthru = true;
10514 break;
10516 case GIMPLE_OMP_SECTIONS_SWITCH:
10517 fallthru = false;
10518 break;
10520 case GIMPLE_OMP_ATOMIC_LOAD:
10521 case GIMPLE_OMP_ATOMIC_STORE:
10522 fallthru = true;
10523 break;
10525 case GIMPLE_OMP_RETURN:
10526 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10527 somewhere other than the next block. This will be
10528 created later. */
10529 cur_region->exit = bb;
10530 fallthru = cur_region->type != GIMPLE_OMP_SECTION;
10531 cur_region = cur_region->outer;
10532 break;
10534 case GIMPLE_OMP_CONTINUE:
10535 cur_region->cont = bb;
10536 switch (cur_region->type)
10538 case GIMPLE_OMP_FOR:
10539 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10540 succs edges as abnormal to prevent splitting
10541 them. */
10542 single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL;
10543 /* Make the loopback edge. */
10544 make_edge (bb, single_succ (cur_region->entry),
10545 EDGE_ABNORMAL);
10547 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10548 corresponds to the case that the body of the loop
10549 is not executed at all. */
10550 make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL);
10551 make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL);
10552 fallthru = false;
10553 break;
10555 case GIMPLE_OMP_SECTIONS:
10556 /* Wire up the edges into and out of the nested sections. */
10558 basic_block switch_bb = single_succ (cur_region->entry);
10560 struct omp_region *i;
10561 for (i = cur_region->inner; i ; i = i->next)
10563 gcc_assert (i->type == GIMPLE_OMP_SECTION);
10564 make_edge (switch_bb, i->entry, 0);
10565 make_edge (i->exit, bb, EDGE_FALLTHRU);
10568 /* Make the loopback edge to the block with
10569 GIMPLE_OMP_SECTIONS_SWITCH. */
10570 make_edge (bb, switch_bb, 0);
10572 /* Make the edge from the switch to exit. */
10573 make_edge (switch_bb, bb->next_bb, 0);
10574 fallthru = false;
10576 break;
10578 default:
10579 gcc_unreachable ();
10581 break;
10583 default:
10584 gcc_unreachable ();
10587 if (*region != cur_region)
10589 *region = cur_region;
10590 if (cur_region)
10591 *region_idx = cur_region->entry->index;
10592 else
10593 *region_idx = 0;
10596 return fallthru;
10599 static unsigned int
10600 diagnose_omp_structured_block_errors (void)
10602 struct walk_stmt_info wi;
10603 gimple_seq body = gimple_body (current_function_decl);
10605 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
10607 memset (&wi, 0, sizeof (wi));
10608 walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
10610 memset (&wi, 0, sizeof (wi));
10611 wi.want_locations = true;
10612 walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
10614 gimple_set_body (current_function_decl, body);
10616 splay_tree_delete (all_labels);
10617 all_labels = NULL;
10619 return 0;
10622 static bool
10623 gate_diagnose_omp_blocks (void)
10625 return flag_openmp || flag_cilkplus;
10628 namespace {
10630 const pass_data pass_data_diagnose_omp_blocks =
10632 GIMPLE_PASS, /* type */
10633 "*diagnose_omp_blocks", /* name */
10634 OPTGROUP_NONE, /* optinfo_flags */
10635 true, /* has_gate */
10636 true, /* has_execute */
10637 TV_NONE, /* tv_id */
10638 PROP_gimple_any, /* properties_required */
10639 0, /* properties_provided */
10640 0, /* properties_destroyed */
10641 0, /* todo_flags_start */
10642 0, /* todo_flags_finish */
10645 class pass_diagnose_omp_blocks : public gimple_opt_pass
10647 public:
10648 pass_diagnose_omp_blocks (gcc::context *ctxt)
10649 : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
10652 /* opt_pass methods: */
10653 bool gate () { return gate_diagnose_omp_blocks (); }
10654 unsigned int execute () {
10655 return diagnose_omp_structured_block_errors ();
10658 }; // class pass_diagnose_omp_blocks
10660 } // anon namespace
10662 gimple_opt_pass *
10663 make_pass_diagnose_omp_blocks (gcc::context *ctxt)
10665 return new pass_diagnose_omp_blocks (ctxt);
10668 /* SIMD clone supporting code. */
10670 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
10671 of arguments to reserve space for. */
10673 static struct cgraph_simd_clone *
10674 simd_clone_struct_alloc (int nargs)
10676 struct cgraph_simd_clone *clone_info;
10677 size_t len = (sizeof (struct cgraph_simd_clone)
10678 + nargs * sizeof (struct cgraph_simd_clone_arg));
10679 clone_info = (struct cgraph_simd_clone *)
10680 ggc_internal_cleared_alloc (len);
10681 return clone_info;
10684 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
10686 static inline void
10687 simd_clone_struct_copy (struct cgraph_simd_clone *to,
10688 struct cgraph_simd_clone *from)
10690 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
10691 + ((from->nargs - from->inbranch)
10692 * sizeof (struct cgraph_simd_clone_arg))));
10695 /* Return vector of parameter types of function FNDECL. This uses
10696 TYPE_ARG_TYPES if available, otherwise falls back to types of
10697 DECL_ARGUMENTS types. */
10699 vec<tree>
10700 simd_clone_vector_of_formal_parm_types (tree fndecl)
10702 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
10703 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
10704 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
10705 unsigned int i;
10706 tree arg;
10707 FOR_EACH_VEC_ELT (args, i, arg)
10708 args[i] = TREE_TYPE (args[i]);
10709 return args;
10712 /* Given a simd function in NODE, extract the simd specific
10713 information from the OMP clauses passed in CLAUSES, and return
10714 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
10715 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
10716 otherwise set to FALSE. */
10718 static struct cgraph_simd_clone *
10719 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
10720 bool *inbranch_specified)
10722 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
10723 tree t;
10724 int n;
10725 *inbranch_specified = false;
10727 n = args.length ();
10728 if (n > 0 && args.last () == void_type_node)
10729 n--;
10731 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
10732 be cloned have a distinctive artificial label in addition to "omp
10733 declare simd". */
10734 bool cilk_clone
10735 = (flag_cilkplus
10736 && lookup_attribute ("cilk simd function",
10737 DECL_ATTRIBUTES (node->decl)));
10739 /* Allocate one more than needed just in case this is an in-branch
10740 clone which will require a mask argument. */
10741 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
10742 clone_info->nargs = n;
10743 clone_info->cilk_elemental = cilk_clone;
10745 if (!clauses)
10747 args.release ();
10748 return clone_info;
10750 clauses = TREE_VALUE (clauses);
10751 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
10752 return clone_info;
10754 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
10756 switch (OMP_CLAUSE_CODE (t))
10758 case OMP_CLAUSE_INBRANCH:
10759 clone_info->inbranch = 1;
10760 *inbranch_specified = true;
10761 break;
10762 case OMP_CLAUSE_NOTINBRANCH:
10763 clone_info->inbranch = 0;
10764 *inbranch_specified = true;
10765 break;
10766 case OMP_CLAUSE_SIMDLEN:
10767 clone_info->simdlen
10768 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
10769 break;
10770 case OMP_CLAUSE_LINEAR:
10772 tree decl = OMP_CLAUSE_DECL (t);
10773 tree step = OMP_CLAUSE_LINEAR_STEP (t);
10774 int argno = TREE_INT_CST_LOW (decl);
10775 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
10777 clone_info->args[argno].arg_type
10778 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
10779 clone_info->args[argno].linear_step = tree_to_shwi (step);
10780 gcc_assert (clone_info->args[argno].linear_step >= 0
10781 && clone_info->args[argno].linear_step < n);
10783 else
10785 if (POINTER_TYPE_P (args[argno]))
10786 step = fold_convert (ssizetype, step);
10787 if (!tree_fits_shwi_p (step))
10789 warning_at (OMP_CLAUSE_LOCATION (t), 0,
10790 "ignoring large linear step");
10791 args.release ();
10792 return NULL;
10794 else if (integer_zerop (step))
10796 warning_at (OMP_CLAUSE_LOCATION (t), 0,
10797 "ignoring zero linear step");
10798 args.release ();
10799 return NULL;
10801 else
10803 clone_info->args[argno].arg_type
10804 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
10805 clone_info->args[argno].linear_step = tree_to_shwi (step);
10808 break;
10810 case OMP_CLAUSE_UNIFORM:
10812 tree decl = OMP_CLAUSE_DECL (t);
10813 int argno = tree_to_uhwi (decl);
10814 clone_info->args[argno].arg_type
10815 = SIMD_CLONE_ARG_TYPE_UNIFORM;
10816 break;
10818 case OMP_CLAUSE_ALIGNED:
10820 tree decl = OMP_CLAUSE_DECL (t);
10821 int argno = tree_to_uhwi (decl);
10822 clone_info->args[argno].alignment
10823 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
10824 break;
10826 default:
10827 break;
10830 args.release ();
10831 return clone_info;
10834 /* Given a SIMD clone in NODE, calculate the characteristic data
10835 type and return the coresponding type. The characteristic data
10836 type is computed as described in the Intel Vector ABI. */
10838 static tree
10839 simd_clone_compute_base_data_type (struct cgraph_node *node,
10840 struct cgraph_simd_clone *clone_info)
10842 tree type = integer_type_node;
10843 tree fndecl = node->decl;
10845 /* a) For non-void function, the characteristic data type is the
10846 return type. */
10847 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
10848 type = TREE_TYPE (TREE_TYPE (fndecl));
10850 /* b) If the function has any non-uniform, non-linear parameters,
10851 then the characteristic data type is the type of the first
10852 such parameter. */
10853 else
10855 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
10856 for (unsigned int i = 0; i < clone_info->nargs; ++i)
10857 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
10859 type = map[i];
10860 break;
10862 map.release ();
10865 /* c) If the characteristic data type determined by a) or b) above
10866 is struct, union, or class type which is pass-by-value (except
10867 for the type that maps to the built-in complex data type), the
10868 characteristic data type is int. */
10869 if (RECORD_OR_UNION_TYPE_P (type)
10870 && !aggregate_value_p (type, NULL)
10871 && TREE_CODE (type) != COMPLEX_TYPE)
10872 return integer_type_node;
10874 /* d) If none of the above three classes is applicable, the
10875 characteristic data type is int. */
10877 return type;
10879 /* e) For Intel Xeon Phi native and offload compilation, if the
10880 resulting characteristic data type is 8-bit or 16-bit integer
10881 data type, the characteristic data type is int. */
10882 /* Well, we don't handle Xeon Phi yet. */
10885 static tree
10886 simd_clone_mangle (struct cgraph_node *node,
10887 struct cgraph_simd_clone *clone_info)
10889 char vecsize_mangle = clone_info->vecsize_mangle;
10890 char mask = clone_info->inbranch ? 'M' : 'N';
10891 unsigned int simdlen = clone_info->simdlen;
10892 unsigned int n;
10893 pretty_printer pp;
10895 gcc_assert (vecsize_mangle && simdlen);
10897 pp_string (&pp, "_ZGV");
10898 pp_character (&pp, vecsize_mangle);
10899 pp_character (&pp, mask);
10900 pp_decimal_int (&pp, simdlen);
10902 for (n = 0; n < clone_info->nargs; ++n)
10904 struct cgraph_simd_clone_arg arg = clone_info->args[n];
10906 if (arg.arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM)
10907 pp_character (&pp, 'u');
10908 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
10910 gcc_assert (arg.linear_step != 0);
10911 pp_character (&pp, 'l');
10912 if (arg.linear_step > 1)
10913 pp_unsigned_wide_integer (&pp, arg.linear_step);
10914 else if (arg.linear_step < 0)
10916 pp_character (&pp, 'n');
10917 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
10918 arg.linear_step));
10921 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
10923 pp_character (&pp, 's');
10924 pp_unsigned_wide_integer (&pp, arg.linear_step);
10926 else
10927 pp_character (&pp, 'v');
10928 if (arg.alignment)
10930 pp_character (&pp, 'a');
10931 pp_decimal_int (&pp, arg.alignment);
10935 pp_underscore (&pp);
10936 pp_string (&pp,
10937 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)));
10938 const char *str = pp_formatted_text (&pp);
10940 /* If there already is a SIMD clone with the same mangled name, don't
10941 add another one. This can happen e.g. for
10942 #pragma omp declare simd
10943 #pragma omp declare simd simdlen(8)
10944 int foo (int, int);
10945 if the simdlen is assumed to be 8 for the first one, etc. */
10946 for (struct cgraph_node *clone = node->simd_clones; clone;
10947 clone = clone->simdclone->next_clone)
10948 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)),
10949 str) == 0)
10950 return NULL_TREE;
10952 return get_identifier (str);
10955 /* Create a simd clone of OLD_NODE and return it. */
10957 static struct cgraph_node *
10958 simd_clone_create (struct cgraph_node *old_node)
10960 struct cgraph_node *new_node;
10961 if (old_node->definition)
10963 if (!cgraph_function_with_gimple_body_p (old_node))
10964 return NULL;
10965 cgraph_get_body (old_node);
10966 new_node = cgraph_function_versioning (old_node, vNULL, NULL, NULL,
10967 false, NULL, NULL, "simdclone");
10969 else
10971 tree old_decl = old_node->decl;
10972 tree new_decl = copy_node (old_node->decl);
10973 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
10974 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
10975 SET_DECL_RTL (new_decl, NULL);
10976 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
10977 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
10978 new_node
10979 = cgraph_copy_node_for_versioning (old_node, new_decl, vNULL, NULL);
10980 cgraph_call_function_insertion_hooks (new_node);
10982 if (new_node == NULL)
10983 return new_node;
10985 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
10987 /* The function cgraph_function_versioning () will force the new
10988 symbol local. Undo this, and inherit external visability from
10989 the old node. */
10990 new_node->local.local = old_node->local.local;
10991 new_node->externally_visible = old_node->externally_visible;
10993 return new_node;
10996 /* Adjust the return type of the given function to its appropriate
10997 vector counterpart. Returns a simd array to be used throughout the
10998 function as a return value. */
11000 static tree
11001 simd_clone_adjust_return_type (struct cgraph_node *node)
11003 tree fndecl = node->decl;
11004 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
11005 unsigned int veclen;
11006 tree t;
11008 /* Adjust the function return type. */
11009 if (orig_rettype == void_type_node)
11010 return NULL_TREE;
11011 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
11012 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl)))
11013 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl))))
11014 veclen = node->simdclone->vecsize_int;
11015 else
11016 veclen = node->simdclone->vecsize_float;
11017 veclen /= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl))));
11018 if (veclen > node->simdclone->simdlen)
11019 veclen = node->simdclone->simdlen;
11020 if (veclen == node->simdclone->simdlen)
11021 TREE_TYPE (TREE_TYPE (fndecl))
11022 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl)),
11023 node->simdclone->simdlen);
11024 else
11026 t = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl)), veclen);
11027 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
11028 TREE_TYPE (TREE_TYPE (fndecl)) = t;
11030 if (!node->definition)
11031 return NULL_TREE;
11033 t = DECL_RESULT (fndecl);
11034 /* Adjust the DECL_RESULT. */
11035 gcc_assert (TREE_TYPE (t) != void_type_node);
11036 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
11037 relayout_decl (t);
11039 tree atype = build_array_type_nelts (orig_rettype,
11040 node->simdclone->simdlen);
11041 if (veclen != node->simdclone->simdlen)
11042 return build1 (VIEW_CONVERT_EXPR, atype, t);
11044 /* Set up a SIMD array to use as the return value. */
11045 tree retval = create_tmp_var_raw (atype, "retval");
11046 gimple_add_tmp_var (retval);
11047 return retval;
11050 /* Each vector argument has a corresponding array to be used locally
11051 as part of the eventual loop. Create such temporary array and
11052 return it.
11054 PREFIX is the prefix to be used for the temporary.
11056 TYPE is the inner element type.
11058 SIMDLEN is the number of elements. */
11060 static tree
11061 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
11063 tree atype = build_array_type_nelts (type, simdlen);
11064 tree avar = create_tmp_var_raw (atype, prefix);
11065 gimple_add_tmp_var (avar);
11066 return avar;
11069 /* Modify the function argument types to their corresponding vector
11070 counterparts if appropriate. Also, create one array for each simd
11071 argument to be used locally when using the function arguments as
11072 part of the loop.
11074 NODE is the function whose arguments are to be adjusted.
11076 Returns an adjustment vector that will be filled describing how the
11077 argument types will be adjusted. */
11079 static ipa_parm_adjustment_vec
11080 simd_clone_adjust_argument_types (struct cgraph_node *node)
11082 vec<tree> args;
11083 ipa_parm_adjustment_vec adjustments;
11085 if (node->definition)
11086 args = ipa_get_vector_of_formal_parms (node->decl);
11087 else
11088 args = simd_clone_vector_of_formal_parm_types (node->decl);
11089 adjustments.create (args.length ());
11090 unsigned i, j, veclen;
11091 struct ipa_parm_adjustment adj;
11092 for (i = 0; i < node->simdclone->nargs; ++i)
11094 memset (&adj, 0, sizeof (adj));
11095 tree parm = args[i];
11096 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
11097 adj.base_index = i;
11098 adj.base = parm;
11100 node->simdclone->args[i].orig_arg = node->definition ? parm : NULL_TREE;
11101 node->simdclone->args[i].orig_type = parm_type;
11103 if (node->simdclone->args[i].arg_type != SIMD_CLONE_ARG_TYPE_VECTOR)
11105 /* No adjustment necessary for scalar arguments. */
11106 adj.op = IPA_PARM_OP_COPY;
11108 else
11110 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
11111 veclen = node->simdclone->vecsize_int;
11112 else
11113 veclen = node->simdclone->vecsize_float;
11114 veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type));
11115 if (veclen > node->simdclone->simdlen)
11116 veclen = node->simdclone->simdlen;
11117 adj.arg_prefix = "simd";
11118 adj.type = build_vector_type (parm_type, veclen);
11119 node->simdclone->args[i].vector_type = adj.type;
11120 for (j = veclen; j < node->simdclone->simdlen; j += veclen)
11122 adjustments.safe_push (adj);
11123 if (j == veclen)
11125 memset (&adj, 0, sizeof (adj));
11126 adj.op = IPA_PARM_OP_NEW;
11127 adj.arg_prefix = "simd";
11128 adj.base_index = i;
11129 adj.type = node->simdclone->args[i].vector_type;
11133 if (node->definition)
11134 node->simdclone->args[i].simd_array
11135 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
11136 parm_type, node->simdclone->simdlen);
11138 adjustments.safe_push (adj);
11141 if (node->simdclone->inbranch)
11143 tree base_type
11144 = simd_clone_compute_base_data_type (node->simdclone->origin,
11145 node->simdclone);
11147 memset (&adj, 0, sizeof (adj));
11148 adj.op = IPA_PARM_OP_NEW;
11149 adj.arg_prefix = "mask";
11151 adj.base_index = i;
11152 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
11153 veclen = node->simdclone->vecsize_int;
11154 else
11155 veclen = node->simdclone->vecsize_float;
11156 veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
11157 if (veclen > node->simdclone->simdlen)
11158 veclen = node->simdclone->simdlen;
11159 adj.type = build_vector_type (base_type, veclen);
11160 adjustments.safe_push (adj);
11162 for (j = veclen; j < node->simdclone->simdlen; j += veclen)
11163 adjustments.safe_push (adj);
11165 /* We have previously allocated one extra entry for the mask. Use
11166 it and fill it. */
11167 struct cgraph_simd_clone *sc = node->simdclone;
11168 sc->nargs++;
11169 if (node->definition)
11171 sc->args[i].orig_arg
11172 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
11173 sc->args[i].simd_array
11174 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
11176 sc->args[i].orig_type = base_type;
11177 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
11180 if (node->definition)
11181 ipa_modify_formal_parameters (node->decl, adjustments);
11182 else
11184 tree new_arg_types = NULL_TREE, new_reversed;
11185 bool last_parm_void = false;
11186 if (args.length () > 0 && args.last () == void_type_node)
11187 last_parm_void = true;
11189 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
11190 j = adjustments.length ();
11191 for (i = 0; i < j; i++)
11193 struct ipa_parm_adjustment *adj = &adjustments[i];
11194 tree ptype;
11195 if (adj->op == IPA_PARM_OP_COPY)
11196 ptype = args[adj->base_index];
11197 else
11198 ptype = adj->type;
11199 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
11201 new_reversed = nreverse (new_arg_types);
11202 if (last_parm_void)
11204 if (new_reversed)
11205 TREE_CHAIN (new_arg_types) = void_list_node;
11206 else
11207 new_reversed = void_list_node;
11210 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
11211 TYPE_ARG_TYPES (new_type) = new_reversed;
11212 TREE_TYPE (node->decl) = new_type;
11214 adjustments.release ();
11216 args.release ();
11217 return adjustments;
11220 /* Initialize and copy the function arguments in NODE to their
11221 corresponding local simd arrays. Returns a fresh gimple_seq with
11222 the instruction sequence generated. */
11224 static gimple_seq
11225 simd_clone_init_simd_arrays (struct cgraph_node *node,
11226 ipa_parm_adjustment_vec adjustments)
11228 gimple_seq seq = NULL;
11229 unsigned i = 0, j = 0, k;
11231 for (tree arg = DECL_ARGUMENTS (node->decl);
11232 arg;
11233 arg = DECL_CHAIN (arg), i++, j++)
11235 if (adjustments[j].op == IPA_PARM_OP_COPY)
11236 continue;
11238 node->simdclone->args[i].vector_arg = arg;
11240 tree array = node->simdclone->args[i].simd_array;
11241 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
11243 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
11244 tree ptr = build_fold_addr_expr (array);
11245 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
11246 build_int_cst (ptype, 0));
11247 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
11248 gimplify_and_add (t, &seq);
11250 else
11252 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
11253 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
11254 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
11256 tree ptr = build_fold_addr_expr (array);
11257 int elemsize;
11258 if (k)
11260 arg = DECL_CHAIN (arg);
11261 j++;
11263 elemsize
11264 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg))));
11265 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
11266 build_int_cst (ptype, k * elemsize));
11267 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
11268 gimplify_and_add (t, &seq);
11272 return seq;
11275 /* Callback info for ipa_simd_modify_stmt_ops below. */
11277 struct modify_stmt_info {
11278 ipa_parm_adjustment_vec adjustments;
11279 gimple stmt;
11280 /* True if the parent statement was modified by
11281 ipa_simd_modify_stmt_ops. */
11282 bool modified;
11285 /* Callback for walk_gimple_op.
11287 Adjust operands from a given statement as specified in the
11288 adjustments vector in the callback data. */
11290 static tree
11291 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
11293 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
11294 if (!SSA_VAR_P (*tp))
11296 /* Make sure we treat subtrees as a RHS. This makes sure that
11297 when examining the `*foo' in *foo=x, the `foo' get treated as
11298 a use properly. */
11299 wi->is_lhs = false;
11300 wi->val_only = true;
11301 if (TYPE_P (*tp))
11302 *walk_subtrees = 0;
11303 return NULL_TREE;
11305 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
11306 struct ipa_parm_adjustment *cand
11307 = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
11308 if (!cand)
11309 return NULL_TREE;
11311 tree t = *tp;
11312 tree repl = make_ssa_name (TREE_TYPE (t), NULL);
11314 gimple stmt;
11315 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
11316 if (wi->is_lhs)
11318 stmt = gimple_build_assign (unshare_expr (cand->new_decl), repl);
11319 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
11320 SSA_NAME_DEF_STMT (repl) = info->stmt;
11322 else
11324 /* You'd think we could skip the extra SSA variable when
11325 wi->val_only=true, but we may have `*var' which will get
11326 replaced into `*var_array[iter]' and will likely be something
11327 not gimple. */
11328 stmt = gimple_build_assign (repl, unshare_expr (cand->new_decl));
11329 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
11332 if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
11334 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
11335 *tp = vce;
11337 else
11338 *tp = repl;
11340 info->modified = true;
11341 wi->is_lhs = false;
11342 wi->val_only = true;
11343 return NULL_TREE;
11346 /* Traverse the function body and perform all modifications as
11347 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11348 modified such that the replacement/reduction value will now be an
11349 offset into the corresponding simd_array.
11351 This function will replace all function argument uses with their
11352 corresponding simd array elements, and ajust the return values
11353 accordingly. */
11355 static void
11356 ipa_simd_modify_function_body (struct cgraph_node *node,
11357 ipa_parm_adjustment_vec adjustments,
11358 tree retval_array, tree iter)
11360 basic_block bb;
11361 unsigned int i, j;
11363 /* Re-use the adjustments array, but this time use it to replace
11364 every function argument use to an offset into the corresponding
11365 simd_array. */
11366 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
11368 if (!node->simdclone->args[i].vector_arg)
11369 continue;
11371 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
11372 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
11373 adjustments[j].new_decl
11374 = build4 (ARRAY_REF,
11375 basetype,
11376 node->simdclone->args[i].simd_array,
11377 iter,
11378 NULL_TREE, NULL_TREE);
11379 if (adjustments[j].op == IPA_PARM_OP_NONE
11380 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
11381 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
11384 struct modify_stmt_info info;
11385 info.adjustments = adjustments;
11387 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
11389 gimple_stmt_iterator gsi;
11391 gsi = gsi_start_bb (bb);
11392 while (!gsi_end_p (gsi))
11394 gimple stmt = gsi_stmt (gsi);
11395 info.stmt = stmt;
11396 struct walk_stmt_info wi;
11398 memset (&wi, 0, sizeof (wi));
11399 info.modified = false;
11400 wi.info = &info;
11401 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
11403 if (gimple_code (stmt) == GIMPLE_RETURN)
11405 tree retval = gimple_return_retval (stmt);
11406 if (!retval)
11408 gsi_remove (&gsi, true);
11409 continue;
11412 /* Replace `return foo' with `retval_array[iter] = foo'. */
11413 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
11414 retval_array, iter, NULL, NULL);
11415 stmt = gimple_build_assign (ref, retval);
11416 gsi_replace (&gsi, stmt, true);
11417 info.modified = true;
11420 if (info.modified)
11422 update_stmt (stmt);
11423 if (maybe_clean_eh_stmt (stmt))
11424 gimple_purge_dead_eh_edges (gimple_bb (stmt));
11426 gsi_next (&gsi);
11431 /* Adjust the argument types in NODE to their appropriate vector
11432 counterparts. */
11434 static void
11435 simd_clone_adjust (struct cgraph_node *node)
11437 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
11439 targetm.simd_clone.adjust (node);
11441 tree retval = simd_clone_adjust_return_type (node);
11442 ipa_parm_adjustment_vec adjustments
11443 = simd_clone_adjust_argument_types (node);
11445 push_gimplify_context ();
11447 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
11449 /* Adjust all uses of vector arguments accordingly. Adjust all
11450 return values accordingly. */
11451 tree iter = create_tmp_var (unsigned_type_node, "iter");
11452 tree iter1 = make_ssa_name (iter, NULL);
11453 tree iter2 = make_ssa_name (iter, NULL);
11454 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
11456 /* Initialize the iteration variable. */
11457 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
11458 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
11459 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
11460 /* Insert the SIMD array and iv initialization at function
11461 entry. */
11462 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
11464 pop_gimplify_context (NULL);
11466 /* Create a new BB right before the original exit BB, to hold the
11467 iteration increment and the condition/branch. */
11468 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
11469 basic_block incr_bb = create_empty_bb (orig_exit);
11470 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
11471 flag. Set it now to be a FALLTHRU_EDGE. */
11472 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
11473 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
11474 for (unsigned i = 0;
11475 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
11477 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
11478 redirect_edge_succ (e, incr_bb);
11480 edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
11481 e->probability = REG_BR_PROB_BASE;
11482 gsi = gsi_last_bb (incr_bb);
11483 gimple g = gimple_build_assign_with_ops (PLUS_EXPR, iter2, iter1,
11484 build_int_cst (unsigned_type_node,
11485 1));
11486 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11488 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
11489 struct loop *loop = alloc_loop ();
11490 cfun->has_force_vect_loops = true;
11491 loop->safelen = node->simdclone->simdlen;
11492 loop->force_vect = true;
11493 loop->header = body_bb;
11494 add_bb_to_loop (incr_bb, loop);
11496 /* Branch around the body if the mask applies. */
11497 if (node->simdclone->inbranch)
11499 gimple_stmt_iterator gsi = gsi_last_bb (loop->header);
11500 tree mask_array
11501 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
11502 tree mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)), NULL);
11503 tree aref = build4 (ARRAY_REF,
11504 TREE_TYPE (TREE_TYPE (mask_array)),
11505 mask_array, iter1,
11506 NULL, NULL);
11507 g = gimple_build_assign (mask, aref);
11508 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11509 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref)));
11510 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
11512 aref = build1 (VIEW_CONVERT_EXPR,
11513 build_nonstandard_integer_type (bitsize, 0), mask);
11514 mask = make_ssa_name (TREE_TYPE (aref), NULL);
11515 g = gimple_build_assign (mask, aref);
11516 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11519 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
11520 NULL, NULL);
11521 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11522 make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
11523 FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE;
11526 /* Generate the condition. */
11527 g = gimple_build_cond (LT_EXPR,
11528 iter2,
11529 build_int_cst (unsigned_type_node,
11530 node->simdclone->simdlen),
11531 NULL, NULL);
11532 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11533 e = split_block (incr_bb, gsi_stmt (gsi));
11534 basic_block latch_bb = e->dest;
11535 basic_block new_exit_bb = e->dest;
11536 new_exit_bb = split_block (latch_bb, NULL)->dest;
11537 loop->latch = latch_bb;
11539 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
11541 make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
11542 /* The successor of incr_bb is already pointing to latch_bb; just
11543 change the flags.
11544 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
11545 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
11547 gimple phi = create_phi_node (iter1, body_bb);
11548 edge preheader_edge = find_edge (entry_bb, body_bb);
11549 edge latch_edge = single_succ_edge (latch_bb);
11550 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
11551 UNKNOWN_LOCATION);
11552 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
11554 /* Generate the new return. */
11555 gsi = gsi_last_bb (new_exit_bb);
11556 if (retval
11557 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
11558 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
11559 retval = TREE_OPERAND (retval, 0);
11560 else if (retval)
11562 retval = build1 (VIEW_CONVERT_EXPR,
11563 TREE_TYPE (TREE_TYPE (node->decl)),
11564 retval);
11565 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
11566 false, GSI_CONTINUE_LINKING);
11568 g = gimple_build_return (retval);
11569 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
11571 /* Handle aligned clauses by replacing default defs of the aligned
11572 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
11573 lhs. Handle linear by adding PHIs. */
11574 for (unsigned i = 0; i < node->simdclone->nargs; i++)
11575 if (node->simdclone->args[i].alignment
11576 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
11577 && (node->simdclone->args[i].alignment
11578 & (node->simdclone->args[i].alignment - 1)) == 0
11579 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
11580 == POINTER_TYPE)
11582 unsigned int alignment = node->simdclone->args[i].alignment;
11583 tree orig_arg = node->simdclone->args[i].orig_arg;
11584 tree def = ssa_default_def (cfun, orig_arg);
11585 if (def && !has_zero_uses (def))
11587 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
11588 gimple_seq seq = NULL;
11589 bool need_cvt = false;
11590 gimple call
11591 = gimple_build_call (fn, 2, def, size_int (alignment));
11592 g = call;
11593 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
11594 ptr_type_node))
11595 need_cvt = true;
11596 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg, NULL);
11597 gimple_call_set_lhs (g, t);
11598 gimple_seq_add_stmt_without_update (&seq, g);
11599 if (need_cvt)
11601 t = make_ssa_name (orig_arg, NULL);
11602 g = gimple_build_assign_with_ops (NOP_EXPR, t,
11603 gimple_call_lhs (g),
11604 NULL_TREE);
11605 gimple_seq_add_stmt_without_update (&seq, g);
11607 gsi_insert_seq_on_edge_immediate
11608 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
11610 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
11611 int freq = compute_call_stmt_bb_frequency (current_function_decl,
11612 entry_bb);
11613 cgraph_create_edge (node, cgraph_get_create_node (fn),
11614 call, entry_bb->count, freq);
11616 imm_use_iterator iter;
11617 use_operand_p use_p;
11618 gimple use_stmt;
11619 tree repl = gimple_get_lhs (g);
11620 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
11621 if (is_gimple_debug (use_stmt) || use_stmt == call)
11622 continue;
11623 else
11624 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
11625 SET_USE (use_p, repl);
11628 else if (node->simdclone->args[i].arg_type
11629 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
11631 tree orig_arg = node->simdclone->args[i].orig_arg;
11632 tree def = ssa_default_def (cfun, orig_arg);
11633 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
11634 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
11635 if (def && !has_zero_uses (def))
11637 iter1 = make_ssa_name (orig_arg, NULL);
11638 iter2 = make_ssa_name (orig_arg, NULL);
11639 phi = create_phi_node (iter1, body_bb);
11640 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
11641 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
11642 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
11643 ? PLUS_EXPR : POINTER_PLUS_EXPR;
11644 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
11645 ? TREE_TYPE (orig_arg) : sizetype;
11646 tree addcst
11647 = build_int_cst (addtype, node->simdclone->args[i].linear_step);
11648 g = gimple_build_assign_with_ops (code, iter2, iter1, addcst);
11649 gsi = gsi_last_bb (incr_bb);
11650 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
11652 imm_use_iterator iter;
11653 use_operand_p use_p;
11654 gimple use_stmt;
11655 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
11656 if (use_stmt == phi)
11657 continue;
11658 else
11659 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
11660 SET_USE (use_p, iter1);
11664 calculate_dominance_info (CDI_DOMINATORS);
11665 add_loop (loop, loop->header->loop_father);
11666 update_ssa (TODO_update_ssa);
11668 pop_cfun ();
11671 /* If the function in NODE is tagged as an elemental SIMD function,
11672 create the appropriate SIMD clones. */
11674 static void
11675 expand_simd_clones (struct cgraph_node *node)
11677 tree attr = lookup_attribute ("omp declare simd",
11678 DECL_ATTRIBUTES (node->decl));
11679 if (attr == NULL_TREE
11680 || node->global.inlined_to
11681 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
11682 return;
11684 /* Ignore
11685 #pragma omp declare simd
11686 extern int foo ();
11687 in C, there we don't know the argument types at all. */
11688 if (!node->definition
11689 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
11690 return;
11694 /* Start with parsing the "omp declare simd" attribute(s). */
11695 bool inbranch_clause_specified;
11696 struct cgraph_simd_clone *clone_info
11697 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
11698 &inbranch_clause_specified);
11699 if (clone_info == NULL)
11700 continue;
11702 int orig_simdlen = clone_info->simdlen;
11703 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
11704 /* The target can return 0 (no simd clones should be created),
11705 1 (just one ISA of simd clones should be created) or higher
11706 count of ISA variants. In that case, clone_info is initialized
11707 for the first ISA variant. */
11708 int count
11709 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
11710 base_type, 0);
11711 if (count == 0)
11712 continue;
11714 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
11715 also create one inbranch and one !inbranch clone of it. */
11716 for (int i = 0; i < count * 2; i++)
11718 struct cgraph_simd_clone *clone = clone_info;
11719 if (inbranch_clause_specified && (i & 1) != 0)
11720 continue;
11722 if (i != 0)
11724 clone = simd_clone_struct_alloc (clone_info->nargs
11725 + ((i & 1) != 0));
11726 simd_clone_struct_copy (clone, clone_info);
11727 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
11728 and simd_clone_adjust_argument_types did to the first
11729 clone's info. */
11730 clone->nargs -= clone_info->inbranch;
11731 clone->simdlen = orig_simdlen;
11732 /* And call the target hook again to get the right ISA. */
11733 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
11734 base_type,
11735 i / 2);
11736 if ((i & 1) != 0)
11737 clone->inbranch = 1;
11740 /* simd_clone_mangle might fail if such a clone has been created
11741 already. */
11742 tree id = simd_clone_mangle (node, clone);
11743 if (id == NULL_TREE)
11744 continue;
11746 /* Only when we are sure we want to create the clone actually
11747 clone the function (or definitions) or create another
11748 extern FUNCTION_DECL (for prototypes without definitions). */
11749 struct cgraph_node *n = simd_clone_create (node);
11750 if (n == NULL)
11751 continue;
11753 n->simdclone = clone;
11754 clone->origin = node;
11755 clone->next_clone = NULL;
11756 if (node->simd_clones == NULL)
11758 clone->prev_clone = n;
11759 node->simd_clones = n;
11761 else
11763 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
11764 clone->prev_clone->simdclone->next_clone = n;
11765 node->simd_clones->simdclone->prev_clone = n;
11767 change_decl_assembler_name (n->decl, id);
11768 /* And finally adjust the return type, parameters and for
11769 definitions also function body. */
11770 if (node->definition)
11771 simd_clone_adjust (n);
11772 else
11774 simd_clone_adjust_return_type (n);
11775 simd_clone_adjust_argument_types (n);
11779 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
11782 /* Entry point for IPA simd clone creation pass. */
11784 static unsigned int
11785 ipa_omp_simd_clone (void)
11787 struct cgraph_node *node;
11788 FOR_EACH_FUNCTION (node)
11789 expand_simd_clones (node);
11790 return 0;
11793 namespace {
11795 const pass_data pass_data_omp_simd_clone =
11797 SIMPLE_IPA_PASS, /* type */
11798 "simdclone", /* name */
11799 OPTGROUP_NONE, /* optinfo_flags */
11800 true, /* has_gate */
11801 true, /* has_execute */
11802 TV_NONE, /* tv_id */
11803 ( PROP_ssa | PROP_cfg ), /* properties_required */
11804 0, /* properties_provided */
11805 0, /* properties_destroyed */
11806 0, /* todo_flags_start */
11807 0, /* todo_flags_finish */
11810 class pass_omp_simd_clone : public simple_ipa_opt_pass
11812 public:
11813 pass_omp_simd_clone(gcc::context *ctxt)
11814 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
11817 /* opt_pass methods: */
11818 bool gate () { return ((flag_openmp || flag_openmp_simd
11819 || flag_cilkplus || (in_lto_p && !flag_wpa))
11820 && (targetm.simd_clone.compute_vecsize_and_simdlen
11821 != NULL)); }
11822 unsigned int execute () { return ipa_omp_simd_clone (); }
11825 } // anon namespace
11827 simple_ipa_opt_pass *
11828 make_pass_omp_simd_clone (gcc::context *ctxt)
11830 return new pass_omp_simd_clone (ctxt);
11833 #include "gt-omp-low.h"