2013-11-21 Edward Smith-Rowland <3dw4rd@verizon.net>
[official-gcc.git] / gcc / omp-low.c
blob67221556f69ef434870303b01c6666dde24ec243
1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2013 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
13 version.
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 for more details.
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
24 #include "config.h"
25 #include "system.h"
26 #include "coretypes.h"
27 #include "tm.h"
28 #include "tree.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
31 #include "rtl.h"
32 #include "gimple.h"
33 #include "gimplify.h"
34 #include "gimple-iterator.h"
35 #include "gimplify-me.h"
36 #include "gimple-walk.h"
37 #include "tree-iterator.h"
38 #include "tree-inline.h"
39 #include "langhooks.h"
40 #include "diagnostic-core.h"
41 #include "gimple-ssa.h"
42 #include "cgraph.h"
43 #include "tree-cfg.h"
44 #include "tree-phinodes.h"
45 #include "ssa-iterators.h"
46 #include "tree-ssanames.h"
47 #include "tree-into-ssa.h"
48 #include "expr.h"
49 #include "tree-dfa.h"
50 #include "tree-ssa.h"
51 #include "flags.h"
52 #include "function.h"
53 #include "expr.h"
54 #include "tree-pass.h"
55 #include "ggc.h"
56 #include "except.h"
57 #include "splay-tree.h"
58 #include "optabs.h"
59 #include "cfgloop.h"
60 #include "target.h"
61 #include "omp-low.h"
62 #include "gimple-low.h"
63 #include "tree-cfgcleanup.h"
64 #include "tree-nested.h"
67 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
68 phases. The first phase scans the function looking for OMP statements
69 and then for variables that must be replaced to satisfy data sharing
70 clauses. The second phase expands code for the constructs, as well as
71 re-gimplifying things when variables have been replaced with complex
72 expressions.
74 Final code generation is done by pass_expand_omp. The flowgraph is
75 scanned for parallel regions which are then moved to a new
76 function, to be invoked by the thread library. */
78 /* Parallel region information. Every parallel and workshare
79 directive is enclosed between two markers, the OMP_* directive
80 and a corresponding OMP_RETURN statement. */
82 struct omp_region
84 /* The enclosing region. */
85 struct omp_region *outer;
87 /* First child region. */
88 struct omp_region *inner;
90 /* Next peer region. */
91 struct omp_region *next;
93 /* Block containing the omp directive as its last stmt. */
94 basic_block entry;
96 /* Block containing the OMP_RETURN as its last stmt. */
97 basic_block exit;
99 /* Block containing the OMP_CONTINUE as its last stmt. */
100 basic_block cont;
102 /* If this is a combined parallel+workshare region, this is a list
103 of additional arguments needed by the combined parallel+workshare
104 library call. */
105 vec<tree, va_gc> *ws_args;
107 /* The code for the omp directive of this region. */
108 enum gimple_code type;
110 /* Schedule kind, only used for OMP_FOR type regions. */
111 enum omp_clause_schedule_kind sched_kind;
113 /* True if this is a combined parallel+workshare region. */
114 bool is_combined_parallel;
117 /* Context structure. Used to store information about each parallel
118 directive in the code. */
120 typedef struct omp_context
122 /* This field must be at the beginning, as we do "inheritance": Some
123 callback functions for tree-inline.c (e.g., omp_copy_decl)
124 receive a copy_body_data pointer that is up-casted to an
125 omp_context pointer. */
126 copy_body_data cb;
128 /* The tree of contexts corresponding to the encountered constructs. */
129 struct omp_context *outer;
130 gimple stmt;
132 /* Map variables to fields in a structure that allows communication
133 between sending and receiving threads. */
134 splay_tree field_map;
135 tree record_type;
136 tree sender_decl;
137 tree receiver_decl;
139 /* These are used just by task contexts, if task firstprivate fn is
140 needed. srecord_type is used to communicate from the thread
141 that encountered the task construct to task firstprivate fn,
142 record_type is allocated by GOMP_task, initialized by task firstprivate
143 fn and passed to the task body fn. */
144 splay_tree sfield_map;
145 tree srecord_type;
147 /* A chain of variables to add to the top-level block surrounding the
148 construct. In the case of a parallel, this is in the child function. */
149 tree block_vars;
151 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
152 barriers should jump to during omplower pass. */
153 tree cancel_label;
155 /* What to do with variables with implicitly determined sharing
156 attributes. */
157 enum omp_clause_default_kind default_kind;
159 /* Nesting depth of this context. Used to beautify error messages re
160 invalid gotos. The outermost ctx is depth 1, with depth 0 being
161 reserved for the main body of the function. */
162 int depth;
164 /* True if this parallel directive is nested within another. */
165 bool is_nested;
167 /* True if this construct can be cancelled. */
168 bool cancellable;
169 } omp_context;
172 struct omp_for_data_loop
174 tree v, n1, n2, step;
175 enum tree_code cond_code;
178 /* A structure describing the main elements of a parallel loop. */
180 struct omp_for_data
182 struct omp_for_data_loop loop;
183 tree chunk_size;
184 gimple for_stmt;
185 tree pre, iter_type;
186 int collapse;
187 bool have_nowait, have_ordered;
188 enum omp_clause_schedule_kind sched_kind;
189 struct omp_for_data_loop *loops;
193 static splay_tree all_contexts;
194 static int taskreg_nesting_level;
195 static int target_nesting_level;
196 static struct omp_region *root_omp_region;
197 static bitmap task_shared_vars;
199 static void scan_omp (gimple_seq *, omp_context *);
200 static tree scan_omp_1_op (tree *, int *, void *);
202 #define WALK_SUBSTMTS \
203 case GIMPLE_BIND: \
204 case GIMPLE_TRY: \
205 case GIMPLE_CATCH: \
206 case GIMPLE_EH_FILTER: \
207 case GIMPLE_TRANSACTION: \
208 /* The sub-statements for these should be walked. */ \
209 *handled_ops_p = false; \
210 break;
212 /* Convenience function for calling scan_omp_1_op on tree operands. */
214 static inline tree
215 scan_omp_op (tree *tp, omp_context *ctx)
217 struct walk_stmt_info wi;
219 memset (&wi, 0, sizeof (wi));
220 wi.info = ctx;
221 wi.want_locations = true;
223 return walk_tree (tp, scan_omp_1_op, &wi, NULL);
226 static void lower_omp (gimple_seq *, omp_context *);
227 static tree lookup_decl_in_outer_ctx (tree, omp_context *);
228 static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
230 /* Find an OpenMP clause of type KIND within CLAUSES. */
232 tree
233 find_omp_clause (tree clauses, enum omp_clause_code kind)
235 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
236 if (OMP_CLAUSE_CODE (clauses) == kind)
237 return clauses;
239 return NULL_TREE;
242 /* Return true if CTX is for an omp parallel. */
244 static inline bool
245 is_parallel_ctx (omp_context *ctx)
247 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
251 /* Return true if CTX is for an omp task. */
253 static inline bool
254 is_task_ctx (omp_context *ctx)
256 return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
260 /* Return true if CTX is for an omp parallel or omp task. */
262 static inline bool
263 is_taskreg_ctx (omp_context *ctx)
265 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL
266 || gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
270 /* Return true if REGION is a combined parallel+workshare region. */
272 static inline bool
273 is_combined_parallel (struct omp_region *region)
275 return region->is_combined_parallel;
279 /* Extract the header elements of parallel loop FOR_STMT and store
280 them into *FD. */
282 static void
283 extract_omp_for_data (gimple for_stmt, struct omp_for_data *fd,
284 struct omp_for_data_loop *loops)
286 tree t, var, *collapse_iter, *collapse_count;
287 tree count = NULL_TREE, iter_type = long_integer_type_node;
288 struct omp_for_data_loop *loop;
289 int i;
290 struct omp_for_data_loop dummy_loop;
291 location_t loc = gimple_location (for_stmt);
292 bool simd = gimple_omp_for_kind (for_stmt) & GF_OMP_FOR_KIND_SIMD;
293 bool distribute = gimple_omp_for_kind (for_stmt)
294 == GF_OMP_FOR_KIND_DISTRIBUTE;
296 fd->for_stmt = for_stmt;
297 fd->pre = NULL;
298 fd->collapse = gimple_omp_for_collapse (for_stmt);
299 if (fd->collapse > 1)
300 fd->loops = loops;
301 else
302 fd->loops = &fd->loop;
304 fd->have_nowait = distribute || simd;
305 fd->have_ordered = false;
306 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
307 fd->chunk_size = NULL_TREE;
308 collapse_iter = NULL;
309 collapse_count = NULL;
311 for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t))
312 switch (OMP_CLAUSE_CODE (t))
314 case OMP_CLAUSE_NOWAIT:
315 fd->have_nowait = true;
316 break;
317 case OMP_CLAUSE_ORDERED:
318 fd->have_ordered = true;
319 break;
320 case OMP_CLAUSE_SCHEDULE:
321 gcc_assert (!distribute);
322 fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t);
323 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t);
324 break;
325 case OMP_CLAUSE_DIST_SCHEDULE:
326 gcc_assert (distribute);
327 fd->chunk_size = OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t);
328 break;
329 case OMP_CLAUSE_COLLAPSE:
330 if (fd->collapse > 1)
332 collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t);
333 collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t);
335 default:
336 break;
339 /* FIXME: for now map schedule(auto) to schedule(static).
340 There should be analysis to determine whether all iterations
341 are approximately the same amount of work (then schedule(static)
342 is best) or if it varies (then schedule(dynamic,N) is better). */
343 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO)
345 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
346 gcc_assert (fd->chunk_size == NULL);
348 gcc_assert (fd->collapse == 1 || collapse_iter != NULL);
349 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
350 gcc_assert (fd->chunk_size == NULL);
351 else if (fd->chunk_size == NULL)
353 /* We only need to compute a default chunk size for ordered
354 static loops and dynamic loops. */
355 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
356 || fd->have_ordered)
357 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
358 ? integer_zero_node : integer_one_node;
361 for (i = 0; i < fd->collapse; i++)
363 if (fd->collapse == 1)
364 loop = &fd->loop;
365 else if (loops != NULL)
366 loop = loops + i;
367 else
368 loop = &dummy_loop;
370 loop->v = gimple_omp_for_index (for_stmt, i);
371 gcc_assert (SSA_VAR_P (loop->v));
372 gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
373 || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE);
374 var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v;
375 loop->n1 = gimple_omp_for_initial (for_stmt, i);
377 loop->cond_code = gimple_omp_for_cond (for_stmt, i);
378 loop->n2 = gimple_omp_for_final (for_stmt, i);
379 switch (loop->cond_code)
381 case LT_EXPR:
382 case GT_EXPR:
383 break;
384 case NE_EXPR:
385 gcc_assert (gimple_omp_for_kind (for_stmt)
386 == GF_OMP_FOR_KIND_CILKSIMD);
387 break;
388 case LE_EXPR:
389 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
390 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, 1);
391 else
392 loop->n2 = fold_build2_loc (loc,
393 PLUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
394 build_int_cst (TREE_TYPE (loop->n2), 1));
395 loop->cond_code = LT_EXPR;
396 break;
397 case GE_EXPR:
398 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
399 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, -1);
400 else
401 loop->n2 = fold_build2_loc (loc,
402 MINUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
403 build_int_cst (TREE_TYPE (loop->n2), 1));
404 loop->cond_code = GT_EXPR;
405 break;
406 default:
407 gcc_unreachable ();
410 t = gimple_omp_for_incr (for_stmt, i);
411 gcc_assert (TREE_OPERAND (t, 0) == var);
412 switch (TREE_CODE (t))
414 case PLUS_EXPR:
415 loop->step = TREE_OPERAND (t, 1);
416 break;
417 case POINTER_PLUS_EXPR:
418 loop->step = fold_convert (ssizetype, TREE_OPERAND (t, 1));
419 break;
420 case MINUS_EXPR:
421 loop->step = TREE_OPERAND (t, 1);
422 loop->step = fold_build1_loc (loc,
423 NEGATE_EXPR, TREE_TYPE (loop->step),
424 loop->step);
425 break;
426 default:
427 gcc_unreachable ();
430 if (simd
431 || (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
432 && !fd->have_ordered))
434 if (fd->collapse == 1)
435 iter_type = TREE_TYPE (loop->v);
436 else if (i == 0
437 || TYPE_PRECISION (iter_type)
438 < TYPE_PRECISION (TREE_TYPE (loop->v)))
439 iter_type
440 = build_nonstandard_integer_type
441 (TYPE_PRECISION (TREE_TYPE (loop->v)), 1);
443 else if (iter_type != long_long_unsigned_type_node)
445 if (POINTER_TYPE_P (TREE_TYPE (loop->v)))
446 iter_type = long_long_unsigned_type_node;
447 else if (TYPE_UNSIGNED (TREE_TYPE (loop->v))
448 && TYPE_PRECISION (TREE_TYPE (loop->v))
449 >= TYPE_PRECISION (iter_type))
451 tree n;
453 if (loop->cond_code == LT_EXPR)
454 n = fold_build2_loc (loc,
455 PLUS_EXPR, TREE_TYPE (loop->v),
456 loop->n2, loop->step);
457 else
458 n = loop->n1;
459 if (TREE_CODE (n) != INTEGER_CST
460 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n))
461 iter_type = long_long_unsigned_type_node;
463 else if (TYPE_PRECISION (TREE_TYPE (loop->v))
464 > TYPE_PRECISION (iter_type))
466 tree n1, n2;
468 if (loop->cond_code == LT_EXPR)
470 n1 = loop->n1;
471 n2 = fold_build2_loc (loc,
472 PLUS_EXPR, TREE_TYPE (loop->v),
473 loop->n2, loop->step);
475 else
477 n1 = fold_build2_loc (loc,
478 MINUS_EXPR, TREE_TYPE (loop->v),
479 loop->n2, loop->step);
480 n2 = loop->n1;
482 if (TREE_CODE (n1) != INTEGER_CST
483 || TREE_CODE (n2) != INTEGER_CST
484 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1)
485 || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type)))
486 iter_type = long_long_unsigned_type_node;
490 if (collapse_count && *collapse_count == NULL)
492 t = fold_binary (loop->cond_code, boolean_type_node,
493 fold_convert (TREE_TYPE (loop->v), loop->n1),
494 fold_convert (TREE_TYPE (loop->v), loop->n2));
495 if (t && integer_zerop (t))
496 count = build_zero_cst (long_long_unsigned_type_node);
497 else if ((i == 0 || count != NULL_TREE)
498 && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
499 && TREE_CONSTANT (loop->n1)
500 && TREE_CONSTANT (loop->n2)
501 && TREE_CODE (loop->step) == INTEGER_CST)
503 tree itype = TREE_TYPE (loop->v);
505 if (POINTER_TYPE_P (itype))
506 itype = signed_type_for (itype);
507 t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1));
508 t = fold_build2_loc (loc,
509 PLUS_EXPR, itype,
510 fold_convert_loc (loc, itype, loop->step), t);
511 t = fold_build2_loc (loc, PLUS_EXPR, itype, t,
512 fold_convert_loc (loc, itype, loop->n2));
513 t = fold_build2_loc (loc, MINUS_EXPR, itype, t,
514 fold_convert_loc (loc, itype, loop->n1));
515 if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR)
516 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype,
517 fold_build1_loc (loc, NEGATE_EXPR, itype, t),
518 fold_build1_loc (loc, NEGATE_EXPR, itype,
519 fold_convert_loc (loc, itype,
520 loop->step)));
521 else
522 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t,
523 fold_convert_loc (loc, itype, loop->step));
524 t = fold_convert_loc (loc, long_long_unsigned_type_node, t);
525 if (count != NULL_TREE)
526 count = fold_build2_loc (loc,
527 MULT_EXPR, long_long_unsigned_type_node,
528 count, t);
529 else
530 count = t;
531 if (TREE_CODE (count) != INTEGER_CST)
532 count = NULL_TREE;
534 else if (count && !integer_zerop (count))
535 count = NULL_TREE;
539 if (count
540 && !simd
541 && (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
542 || fd->have_ordered))
544 if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node)))
545 iter_type = long_long_unsigned_type_node;
546 else
547 iter_type = long_integer_type_node;
549 else if (collapse_iter && *collapse_iter != NULL)
550 iter_type = TREE_TYPE (*collapse_iter);
551 fd->iter_type = iter_type;
552 if (collapse_iter && *collapse_iter == NULL)
553 *collapse_iter = create_tmp_var (iter_type, ".iter");
554 if (collapse_count && *collapse_count == NULL)
556 if (count)
557 *collapse_count = fold_convert_loc (loc, iter_type, count);
558 else
559 *collapse_count = create_tmp_var (iter_type, ".count");
562 if (fd->collapse > 1)
564 fd->loop.v = *collapse_iter;
565 fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0);
566 fd->loop.n2 = *collapse_count;
567 fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1);
568 fd->loop.cond_code = LT_EXPR;
573 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
574 is the immediate dominator of PAR_ENTRY_BB, return true if there
575 are no data dependencies that would prevent expanding the parallel
576 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
578 When expanding a combined parallel+workshare region, the call to
579 the child function may need additional arguments in the case of
580 GIMPLE_OMP_FOR regions. In some cases, these arguments are
581 computed out of variables passed in from the parent to the child
582 via 'struct .omp_data_s'. For instance:
584 #pragma omp parallel for schedule (guided, i * 4)
585 for (j ...)
587 Is lowered into:
589 # BLOCK 2 (PAR_ENTRY_BB)
590 .omp_data_o.i = i;
591 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
593 # BLOCK 3 (WS_ENTRY_BB)
594 .omp_data_i = &.omp_data_o;
595 D.1667 = .omp_data_i->i;
596 D.1598 = D.1667 * 4;
597 #pragma omp for schedule (guided, D.1598)
599 When we outline the parallel region, the call to the child function
600 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
601 that value is computed *after* the call site. So, in principle we
602 cannot do the transformation.
604 To see whether the code in WS_ENTRY_BB blocks the combined
605 parallel+workshare call, we collect all the variables used in the
606 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
607 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
608 call.
610 FIXME. If we had the SSA form built at this point, we could merely
611 hoist the code in block 3 into block 2 and be done with it. But at
612 this point we don't have dataflow information and though we could
613 hack something up here, it is really not worth the aggravation. */
615 static bool
616 workshare_safe_to_combine_p (basic_block ws_entry_bb)
618 struct omp_for_data fd;
619 gimple ws_stmt = last_stmt (ws_entry_bb);
621 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
622 return true;
624 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
626 extract_omp_for_data (ws_stmt, &fd, NULL);
628 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
629 return false;
630 if (fd.iter_type != long_integer_type_node)
631 return false;
633 /* FIXME. We give up too easily here. If any of these arguments
634 are not constants, they will likely involve variables that have
635 been mapped into fields of .omp_data_s for sharing with the child
636 function. With appropriate data flow, it would be possible to
637 see through this. */
638 if (!is_gimple_min_invariant (fd.loop.n1)
639 || !is_gimple_min_invariant (fd.loop.n2)
640 || !is_gimple_min_invariant (fd.loop.step)
641 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
642 return false;
644 return true;
648 /* Collect additional arguments needed to emit a combined
649 parallel+workshare call. WS_STMT is the workshare directive being
650 expanded. */
652 static vec<tree, va_gc> *
653 get_ws_args_for (gimple par_stmt, gimple ws_stmt)
655 tree t;
656 location_t loc = gimple_location (ws_stmt);
657 vec<tree, va_gc> *ws_args;
659 if (gimple_code (ws_stmt) == GIMPLE_OMP_FOR)
661 struct omp_for_data fd;
662 tree n1, n2;
664 extract_omp_for_data (ws_stmt, &fd, NULL);
665 n1 = fd.loop.n1;
666 n2 = fd.loop.n2;
668 if (gimple_omp_for_combined_into_p (ws_stmt))
670 tree innerc
671 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt),
672 OMP_CLAUSE__LOOPTEMP_);
673 gcc_assert (innerc);
674 n1 = OMP_CLAUSE_DECL (innerc);
675 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
676 OMP_CLAUSE__LOOPTEMP_);
677 gcc_assert (innerc);
678 n2 = OMP_CLAUSE_DECL (innerc);
681 vec_alloc (ws_args, 3 + (fd.chunk_size != 0));
683 t = fold_convert_loc (loc, long_integer_type_node, n1);
684 ws_args->quick_push (t);
686 t = fold_convert_loc (loc, long_integer_type_node, n2);
687 ws_args->quick_push (t);
689 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step);
690 ws_args->quick_push (t);
692 if (fd.chunk_size)
694 t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size);
695 ws_args->quick_push (t);
698 return ws_args;
700 else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
702 /* Number of sections is equal to the number of edges from the
703 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
704 the exit of the sections region. */
705 basic_block bb = single_succ (gimple_bb (ws_stmt));
706 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
707 vec_alloc (ws_args, 1);
708 ws_args->quick_push (t);
709 return ws_args;
712 gcc_unreachable ();
716 /* Discover whether REGION is a combined parallel+workshare region. */
718 static void
719 determine_parallel_type (struct omp_region *region)
721 basic_block par_entry_bb, par_exit_bb;
722 basic_block ws_entry_bb, ws_exit_bb;
724 if (region == NULL || region->inner == NULL
725 || region->exit == NULL || region->inner->exit == NULL
726 || region->inner->cont == NULL)
727 return;
729 /* We only support parallel+for and parallel+sections. */
730 if (region->type != GIMPLE_OMP_PARALLEL
731 || (region->inner->type != GIMPLE_OMP_FOR
732 && region->inner->type != GIMPLE_OMP_SECTIONS))
733 return;
735 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
736 WS_EXIT_BB -> PAR_EXIT_BB. */
737 par_entry_bb = region->entry;
738 par_exit_bb = region->exit;
739 ws_entry_bb = region->inner->entry;
740 ws_exit_bb = region->inner->exit;
742 if (single_succ (par_entry_bb) == ws_entry_bb
743 && single_succ (ws_exit_bb) == par_exit_bb
744 && workshare_safe_to_combine_p (ws_entry_bb)
745 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb))
746 || (last_and_only_stmt (ws_entry_bb)
747 && last_and_only_stmt (par_exit_bb))))
749 gimple par_stmt = last_stmt (par_entry_bb);
750 gimple ws_stmt = last_stmt (ws_entry_bb);
752 if (region->inner->type == GIMPLE_OMP_FOR)
754 /* If this is a combined parallel loop, we need to determine
755 whether or not to use the combined library calls. There
756 are two cases where we do not apply the transformation:
757 static loops and any kind of ordered loop. In the first
758 case, we already open code the loop so there is no need
759 to do anything else. In the latter case, the combined
760 parallel loop call would still need extra synchronization
761 to implement ordered semantics, so there would not be any
762 gain in using the combined call. */
763 tree clauses = gimple_omp_for_clauses (ws_stmt);
764 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE);
765 if (c == NULL
766 || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC
767 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED))
769 region->is_combined_parallel = false;
770 region->inner->is_combined_parallel = false;
771 return;
775 region->is_combined_parallel = true;
776 region->inner->is_combined_parallel = true;
777 region->ws_args = get_ws_args_for (par_stmt, ws_stmt);
782 /* Return true if EXPR is variable sized. */
784 static inline bool
785 is_variable_sized (const_tree expr)
787 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
790 /* Return true if DECL is a reference type. */
792 static inline bool
793 is_reference (tree decl)
795 return lang_hooks.decls.omp_privatize_by_reference (decl);
798 /* Lookup variables in the decl or field splay trees. The "maybe" form
799 allows for the variable form to not have been entered, otherwise we
800 assert that the variable must have been entered. */
802 static inline tree
803 lookup_decl (tree var, omp_context *ctx)
805 tree *n;
806 n = (tree *) pointer_map_contains (ctx->cb.decl_map, var);
807 return *n;
810 static inline tree
811 maybe_lookup_decl (const_tree var, omp_context *ctx)
813 tree *n;
814 n = (tree *) pointer_map_contains (ctx->cb.decl_map, var);
815 return n ? *n : NULL_TREE;
818 static inline tree
819 lookup_field (tree var, omp_context *ctx)
821 splay_tree_node n;
822 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
823 return (tree) n->value;
826 static inline tree
827 lookup_sfield (tree var, omp_context *ctx)
829 splay_tree_node n;
830 n = splay_tree_lookup (ctx->sfield_map
831 ? ctx->sfield_map : ctx->field_map,
832 (splay_tree_key) var);
833 return (tree) n->value;
836 static inline tree
837 maybe_lookup_field (tree var, omp_context *ctx)
839 splay_tree_node n;
840 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
841 return n ? (tree) n->value : NULL_TREE;
844 /* Return true if DECL should be copied by pointer. SHARED_CTX is
845 the parallel context if DECL is to be shared. */
847 static bool
848 use_pointer_for_field (tree decl, omp_context *shared_ctx)
850 if (AGGREGATE_TYPE_P (TREE_TYPE (decl)))
851 return true;
853 /* We can only use copy-in/copy-out semantics for shared variables
854 when we know the value is not accessible from an outer scope. */
855 if (shared_ctx)
857 /* ??? Trivially accessible from anywhere. But why would we even
858 be passing an address in this case? Should we simply assert
859 this to be false, or should we have a cleanup pass that removes
860 these from the list of mappings? */
861 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
862 return true;
864 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
865 without analyzing the expression whether or not its location
866 is accessible to anyone else. In the case of nested parallel
867 regions it certainly may be. */
868 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
869 return true;
871 /* Do not use copy-in/copy-out for variables that have their
872 address taken. */
873 if (TREE_ADDRESSABLE (decl))
874 return true;
876 /* lower_send_shared_vars only uses copy-in, but not copy-out
877 for these. */
878 if (TREE_READONLY (decl)
879 || ((TREE_CODE (decl) == RESULT_DECL
880 || TREE_CODE (decl) == PARM_DECL)
881 && DECL_BY_REFERENCE (decl)))
882 return false;
884 /* Disallow copy-in/out in nested parallel if
885 decl is shared in outer parallel, otherwise
886 each thread could store the shared variable
887 in its own copy-in location, making the
888 variable no longer really shared. */
889 if (shared_ctx->is_nested)
891 omp_context *up;
893 for (up = shared_ctx->outer; up; up = up->outer)
894 if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up))
895 break;
897 if (up)
899 tree c;
901 for (c = gimple_omp_taskreg_clauses (up->stmt);
902 c; c = OMP_CLAUSE_CHAIN (c))
903 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
904 && OMP_CLAUSE_DECL (c) == decl)
905 break;
907 if (c)
908 goto maybe_mark_addressable_and_ret;
912 /* For tasks avoid using copy-in/out. As tasks can be
913 deferred or executed in different thread, when GOMP_task
914 returns, the task hasn't necessarily terminated. */
915 if (is_task_ctx (shared_ctx))
917 tree outer;
918 maybe_mark_addressable_and_ret:
919 outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
920 if (is_gimple_reg (outer))
922 /* Taking address of OUTER in lower_send_shared_vars
923 might need regimplification of everything that uses the
924 variable. */
925 if (!task_shared_vars)
926 task_shared_vars = BITMAP_ALLOC (NULL);
927 bitmap_set_bit (task_shared_vars, DECL_UID (outer));
928 TREE_ADDRESSABLE (outer) = 1;
930 return true;
934 return false;
937 /* Construct a new automatic decl similar to VAR. */
939 static tree
940 omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
942 tree copy = copy_var_decl (var, name, type);
944 DECL_CONTEXT (copy) = current_function_decl;
945 DECL_CHAIN (copy) = ctx->block_vars;
946 ctx->block_vars = copy;
948 return copy;
951 static tree
952 omp_copy_decl_1 (tree var, omp_context *ctx)
954 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
957 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
958 as appropriate. */
959 static tree
960 omp_build_component_ref (tree obj, tree field)
962 tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL);
963 if (TREE_THIS_VOLATILE (field))
964 TREE_THIS_VOLATILE (ret) |= 1;
965 if (TREE_READONLY (field))
966 TREE_READONLY (ret) |= 1;
967 return ret;
970 /* Build tree nodes to access the field for VAR on the receiver side. */
972 static tree
973 build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
975 tree x, field = lookup_field (var, ctx);
977 /* If the receiver record type was remapped in the child function,
978 remap the field into the new record type. */
979 x = maybe_lookup_field (field, ctx);
980 if (x != NULL)
981 field = x;
983 x = build_simple_mem_ref (ctx->receiver_decl);
984 x = omp_build_component_ref (x, field);
985 if (by_ref)
986 x = build_simple_mem_ref (x);
988 return x;
991 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
992 of a parallel, this is a component reference; for workshare constructs
993 this is some variable. */
995 static tree
996 build_outer_var_ref (tree var, omp_context *ctx)
998 tree x;
1000 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
1001 x = var;
1002 else if (is_variable_sized (var))
1004 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
1005 x = build_outer_var_ref (x, ctx);
1006 x = build_simple_mem_ref (x);
1008 else if (is_taskreg_ctx (ctx))
1010 bool by_ref = use_pointer_for_field (var, NULL);
1011 x = build_receiver_ref (var, by_ref, ctx);
1013 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1014 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD)
1016 /* #pragma omp simd isn't a worksharing construct, and can reference even
1017 private vars in its linear etc. clauses. */
1018 x = NULL_TREE;
1019 if (ctx->outer && is_taskreg_ctx (ctx))
1020 x = lookup_decl (var, ctx->outer);
1021 else if (ctx->outer)
1022 x = maybe_lookup_decl_in_outer_ctx (var, ctx);
1023 if (x == NULL_TREE)
1024 x = var;
1026 else if (ctx->outer)
1027 x = lookup_decl (var, ctx->outer);
1028 else if (is_reference (var))
1029 /* This can happen with orphaned constructs. If var is reference, it is
1030 possible it is shared and as such valid. */
1031 x = var;
1032 else
1033 gcc_unreachable ();
1035 if (is_reference (var))
1036 x = build_simple_mem_ref (x);
1038 return x;
1041 /* Build tree nodes to access the field for VAR on the sender side. */
1043 static tree
1044 build_sender_ref (tree var, omp_context *ctx)
1046 tree field = lookup_sfield (var, ctx);
1047 return omp_build_component_ref (ctx->sender_decl, field);
1050 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1052 static void
1053 install_var_field (tree var, bool by_ref, int mask, omp_context *ctx)
1055 tree field, type, sfield = NULL_TREE;
1057 gcc_assert ((mask & 1) == 0
1058 || !splay_tree_lookup (ctx->field_map, (splay_tree_key) var));
1059 gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
1060 || !splay_tree_lookup (ctx->sfield_map, (splay_tree_key) var));
1062 type = TREE_TYPE (var);
1063 if (mask & 4)
1065 gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
1066 type = build_pointer_type (build_pointer_type (type));
1068 else if (by_ref)
1069 type = build_pointer_type (type);
1070 else if ((mask & 3) == 1 && is_reference (var))
1071 type = TREE_TYPE (type);
1073 field = build_decl (DECL_SOURCE_LOCATION (var),
1074 FIELD_DECL, DECL_NAME (var), type);
1076 /* Remember what variable this field was created for. This does have a
1077 side effect of making dwarf2out ignore this member, so for helpful
1078 debugging we clear it later in delete_omp_context. */
1079 DECL_ABSTRACT_ORIGIN (field) = var;
1080 if (type == TREE_TYPE (var))
1082 DECL_ALIGN (field) = DECL_ALIGN (var);
1083 DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
1084 TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
1086 else
1087 DECL_ALIGN (field) = TYPE_ALIGN (type);
1089 if ((mask & 3) == 3)
1091 insert_field_into_struct (ctx->record_type, field);
1092 if (ctx->srecord_type)
1094 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1095 FIELD_DECL, DECL_NAME (var), type);
1096 DECL_ABSTRACT_ORIGIN (sfield) = var;
1097 DECL_ALIGN (sfield) = DECL_ALIGN (field);
1098 DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
1099 TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
1100 insert_field_into_struct (ctx->srecord_type, sfield);
1103 else
1105 if (ctx->srecord_type == NULL_TREE)
1107 tree t;
1109 ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
1110 ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1111 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
1113 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1114 FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
1115 DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
1116 insert_field_into_struct (ctx->srecord_type, sfield);
1117 splay_tree_insert (ctx->sfield_map,
1118 (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
1119 (splay_tree_value) sfield);
1122 sfield = field;
1123 insert_field_into_struct ((mask & 1) ? ctx->record_type
1124 : ctx->srecord_type, field);
1127 if (mask & 1)
1128 splay_tree_insert (ctx->field_map, (splay_tree_key) var,
1129 (splay_tree_value) field);
1130 if ((mask & 2) && ctx->sfield_map)
1131 splay_tree_insert (ctx->sfield_map, (splay_tree_key) var,
1132 (splay_tree_value) sfield);
1135 static tree
1136 install_var_local (tree var, omp_context *ctx)
1138 tree new_var = omp_copy_decl_1 (var, ctx);
1139 insert_decl_map (&ctx->cb, var, new_var);
1140 return new_var;
1143 /* Adjust the replacement for DECL in CTX for the new context. This means
1144 copying the DECL_VALUE_EXPR, and fixing up the type. */
1146 static void
1147 fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
1149 tree new_decl, size;
1151 new_decl = lookup_decl (decl, ctx);
1153 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
1155 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
1156 && DECL_HAS_VALUE_EXPR_P (decl))
1158 tree ve = DECL_VALUE_EXPR (decl);
1159 walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
1160 SET_DECL_VALUE_EXPR (new_decl, ve);
1161 DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
1164 if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
1166 size = remap_decl (DECL_SIZE (decl), &ctx->cb);
1167 if (size == error_mark_node)
1168 size = TYPE_SIZE (TREE_TYPE (new_decl));
1169 DECL_SIZE (new_decl) = size;
1171 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
1172 if (size == error_mark_node)
1173 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
1174 DECL_SIZE_UNIT (new_decl) = size;
1178 /* The callback for remap_decl. Search all containing contexts for a
1179 mapping of the variable; this avoids having to duplicate the splay
1180 tree ahead of time. We know a mapping doesn't already exist in the
1181 given context. Create new mappings to implement default semantics. */
1183 static tree
1184 omp_copy_decl (tree var, copy_body_data *cb)
1186 omp_context *ctx = (omp_context *) cb;
1187 tree new_var;
1189 if (TREE_CODE (var) == LABEL_DECL)
1191 new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
1192 DECL_CONTEXT (new_var) = current_function_decl;
1193 insert_decl_map (&ctx->cb, var, new_var);
1194 return new_var;
1197 while (!is_taskreg_ctx (ctx))
1199 ctx = ctx->outer;
1200 if (ctx == NULL)
1201 return var;
1202 new_var = maybe_lookup_decl (var, ctx);
1203 if (new_var)
1204 return new_var;
1207 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
1208 return var;
1210 return error_mark_node;
1214 /* Return the parallel region associated with STMT. */
1216 /* Debugging dumps for parallel regions. */
1217 void dump_omp_region (FILE *, struct omp_region *, int);
1218 void debug_omp_region (struct omp_region *);
1219 void debug_all_omp_regions (void);
1221 /* Dump the parallel region tree rooted at REGION. */
1223 void
1224 dump_omp_region (FILE *file, struct omp_region *region, int indent)
1226 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
1227 gimple_code_name[region->type]);
1229 if (region->inner)
1230 dump_omp_region (file, region->inner, indent + 4);
1232 if (region->cont)
1234 fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "",
1235 region->cont->index);
1238 if (region->exit)
1239 fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "",
1240 region->exit->index);
1241 else
1242 fprintf (file, "%*s[no exit marker]\n", indent, "");
1244 if (region->next)
1245 dump_omp_region (file, region->next, indent);
1248 DEBUG_FUNCTION void
1249 debug_omp_region (struct omp_region *region)
1251 dump_omp_region (stderr, region, 0);
1254 DEBUG_FUNCTION void
1255 debug_all_omp_regions (void)
1257 dump_omp_region (stderr, root_omp_region, 0);
1261 /* Create a new parallel region starting at STMT inside region PARENT. */
1263 static struct omp_region *
1264 new_omp_region (basic_block bb, enum gimple_code type,
1265 struct omp_region *parent)
1267 struct omp_region *region = XCNEW (struct omp_region);
1269 region->outer = parent;
1270 region->entry = bb;
1271 region->type = type;
1273 if (parent)
1275 /* This is a nested region. Add it to the list of inner
1276 regions in PARENT. */
1277 region->next = parent->inner;
1278 parent->inner = region;
1280 else
1282 /* This is a toplevel region. Add it to the list of toplevel
1283 regions in ROOT_OMP_REGION. */
1284 region->next = root_omp_region;
1285 root_omp_region = region;
1288 return region;
1291 /* Release the memory associated with the region tree rooted at REGION. */
1293 static void
1294 free_omp_region_1 (struct omp_region *region)
1296 struct omp_region *i, *n;
1298 for (i = region->inner; i ; i = n)
1300 n = i->next;
1301 free_omp_region_1 (i);
1304 free (region);
1307 /* Release the memory for the entire omp region tree. */
1309 void
1310 free_omp_regions (void)
1312 struct omp_region *r, *n;
1313 for (r = root_omp_region; r ; r = n)
1315 n = r->next;
1316 free_omp_region_1 (r);
1318 root_omp_region = NULL;
1322 /* Create a new context, with OUTER_CTX being the surrounding context. */
1324 static omp_context *
1325 new_omp_context (gimple stmt, omp_context *outer_ctx)
1327 omp_context *ctx = XCNEW (omp_context);
1329 splay_tree_insert (all_contexts, (splay_tree_key) stmt,
1330 (splay_tree_value) ctx);
1331 ctx->stmt = stmt;
1333 if (outer_ctx)
1335 ctx->outer = outer_ctx;
1336 ctx->cb = outer_ctx->cb;
1337 ctx->cb.block = NULL;
1338 ctx->depth = outer_ctx->depth + 1;
1340 else
1342 ctx->cb.src_fn = current_function_decl;
1343 ctx->cb.dst_fn = current_function_decl;
1344 ctx->cb.src_node = cgraph_get_node (current_function_decl);
1345 gcc_checking_assert (ctx->cb.src_node);
1346 ctx->cb.dst_node = ctx->cb.src_node;
1347 ctx->cb.src_cfun = cfun;
1348 ctx->cb.copy_decl = omp_copy_decl;
1349 ctx->cb.eh_lp_nr = 0;
1350 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1351 ctx->depth = 1;
1354 ctx->cb.decl_map = pointer_map_create ();
1356 return ctx;
1359 static gimple_seq maybe_catch_exception (gimple_seq);
1361 /* Finalize task copyfn. */
1363 static void
1364 finalize_task_copyfn (gimple task_stmt)
1366 struct function *child_cfun;
1367 tree child_fn;
1368 gimple_seq seq = NULL, new_seq;
1369 gimple bind;
1371 child_fn = gimple_omp_task_copy_fn (task_stmt);
1372 if (child_fn == NULL_TREE)
1373 return;
1375 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1376 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1378 push_cfun (child_cfun);
1379 bind = gimplify_body (child_fn, false);
1380 gimple_seq_add_stmt (&seq, bind);
1381 new_seq = maybe_catch_exception (seq);
1382 if (new_seq != seq)
1384 bind = gimple_build_bind (NULL, new_seq, NULL);
1385 seq = NULL;
1386 gimple_seq_add_stmt (&seq, bind);
1388 gimple_set_body (child_fn, seq);
1389 pop_cfun ();
1391 /* Inform the callgraph about the new function. */
1392 cgraph_add_new_function (child_fn, false);
1395 /* Destroy a omp_context data structures. Called through the splay tree
1396 value delete callback. */
1398 static void
1399 delete_omp_context (splay_tree_value value)
1401 omp_context *ctx = (omp_context *) value;
1403 pointer_map_destroy (ctx->cb.decl_map);
1405 if (ctx->field_map)
1406 splay_tree_delete (ctx->field_map);
1407 if (ctx->sfield_map)
1408 splay_tree_delete (ctx->sfield_map);
1410 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1411 it produces corrupt debug information. */
1412 if (ctx->record_type)
1414 tree t;
1415 for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1416 DECL_ABSTRACT_ORIGIN (t) = NULL;
1418 if (ctx->srecord_type)
1420 tree t;
1421 for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1422 DECL_ABSTRACT_ORIGIN (t) = NULL;
1425 if (is_task_ctx (ctx))
1426 finalize_task_copyfn (ctx->stmt);
1428 XDELETE (ctx);
1431 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1432 context. */
1434 static void
1435 fixup_child_record_type (omp_context *ctx)
1437 tree f, type = ctx->record_type;
1439 /* ??? It isn't sufficient to just call remap_type here, because
1440 variably_modified_type_p doesn't work the way we expect for
1441 record types. Testing each field for whether it needs remapping
1442 and creating a new record by hand works, however. */
1443 for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1444 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1445 break;
1446 if (f)
1448 tree name, new_fields = NULL;
1450 type = lang_hooks.types.make_type (RECORD_TYPE);
1451 name = DECL_NAME (TYPE_NAME (ctx->record_type));
1452 name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1453 TYPE_DECL, name, type);
1454 TYPE_NAME (type) = name;
1456 for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1458 tree new_f = copy_node (f);
1459 DECL_CONTEXT (new_f) = type;
1460 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1461 DECL_CHAIN (new_f) = new_fields;
1462 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1463 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1464 &ctx->cb, NULL);
1465 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1466 &ctx->cb, NULL);
1467 new_fields = new_f;
1469 /* Arrange to be able to look up the receiver field
1470 given the sender field. */
1471 splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1472 (splay_tree_value) new_f);
1474 TYPE_FIELDS (type) = nreverse (new_fields);
1475 layout_type (type);
1478 TREE_TYPE (ctx->receiver_decl) = build_pointer_type (type);
1481 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1482 specified by CLAUSES. */
1484 static void
1485 scan_sharing_clauses (tree clauses, omp_context *ctx)
1487 tree c, decl;
1488 bool scan_array_reductions = false;
1490 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1492 bool by_ref;
1494 switch (OMP_CLAUSE_CODE (c))
1496 case OMP_CLAUSE_PRIVATE:
1497 decl = OMP_CLAUSE_DECL (c);
1498 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1499 goto do_private;
1500 else if (!is_variable_sized (decl))
1501 install_var_local (decl, ctx);
1502 break;
1504 case OMP_CLAUSE_SHARED:
1505 /* Ignore shared directives in teams construct. */
1506 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1507 break;
1508 gcc_assert (is_taskreg_ctx (ctx));
1509 decl = OMP_CLAUSE_DECL (c);
1510 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1511 || !is_variable_sized (decl));
1512 /* Global variables don't need to be copied,
1513 the receiver side will use them directly. */
1514 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1515 break;
1516 by_ref = use_pointer_for_field (decl, ctx);
1517 if (! TREE_READONLY (decl)
1518 || TREE_ADDRESSABLE (decl)
1519 || by_ref
1520 || is_reference (decl))
1522 install_var_field (decl, by_ref, 3, ctx);
1523 install_var_local (decl, ctx);
1524 break;
1526 /* We don't need to copy const scalar vars back. */
1527 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1528 goto do_private;
1530 case OMP_CLAUSE_LASTPRIVATE:
1531 /* Let the corresponding firstprivate clause create
1532 the variable. */
1533 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1534 break;
1535 /* FALLTHRU */
1537 case OMP_CLAUSE_FIRSTPRIVATE:
1538 case OMP_CLAUSE_REDUCTION:
1539 case OMP_CLAUSE_LINEAR:
1540 decl = OMP_CLAUSE_DECL (c);
1541 do_private:
1542 if (is_variable_sized (decl))
1544 if (is_task_ctx (ctx))
1545 install_var_field (decl, false, 1, ctx);
1546 break;
1548 else if (is_taskreg_ctx (ctx))
1550 bool global
1551 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1552 by_ref = use_pointer_for_field (decl, NULL);
1554 if (is_task_ctx (ctx)
1555 && (global || by_ref || is_reference (decl)))
1557 install_var_field (decl, false, 1, ctx);
1558 if (!global)
1559 install_var_field (decl, by_ref, 2, ctx);
1561 else if (!global)
1562 install_var_field (decl, by_ref, 3, ctx);
1564 install_var_local (decl, ctx);
1565 break;
1567 case OMP_CLAUSE__LOOPTEMP_:
1568 gcc_assert (is_parallel_ctx (ctx));
1569 decl = OMP_CLAUSE_DECL (c);
1570 install_var_field (decl, false, 3, ctx);
1571 install_var_local (decl, ctx);
1572 break;
1574 case OMP_CLAUSE_COPYPRIVATE:
1575 case OMP_CLAUSE_COPYIN:
1576 decl = OMP_CLAUSE_DECL (c);
1577 by_ref = use_pointer_for_field (decl, NULL);
1578 install_var_field (decl, by_ref, 3, ctx);
1579 break;
1581 case OMP_CLAUSE_DEFAULT:
1582 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c);
1583 break;
1585 case OMP_CLAUSE_FINAL:
1586 case OMP_CLAUSE_IF:
1587 case OMP_CLAUSE_NUM_THREADS:
1588 case OMP_CLAUSE_NUM_TEAMS:
1589 case OMP_CLAUSE_THREAD_LIMIT:
1590 case OMP_CLAUSE_DEVICE:
1591 case OMP_CLAUSE_SCHEDULE:
1592 case OMP_CLAUSE_DIST_SCHEDULE:
1593 case OMP_CLAUSE_DEPEND:
1594 if (ctx->outer)
1595 scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1596 break;
1598 case OMP_CLAUSE_TO:
1599 case OMP_CLAUSE_FROM:
1600 case OMP_CLAUSE_MAP:
1601 if (ctx->outer)
1602 scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1603 decl = OMP_CLAUSE_DECL (c);
1604 /* Global variables with "omp declare target" attribute
1605 don't need to be copied, the receiver side will use them
1606 directly. */
1607 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1608 && DECL_P (decl)
1609 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1610 && lookup_attribute ("omp declare target",
1611 DECL_ATTRIBUTES (decl)))
1612 break;
1613 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1614 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER)
1616 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1617 #pragma omp target data, there is nothing to map for
1618 those. */
1619 if (gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_DATA
1620 && !POINTER_TYPE_P (TREE_TYPE (decl)))
1621 break;
1623 if (DECL_P (decl))
1625 if (DECL_SIZE (decl)
1626 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1628 tree decl2 = DECL_VALUE_EXPR (decl);
1629 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
1630 decl2 = TREE_OPERAND (decl2, 0);
1631 gcc_assert (DECL_P (decl2));
1632 install_var_field (decl2, true, 3, ctx);
1633 install_var_local (decl2, ctx);
1634 install_var_local (decl, ctx);
1636 else
1638 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1639 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
1640 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1641 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1642 install_var_field (decl, true, 7, ctx);
1643 else
1644 install_var_field (decl, true, 3, ctx);
1645 if (gimple_omp_target_kind (ctx->stmt)
1646 == GF_OMP_TARGET_KIND_REGION)
1647 install_var_local (decl, ctx);
1650 else
1652 tree base = get_base_address (decl);
1653 tree nc = OMP_CLAUSE_CHAIN (c);
1654 if (DECL_P (base)
1655 && nc != NULL_TREE
1656 && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1657 && OMP_CLAUSE_DECL (nc) == base
1658 && OMP_CLAUSE_MAP_KIND (nc) == OMP_CLAUSE_MAP_POINTER
1659 && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1661 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1662 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1664 else
1666 gcc_assert (!splay_tree_lookup (ctx->field_map,
1667 (splay_tree_key) decl));
1668 tree field
1669 = build_decl (OMP_CLAUSE_LOCATION (c),
1670 FIELD_DECL, NULL_TREE, ptr_type_node);
1671 DECL_ALIGN (field) = TYPE_ALIGN (ptr_type_node);
1672 insert_field_into_struct (ctx->record_type, field);
1673 splay_tree_insert (ctx->field_map, (splay_tree_key) decl,
1674 (splay_tree_value) field);
1677 break;
1679 case OMP_CLAUSE_NOWAIT:
1680 case OMP_CLAUSE_ORDERED:
1681 case OMP_CLAUSE_COLLAPSE:
1682 case OMP_CLAUSE_UNTIED:
1683 case OMP_CLAUSE_MERGEABLE:
1684 case OMP_CLAUSE_PROC_BIND:
1685 case OMP_CLAUSE_SAFELEN:
1686 break;
1688 case OMP_CLAUSE_ALIGNED:
1689 decl = OMP_CLAUSE_DECL (c);
1690 if (is_global_var (decl)
1691 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1692 install_var_local (decl, ctx);
1693 break;
1695 default:
1696 gcc_unreachable ();
1700 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1702 switch (OMP_CLAUSE_CODE (c))
1704 case OMP_CLAUSE_LASTPRIVATE:
1705 /* Let the corresponding firstprivate clause create
1706 the variable. */
1707 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1708 scan_array_reductions = true;
1709 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1710 break;
1711 /* FALLTHRU */
1713 case OMP_CLAUSE_PRIVATE:
1714 case OMP_CLAUSE_FIRSTPRIVATE:
1715 case OMP_CLAUSE_REDUCTION:
1716 case OMP_CLAUSE_LINEAR:
1717 decl = OMP_CLAUSE_DECL (c);
1718 if (is_variable_sized (decl))
1719 install_var_local (decl, ctx);
1720 fixup_remapped_decl (decl, ctx,
1721 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1722 && OMP_CLAUSE_PRIVATE_DEBUG (c));
1723 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1724 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1725 scan_array_reductions = true;
1726 break;
1728 case OMP_CLAUSE_SHARED:
1729 /* Ignore shared directives in teams construct. */
1730 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1731 break;
1732 decl = OMP_CLAUSE_DECL (c);
1733 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1734 fixup_remapped_decl (decl, ctx, false);
1735 break;
1737 case OMP_CLAUSE_MAP:
1738 if (gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_DATA)
1739 break;
1740 decl = OMP_CLAUSE_DECL (c);
1741 if (DECL_P (decl)
1742 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1743 && lookup_attribute ("omp declare target",
1744 DECL_ATTRIBUTES (decl)))
1745 break;
1746 if (DECL_P (decl))
1748 if (OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
1749 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
1750 && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
1752 tree new_decl = lookup_decl (decl, ctx);
1753 TREE_TYPE (new_decl)
1754 = remap_type (TREE_TYPE (decl), &ctx->cb);
1756 else if (DECL_SIZE (decl)
1757 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1759 tree decl2 = DECL_VALUE_EXPR (decl);
1760 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
1761 decl2 = TREE_OPERAND (decl2, 0);
1762 gcc_assert (DECL_P (decl2));
1763 fixup_remapped_decl (decl2, ctx, false);
1764 fixup_remapped_decl (decl, ctx, true);
1766 else
1767 fixup_remapped_decl (decl, ctx, false);
1769 break;
1771 case OMP_CLAUSE_COPYPRIVATE:
1772 case OMP_CLAUSE_COPYIN:
1773 case OMP_CLAUSE_DEFAULT:
1774 case OMP_CLAUSE_IF:
1775 case OMP_CLAUSE_NUM_THREADS:
1776 case OMP_CLAUSE_NUM_TEAMS:
1777 case OMP_CLAUSE_THREAD_LIMIT:
1778 case OMP_CLAUSE_DEVICE:
1779 case OMP_CLAUSE_SCHEDULE:
1780 case OMP_CLAUSE_DIST_SCHEDULE:
1781 case OMP_CLAUSE_NOWAIT:
1782 case OMP_CLAUSE_ORDERED:
1783 case OMP_CLAUSE_COLLAPSE:
1784 case OMP_CLAUSE_UNTIED:
1785 case OMP_CLAUSE_FINAL:
1786 case OMP_CLAUSE_MERGEABLE:
1787 case OMP_CLAUSE_PROC_BIND:
1788 case OMP_CLAUSE_SAFELEN:
1789 case OMP_CLAUSE_ALIGNED:
1790 case OMP_CLAUSE_DEPEND:
1791 case OMP_CLAUSE__LOOPTEMP_:
1792 case OMP_CLAUSE_TO:
1793 case OMP_CLAUSE_FROM:
1794 break;
1796 default:
1797 gcc_unreachable ();
1801 if (scan_array_reductions)
1802 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1803 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1804 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1806 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx);
1807 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
1809 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
1810 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
1811 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
1814 /* Create a new name for omp child function. Returns an identifier. */
1816 static GTY(()) unsigned int tmp_ompfn_id_num;
1818 static tree
1819 create_omp_child_function_name (bool task_copy)
1821 return (clone_function_name (current_function_decl,
1822 task_copy ? "_omp_cpyfn" : "_omp_fn"));
1825 /* Build a decl for the omp child function. It'll not contain a body
1826 yet, just the bare decl. */
1828 static void
1829 create_omp_child_function (omp_context *ctx, bool task_copy)
1831 tree decl, type, name, t;
1833 name = create_omp_child_function_name (task_copy);
1834 if (task_copy)
1835 type = build_function_type_list (void_type_node, ptr_type_node,
1836 ptr_type_node, NULL_TREE);
1837 else
1838 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
1840 decl = build_decl (gimple_location (ctx->stmt),
1841 FUNCTION_DECL, name, type);
1843 if (!task_copy)
1844 ctx->cb.dst_fn = decl;
1845 else
1846 gimple_omp_task_set_copy_fn (ctx->stmt, decl);
1848 TREE_STATIC (decl) = 1;
1849 TREE_USED (decl) = 1;
1850 DECL_ARTIFICIAL (decl) = 1;
1851 DECL_NAMELESS (decl) = 1;
1852 DECL_IGNORED_P (decl) = 0;
1853 TREE_PUBLIC (decl) = 0;
1854 DECL_UNINLINABLE (decl) = 1;
1855 DECL_EXTERNAL (decl) = 0;
1856 DECL_CONTEXT (decl) = NULL_TREE;
1857 DECL_INITIAL (decl) = make_node (BLOCK);
1858 bool target_p = false;
1859 if (lookup_attribute ("omp declare target",
1860 DECL_ATTRIBUTES (current_function_decl)))
1861 target_p = true;
1862 else
1864 omp_context *octx;
1865 for (octx = ctx; octx; octx = octx->outer)
1866 if (gimple_code (octx->stmt) == GIMPLE_OMP_TARGET
1867 && gimple_omp_target_kind (octx->stmt)
1868 == GF_OMP_TARGET_KIND_REGION)
1870 target_p = true;
1871 break;
1874 if (target_p)
1875 DECL_ATTRIBUTES (decl)
1876 = tree_cons (get_identifier ("omp declare target"),
1877 NULL_TREE, DECL_ATTRIBUTES (decl));
1879 t = build_decl (DECL_SOURCE_LOCATION (decl),
1880 RESULT_DECL, NULL_TREE, void_type_node);
1881 DECL_ARTIFICIAL (t) = 1;
1882 DECL_IGNORED_P (t) = 1;
1883 DECL_CONTEXT (t) = decl;
1884 DECL_RESULT (decl) = t;
1886 t = build_decl (DECL_SOURCE_LOCATION (decl),
1887 PARM_DECL, get_identifier (".omp_data_i"), ptr_type_node);
1888 DECL_ARTIFICIAL (t) = 1;
1889 DECL_NAMELESS (t) = 1;
1890 DECL_ARG_TYPE (t) = ptr_type_node;
1891 DECL_CONTEXT (t) = current_function_decl;
1892 TREE_USED (t) = 1;
1893 DECL_ARGUMENTS (decl) = t;
1894 if (!task_copy)
1895 ctx->receiver_decl = t;
1896 else
1898 t = build_decl (DECL_SOURCE_LOCATION (decl),
1899 PARM_DECL, get_identifier (".omp_data_o"),
1900 ptr_type_node);
1901 DECL_ARTIFICIAL (t) = 1;
1902 DECL_NAMELESS (t) = 1;
1903 DECL_ARG_TYPE (t) = ptr_type_node;
1904 DECL_CONTEXT (t) = current_function_decl;
1905 TREE_USED (t) = 1;
1906 TREE_ADDRESSABLE (t) = 1;
1907 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
1908 DECL_ARGUMENTS (decl) = t;
1911 /* Allocate memory for the function structure. The call to
1912 allocate_struct_function clobbers CFUN, so we need to restore
1913 it afterward. */
1914 push_struct_function (decl);
1915 cfun->function_end_locus = gimple_location (ctx->stmt);
1916 pop_cfun ();
1919 /* Callback for walk_gimple_seq. Check if combined parallel
1920 contains gimple_omp_for_combined_into_p OMP_FOR. */
1922 static tree
1923 find_combined_for (gimple_stmt_iterator *gsi_p,
1924 bool *handled_ops_p,
1925 struct walk_stmt_info *wi)
1927 gimple stmt = gsi_stmt (*gsi_p);
1929 *handled_ops_p = true;
1930 switch (gimple_code (stmt))
1932 WALK_SUBSTMTS;
1934 case GIMPLE_OMP_FOR:
1935 if (gimple_omp_for_combined_into_p (stmt)
1936 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
1938 wi->info = stmt;
1939 return integer_zero_node;
1941 break;
1942 default:
1943 break;
1945 return NULL;
1948 /* Scan an OpenMP parallel directive. */
1950 static void
1951 scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
1953 omp_context *ctx;
1954 tree name;
1955 gimple stmt = gsi_stmt (*gsi);
1957 /* Ignore parallel directives with empty bodies, unless there
1958 are copyin clauses. */
1959 if (optimize > 0
1960 && empty_body_p (gimple_omp_body (stmt))
1961 && find_omp_clause (gimple_omp_parallel_clauses (stmt),
1962 OMP_CLAUSE_COPYIN) == NULL)
1964 gsi_replace (gsi, gimple_build_nop (), false);
1965 return;
1968 if (gimple_omp_parallel_combined_p (stmt))
1970 gimple for_stmt;
1971 struct walk_stmt_info wi;
1973 memset (&wi, 0, sizeof (wi));
1974 wi.val_only = true;
1975 walk_gimple_seq (gimple_omp_body (stmt),
1976 find_combined_for, NULL, &wi);
1977 for_stmt = (gimple) wi.info;
1978 if (for_stmt)
1980 struct omp_for_data fd;
1981 extract_omp_for_data (for_stmt, &fd, NULL);
1982 /* We need two temporaries with fd.loop.v type (istart/iend)
1983 and then (fd.collapse - 1) temporaries with the same
1984 type for count2 ... countN-1 vars if not constant. */
1985 size_t count = 2, i;
1986 tree type = fd.iter_type;
1987 if (fd.collapse > 1
1988 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
1989 count += fd.collapse - 1;
1990 for (i = 0; i < count; i++)
1992 tree temp = create_tmp_var (type, NULL);
1993 tree c = build_omp_clause (UNKNOWN_LOCATION,
1994 OMP_CLAUSE__LOOPTEMP_);
1995 OMP_CLAUSE_DECL (c) = temp;
1996 OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
1997 gimple_omp_parallel_set_clauses (stmt, c);
2002 ctx = new_omp_context (stmt, outer_ctx);
2003 if (taskreg_nesting_level > 1)
2004 ctx->is_nested = true;
2005 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2006 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2007 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2008 name = create_tmp_var_name (".omp_data_s");
2009 name = build_decl (gimple_location (stmt),
2010 TYPE_DECL, name, ctx->record_type);
2011 DECL_ARTIFICIAL (name) = 1;
2012 DECL_NAMELESS (name) = 1;
2013 TYPE_NAME (ctx->record_type) = name;
2014 create_omp_child_function (ctx, false);
2015 gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2017 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2018 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2020 if (TYPE_FIELDS (ctx->record_type) == NULL)
2021 ctx->record_type = ctx->receiver_decl = NULL;
2022 else
2024 layout_type (ctx->record_type);
2025 fixup_child_record_type (ctx);
2029 /* Scan an OpenMP task directive. */
2031 static void
2032 scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2034 omp_context *ctx;
2035 tree name, t;
2036 gimple stmt = gsi_stmt (*gsi);
2037 location_t loc = gimple_location (stmt);
2039 /* Ignore task directives with empty bodies. */
2040 if (optimize > 0
2041 && empty_body_p (gimple_omp_body (stmt)))
2043 gsi_replace (gsi, gimple_build_nop (), false);
2044 return;
2047 ctx = new_omp_context (stmt, outer_ctx);
2048 if (taskreg_nesting_level > 1)
2049 ctx->is_nested = true;
2050 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2051 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2052 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2053 name = create_tmp_var_name (".omp_data_s");
2054 name = build_decl (gimple_location (stmt),
2055 TYPE_DECL, name, ctx->record_type);
2056 DECL_ARTIFICIAL (name) = 1;
2057 DECL_NAMELESS (name) = 1;
2058 TYPE_NAME (ctx->record_type) = name;
2059 create_omp_child_function (ctx, false);
2060 gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2062 scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2064 if (ctx->srecord_type)
2066 name = create_tmp_var_name (".omp_data_a");
2067 name = build_decl (gimple_location (stmt),
2068 TYPE_DECL, name, ctx->srecord_type);
2069 DECL_ARTIFICIAL (name) = 1;
2070 DECL_NAMELESS (name) = 1;
2071 TYPE_NAME (ctx->srecord_type) = name;
2072 create_omp_child_function (ctx, true);
2075 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2077 if (TYPE_FIELDS (ctx->record_type) == NULL)
2079 ctx->record_type = ctx->receiver_decl = NULL;
2080 t = build_int_cst (long_integer_type_node, 0);
2081 gimple_omp_task_set_arg_size (stmt, t);
2082 t = build_int_cst (long_integer_type_node, 1);
2083 gimple_omp_task_set_arg_align (stmt, t);
2085 else
2087 tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2088 /* Move VLA fields to the end. */
2089 p = &TYPE_FIELDS (ctx->record_type);
2090 while (*p)
2091 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2092 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2094 *q = *p;
2095 *p = TREE_CHAIN (*p);
2096 TREE_CHAIN (*q) = NULL_TREE;
2097 q = &TREE_CHAIN (*q);
2099 else
2100 p = &DECL_CHAIN (*p);
2101 *p = vla_fields;
2102 layout_type (ctx->record_type);
2103 fixup_child_record_type (ctx);
2104 if (ctx->srecord_type)
2105 layout_type (ctx->srecord_type);
2106 t = fold_convert_loc (loc, long_integer_type_node,
2107 TYPE_SIZE_UNIT (ctx->record_type));
2108 gimple_omp_task_set_arg_size (stmt, t);
2109 t = build_int_cst (long_integer_type_node,
2110 TYPE_ALIGN_UNIT (ctx->record_type));
2111 gimple_omp_task_set_arg_align (stmt, t);
2116 /* Scan an OpenMP loop directive. */
2118 static void
2119 scan_omp_for (gimple stmt, omp_context *outer_ctx)
2121 omp_context *ctx;
2122 size_t i;
2124 ctx = new_omp_context (stmt, outer_ctx);
2126 scan_sharing_clauses (gimple_omp_for_clauses (stmt), ctx);
2128 scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
2129 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
2131 scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
2132 scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
2133 scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
2134 scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
2136 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2139 /* Scan an OpenMP sections directive. */
2141 static void
2142 scan_omp_sections (gimple stmt, omp_context *outer_ctx)
2144 omp_context *ctx;
2146 ctx = new_omp_context (stmt, outer_ctx);
2147 scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
2148 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2151 /* Scan an OpenMP single directive. */
2153 static void
2154 scan_omp_single (gimple stmt, omp_context *outer_ctx)
2156 omp_context *ctx;
2157 tree name;
2159 ctx = new_omp_context (stmt, outer_ctx);
2160 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2161 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2162 name = create_tmp_var_name (".omp_copy_s");
2163 name = build_decl (gimple_location (stmt),
2164 TYPE_DECL, name, ctx->record_type);
2165 TYPE_NAME (ctx->record_type) = name;
2167 scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
2168 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2170 if (TYPE_FIELDS (ctx->record_type) == NULL)
2171 ctx->record_type = NULL;
2172 else
2173 layout_type (ctx->record_type);
2176 /* Scan an OpenMP target{, data, update} directive. */
2178 static void
2179 scan_omp_target (gimple stmt, omp_context *outer_ctx)
2181 omp_context *ctx;
2182 tree name;
2183 int kind = gimple_omp_target_kind (stmt);
2185 ctx = new_omp_context (stmt, outer_ctx);
2186 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2187 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2188 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2189 name = create_tmp_var_name (".omp_data_t");
2190 name = build_decl (gimple_location (stmt),
2191 TYPE_DECL, name, ctx->record_type);
2192 DECL_ARTIFICIAL (name) = 1;
2193 DECL_NAMELESS (name) = 1;
2194 TYPE_NAME (ctx->record_type) = name;
2195 if (kind == GF_OMP_TARGET_KIND_REGION)
2197 create_omp_child_function (ctx, false);
2198 gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
2201 scan_sharing_clauses (gimple_omp_target_clauses (stmt), ctx);
2202 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2204 if (TYPE_FIELDS (ctx->record_type) == NULL)
2205 ctx->record_type = ctx->receiver_decl = NULL;
2206 else
2208 TYPE_FIELDS (ctx->record_type)
2209 = nreverse (TYPE_FIELDS (ctx->record_type));
2210 #ifdef ENABLE_CHECKING
2211 tree field;
2212 unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
2213 for (field = TYPE_FIELDS (ctx->record_type);
2214 field;
2215 field = DECL_CHAIN (field))
2216 gcc_assert (DECL_ALIGN (field) == align);
2217 #endif
2218 layout_type (ctx->record_type);
2219 if (kind == GF_OMP_TARGET_KIND_REGION)
2220 fixup_child_record_type (ctx);
2224 /* Scan an OpenMP teams directive. */
2226 static void
2227 scan_omp_teams (gimple stmt, omp_context *outer_ctx)
2229 omp_context *ctx = new_omp_context (stmt, outer_ctx);
2230 scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
2231 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2234 /* Check OpenMP nesting restrictions. */
2235 static bool
2236 check_omp_nesting_restrictions (gimple stmt, omp_context *ctx)
2238 if (ctx != NULL)
2240 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
2241 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD)
2243 error_at (gimple_location (stmt),
2244 "OpenMP constructs may not be nested inside simd region");
2245 return false;
2247 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2249 if ((gimple_code (stmt) != GIMPLE_OMP_FOR
2250 || (gimple_omp_for_kind (stmt)
2251 != GF_OMP_FOR_KIND_DISTRIBUTE))
2252 && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
2254 error_at (gimple_location (stmt),
2255 "only distribute or parallel constructs are allowed to "
2256 "be closely nested inside teams construct");
2257 return false;
2261 switch (gimple_code (stmt))
2263 case GIMPLE_OMP_FOR:
2264 if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_KIND_SIMD)
2265 return true;
2266 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
2268 if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
2270 error_at (gimple_location (stmt),
2271 "distribute construct must be closely nested inside "
2272 "teams construct");
2273 return false;
2275 return true;
2277 /* FALLTHRU */
2278 case GIMPLE_CALL:
2279 if (is_gimple_call (stmt)
2280 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2281 == BUILT_IN_GOMP_CANCEL
2282 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2283 == BUILT_IN_GOMP_CANCELLATION_POINT))
2285 const char *bad = NULL;
2286 const char *kind = NULL;
2287 if (ctx == NULL)
2289 error_at (gimple_location (stmt), "orphaned %qs construct",
2290 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2291 == BUILT_IN_GOMP_CANCEL
2292 ? "#pragma omp cancel"
2293 : "#pragma omp cancellation point");
2294 return false;
2296 switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
2297 ? tree_to_shwi (gimple_call_arg (stmt, 0))
2298 : 0)
2300 case 1:
2301 if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
2302 bad = "#pragma omp parallel";
2303 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2304 == BUILT_IN_GOMP_CANCEL
2305 && !integer_zerop (gimple_call_arg (stmt, 1)))
2306 ctx->cancellable = true;
2307 kind = "parallel";
2308 break;
2309 case 2:
2310 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
2311 || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
2312 bad = "#pragma omp for";
2313 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2314 == BUILT_IN_GOMP_CANCEL
2315 && !integer_zerop (gimple_call_arg (stmt, 1)))
2317 ctx->cancellable = true;
2318 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2319 OMP_CLAUSE_NOWAIT))
2320 warning_at (gimple_location (stmt), 0,
2321 "%<#pragma omp cancel for%> inside "
2322 "%<nowait%> for construct");
2323 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2324 OMP_CLAUSE_ORDERED))
2325 warning_at (gimple_location (stmt), 0,
2326 "%<#pragma omp cancel for%> inside "
2327 "%<ordered%> for construct");
2329 kind = "for";
2330 break;
2331 case 4:
2332 if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
2333 && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
2334 bad = "#pragma omp sections";
2335 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2336 == BUILT_IN_GOMP_CANCEL
2337 && !integer_zerop (gimple_call_arg (stmt, 1)))
2339 if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
2341 ctx->cancellable = true;
2342 if (find_omp_clause (gimple_omp_sections_clauses
2343 (ctx->stmt),
2344 OMP_CLAUSE_NOWAIT))
2345 warning_at (gimple_location (stmt), 0,
2346 "%<#pragma omp cancel sections%> inside "
2347 "%<nowait%> sections construct");
2349 else
2351 gcc_assert (ctx->outer
2352 && gimple_code (ctx->outer->stmt)
2353 == GIMPLE_OMP_SECTIONS);
2354 ctx->outer->cancellable = true;
2355 if (find_omp_clause (gimple_omp_sections_clauses
2356 (ctx->outer->stmt),
2357 OMP_CLAUSE_NOWAIT))
2358 warning_at (gimple_location (stmt), 0,
2359 "%<#pragma omp cancel sections%> inside "
2360 "%<nowait%> sections construct");
2363 kind = "sections";
2364 break;
2365 case 8:
2366 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK)
2367 bad = "#pragma omp task";
2368 else
2369 ctx->cancellable = true;
2370 kind = "taskgroup";
2371 break;
2372 default:
2373 error_at (gimple_location (stmt), "invalid arguments");
2374 return false;
2376 if (bad)
2378 error_at (gimple_location (stmt),
2379 "%<%s %s%> construct not closely nested inside of %qs",
2380 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2381 == BUILT_IN_GOMP_CANCEL
2382 ? "#pragma omp cancel"
2383 : "#pragma omp cancellation point", kind, bad);
2384 return false;
2387 /* FALLTHRU */
2388 case GIMPLE_OMP_SECTIONS:
2389 case GIMPLE_OMP_SINGLE:
2390 for (; ctx != NULL; ctx = ctx->outer)
2391 switch (gimple_code (ctx->stmt))
2393 case GIMPLE_OMP_FOR:
2394 case GIMPLE_OMP_SECTIONS:
2395 case GIMPLE_OMP_SINGLE:
2396 case GIMPLE_OMP_ORDERED:
2397 case GIMPLE_OMP_MASTER:
2398 case GIMPLE_OMP_TASK:
2399 case GIMPLE_OMP_CRITICAL:
2400 if (is_gimple_call (stmt))
2402 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2403 != BUILT_IN_GOMP_BARRIER)
2404 return true;
2405 error_at (gimple_location (stmt),
2406 "barrier region may not be closely nested inside "
2407 "of work-sharing, critical, ordered, master or "
2408 "explicit task region");
2409 return false;
2411 error_at (gimple_location (stmt),
2412 "work-sharing region may not be closely nested inside "
2413 "of work-sharing, critical, ordered, master or explicit "
2414 "task region");
2415 return false;
2416 case GIMPLE_OMP_PARALLEL:
2417 return true;
2418 default:
2419 break;
2421 break;
2422 case GIMPLE_OMP_MASTER:
2423 for (; ctx != NULL; ctx = ctx->outer)
2424 switch (gimple_code (ctx->stmt))
2426 case GIMPLE_OMP_FOR:
2427 case GIMPLE_OMP_SECTIONS:
2428 case GIMPLE_OMP_SINGLE:
2429 case GIMPLE_OMP_TASK:
2430 error_at (gimple_location (stmt),
2431 "master region may not be closely nested inside "
2432 "of work-sharing or explicit task region");
2433 return false;
2434 case GIMPLE_OMP_PARALLEL:
2435 return true;
2436 default:
2437 break;
2439 break;
2440 case GIMPLE_OMP_ORDERED:
2441 for (; ctx != NULL; ctx = ctx->outer)
2442 switch (gimple_code (ctx->stmt))
2444 case GIMPLE_OMP_CRITICAL:
2445 case GIMPLE_OMP_TASK:
2446 error_at (gimple_location (stmt),
2447 "ordered region may not be closely nested inside "
2448 "of critical or explicit task region");
2449 return false;
2450 case GIMPLE_OMP_FOR:
2451 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2452 OMP_CLAUSE_ORDERED) == NULL)
2454 error_at (gimple_location (stmt),
2455 "ordered region must be closely nested inside "
2456 "a loop region with an ordered clause");
2457 return false;
2459 return true;
2460 case GIMPLE_OMP_PARALLEL:
2461 error_at (gimple_location (stmt),
2462 "ordered region must be closely nested inside "
2463 "a loop region with an ordered clause");
2464 return false;
2465 default:
2466 break;
2468 break;
2469 case GIMPLE_OMP_CRITICAL:
2470 for (; ctx != NULL; ctx = ctx->outer)
2471 if (gimple_code (ctx->stmt) == GIMPLE_OMP_CRITICAL
2472 && (gimple_omp_critical_name (stmt)
2473 == gimple_omp_critical_name (ctx->stmt)))
2475 error_at (gimple_location (stmt),
2476 "critical region may not be nested inside a critical "
2477 "region with the same name");
2478 return false;
2480 break;
2481 case GIMPLE_OMP_TEAMS:
2482 if (ctx == NULL
2483 || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
2484 || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION)
2486 error_at (gimple_location (stmt),
2487 "teams construct not closely nested inside of target "
2488 "region");
2489 return false;
2491 break;
2492 default:
2493 break;
2495 return true;
2499 /* Helper function scan_omp.
2501 Callback for walk_tree or operators in walk_gimple_stmt used to
2502 scan for OpenMP directives in TP. */
2504 static tree
2505 scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
2507 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
2508 omp_context *ctx = (omp_context *) wi->info;
2509 tree t = *tp;
2511 switch (TREE_CODE (t))
2513 case VAR_DECL:
2514 case PARM_DECL:
2515 case LABEL_DECL:
2516 case RESULT_DECL:
2517 if (ctx)
2518 *tp = remap_decl (t, &ctx->cb);
2519 break;
2521 default:
2522 if (ctx && TYPE_P (t))
2523 *tp = remap_type (t, &ctx->cb);
2524 else if (!DECL_P (t))
2526 *walk_subtrees = 1;
2527 if (ctx)
2529 tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
2530 if (tem != TREE_TYPE (t))
2532 if (TREE_CODE (t) == INTEGER_CST)
2533 *tp = build_int_cst_wide (tem,
2534 TREE_INT_CST_LOW (t),
2535 TREE_INT_CST_HIGH (t));
2536 else
2537 TREE_TYPE (t) = tem;
2541 break;
2544 return NULL_TREE;
2547 /* Return true if FNDECL is a setjmp or a longjmp. */
2549 static bool
2550 setjmp_or_longjmp_p (const_tree fndecl)
2552 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
2553 && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP
2554 || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP))
2555 return true;
2557 tree declname = DECL_NAME (fndecl);
2558 if (!declname)
2559 return false;
2560 const char *name = IDENTIFIER_POINTER (declname);
2561 return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
2565 /* Helper function for scan_omp.
2567 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2568 the current statement in GSI. */
2570 static tree
2571 scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
2572 struct walk_stmt_info *wi)
2574 gimple stmt = gsi_stmt (*gsi);
2575 omp_context *ctx = (omp_context *) wi->info;
2577 if (gimple_has_location (stmt))
2578 input_location = gimple_location (stmt);
2580 /* Check the OpenMP nesting restrictions. */
2581 bool remove = false;
2582 if (is_gimple_omp (stmt))
2583 remove = !check_omp_nesting_restrictions (stmt, ctx);
2584 else if (is_gimple_call (stmt))
2586 tree fndecl = gimple_call_fndecl (stmt);
2587 if (fndecl)
2589 if (setjmp_or_longjmp_p (fndecl)
2590 && ctx
2591 && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
2592 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD)
2594 remove = true;
2595 error_at (gimple_location (stmt),
2596 "setjmp/longjmp inside simd construct");
2598 else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
2599 switch (DECL_FUNCTION_CODE (fndecl))
2601 case BUILT_IN_GOMP_BARRIER:
2602 case BUILT_IN_GOMP_CANCEL:
2603 case BUILT_IN_GOMP_CANCELLATION_POINT:
2604 case BUILT_IN_GOMP_TASKYIELD:
2605 case BUILT_IN_GOMP_TASKWAIT:
2606 case BUILT_IN_GOMP_TASKGROUP_START:
2607 case BUILT_IN_GOMP_TASKGROUP_END:
2608 remove = !check_omp_nesting_restrictions (stmt, ctx);
2609 break;
2610 default:
2611 break;
2615 if (remove)
2617 stmt = gimple_build_nop ();
2618 gsi_replace (gsi, stmt, false);
2621 *handled_ops_p = true;
2623 switch (gimple_code (stmt))
2625 case GIMPLE_OMP_PARALLEL:
2626 taskreg_nesting_level++;
2627 scan_omp_parallel (gsi, ctx);
2628 taskreg_nesting_level--;
2629 break;
2631 case GIMPLE_OMP_TASK:
2632 taskreg_nesting_level++;
2633 scan_omp_task (gsi, ctx);
2634 taskreg_nesting_level--;
2635 break;
2637 case GIMPLE_OMP_FOR:
2638 scan_omp_for (stmt, ctx);
2639 break;
2641 case GIMPLE_OMP_SECTIONS:
2642 scan_omp_sections (stmt, ctx);
2643 break;
2645 case GIMPLE_OMP_SINGLE:
2646 scan_omp_single (stmt, ctx);
2647 break;
2649 case GIMPLE_OMP_SECTION:
2650 case GIMPLE_OMP_MASTER:
2651 case GIMPLE_OMP_TASKGROUP:
2652 case GIMPLE_OMP_ORDERED:
2653 case GIMPLE_OMP_CRITICAL:
2654 ctx = new_omp_context (stmt, ctx);
2655 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2656 break;
2658 case GIMPLE_OMP_TARGET:
2659 scan_omp_target (stmt, ctx);
2660 break;
2662 case GIMPLE_OMP_TEAMS:
2663 scan_omp_teams (stmt, ctx);
2664 break;
2666 case GIMPLE_BIND:
2668 tree var;
2670 *handled_ops_p = false;
2671 if (ctx)
2672 for (var = gimple_bind_vars (stmt); var ; var = DECL_CHAIN (var))
2673 insert_decl_map (&ctx->cb, var, var);
2675 break;
2676 default:
2677 *handled_ops_p = false;
2678 break;
2681 return NULL_TREE;
2685 /* Scan all the statements starting at the current statement. CTX
2686 contains context information about the OpenMP directives and
2687 clauses found during the scan. */
2689 static void
2690 scan_omp (gimple_seq *body_p, omp_context *ctx)
2692 location_t saved_location;
2693 struct walk_stmt_info wi;
2695 memset (&wi, 0, sizeof (wi));
2696 wi.info = ctx;
2697 wi.want_locations = true;
2699 saved_location = input_location;
2700 walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
2701 input_location = saved_location;
2704 /* Re-gimplification and code generation routines. */
2706 /* Build a call to GOMP_barrier. */
2708 static gimple
2709 build_omp_barrier (tree lhs)
2711 tree fndecl = builtin_decl_explicit (lhs ? BUILT_IN_GOMP_BARRIER_CANCEL
2712 : BUILT_IN_GOMP_BARRIER);
2713 gimple g = gimple_build_call (fndecl, 0);
2714 if (lhs)
2715 gimple_call_set_lhs (g, lhs);
2716 return g;
2719 /* If a context was created for STMT when it was scanned, return it. */
2721 static omp_context *
2722 maybe_lookup_ctx (gimple stmt)
2724 splay_tree_node n;
2725 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
2726 return n ? (omp_context *) n->value : NULL;
2730 /* Find the mapping for DECL in CTX or the immediately enclosing
2731 context that has a mapping for DECL.
2733 If CTX is a nested parallel directive, we may have to use the decl
2734 mappings created in CTX's parent context. Suppose that we have the
2735 following parallel nesting (variable UIDs showed for clarity):
2737 iD.1562 = 0;
2738 #omp parallel shared(iD.1562) -> outer parallel
2739 iD.1562 = iD.1562 + 1;
2741 #omp parallel shared (iD.1562) -> inner parallel
2742 iD.1562 = iD.1562 - 1;
2744 Each parallel structure will create a distinct .omp_data_s structure
2745 for copying iD.1562 in/out of the directive:
2747 outer parallel .omp_data_s.1.i -> iD.1562
2748 inner parallel .omp_data_s.2.i -> iD.1562
2750 A shared variable mapping will produce a copy-out operation before
2751 the parallel directive and a copy-in operation after it. So, in
2752 this case we would have:
2754 iD.1562 = 0;
2755 .omp_data_o.1.i = iD.1562;
2756 #omp parallel shared(iD.1562) -> outer parallel
2757 .omp_data_i.1 = &.omp_data_o.1
2758 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2760 .omp_data_o.2.i = iD.1562; -> **
2761 #omp parallel shared(iD.1562) -> inner parallel
2762 .omp_data_i.2 = &.omp_data_o.2
2763 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2766 ** This is a problem. The symbol iD.1562 cannot be referenced
2767 inside the body of the outer parallel region. But since we are
2768 emitting this copy operation while expanding the inner parallel
2769 directive, we need to access the CTX structure of the outer
2770 parallel directive to get the correct mapping:
2772 .omp_data_o.2.i = .omp_data_i.1->i
2774 Since there may be other workshare or parallel directives enclosing
2775 the parallel directive, it may be necessary to walk up the context
2776 parent chain. This is not a problem in general because nested
2777 parallelism happens only rarely. */
2779 static tree
2780 lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
2782 tree t;
2783 omp_context *up;
2785 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
2786 t = maybe_lookup_decl (decl, up);
2788 gcc_assert (!ctx->is_nested || t || is_global_var (decl));
2790 return t ? t : decl;
2794 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2795 in outer contexts. */
2797 static tree
2798 maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
2800 tree t = NULL;
2801 omp_context *up;
2803 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
2804 t = maybe_lookup_decl (decl, up);
2806 return t ? t : decl;
2810 /* Construct the initialization value for reduction CLAUSE. */
2812 tree
2813 omp_reduction_init (tree clause, tree type)
2815 location_t loc = OMP_CLAUSE_LOCATION (clause);
2816 switch (OMP_CLAUSE_REDUCTION_CODE (clause))
2818 case PLUS_EXPR:
2819 case MINUS_EXPR:
2820 case BIT_IOR_EXPR:
2821 case BIT_XOR_EXPR:
2822 case TRUTH_OR_EXPR:
2823 case TRUTH_ORIF_EXPR:
2824 case TRUTH_XOR_EXPR:
2825 case NE_EXPR:
2826 return build_zero_cst (type);
2828 case MULT_EXPR:
2829 case TRUTH_AND_EXPR:
2830 case TRUTH_ANDIF_EXPR:
2831 case EQ_EXPR:
2832 return fold_convert_loc (loc, type, integer_one_node);
2834 case BIT_AND_EXPR:
2835 return fold_convert_loc (loc, type, integer_minus_one_node);
2837 case MAX_EXPR:
2838 if (SCALAR_FLOAT_TYPE_P (type))
2840 REAL_VALUE_TYPE max, min;
2841 if (HONOR_INFINITIES (TYPE_MODE (type)))
2843 real_inf (&max);
2844 real_arithmetic (&min, NEGATE_EXPR, &max, NULL);
2846 else
2847 real_maxval (&min, 1, TYPE_MODE (type));
2848 return build_real (type, min);
2850 else
2852 gcc_assert (INTEGRAL_TYPE_P (type));
2853 return TYPE_MIN_VALUE (type);
2856 case MIN_EXPR:
2857 if (SCALAR_FLOAT_TYPE_P (type))
2859 REAL_VALUE_TYPE max;
2860 if (HONOR_INFINITIES (TYPE_MODE (type)))
2861 real_inf (&max);
2862 else
2863 real_maxval (&max, 0, TYPE_MODE (type));
2864 return build_real (type, max);
2866 else
2868 gcc_assert (INTEGRAL_TYPE_P (type));
2869 return TYPE_MAX_VALUE (type);
2872 default:
2873 gcc_unreachable ();
2877 /* Return alignment to be assumed for var in CLAUSE, which should be
2878 OMP_CLAUSE_ALIGNED. */
2880 static tree
2881 omp_clause_aligned_alignment (tree clause)
2883 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
2884 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
2886 /* Otherwise return implementation defined alignment. */
2887 unsigned int al = 1;
2888 enum machine_mode mode, vmode;
2889 int vs = targetm.vectorize.autovectorize_vector_sizes ();
2890 if (vs)
2891 vs = 1 << floor_log2 (vs);
2892 static enum mode_class classes[]
2893 = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
2894 for (int i = 0; i < 4; i += 2)
2895 for (mode = GET_CLASS_NARROWEST_MODE (classes[i]);
2896 mode != VOIDmode;
2897 mode = GET_MODE_WIDER_MODE (mode))
2899 vmode = targetm.vectorize.preferred_simd_mode (mode);
2900 if (GET_MODE_CLASS (vmode) != classes[i + 1])
2901 continue;
2902 while (vs
2903 && GET_MODE_SIZE (vmode) < vs
2904 && GET_MODE_2XWIDER_MODE (vmode) != VOIDmode)
2905 vmode = GET_MODE_2XWIDER_MODE (vmode);
2907 tree type = lang_hooks.types.type_for_mode (mode, 1);
2908 if (type == NULL_TREE || TYPE_MODE (type) != mode)
2909 continue;
2910 type = build_vector_type (type, GET_MODE_SIZE (vmode)
2911 / GET_MODE_SIZE (mode));
2912 if (TYPE_MODE (type) != vmode)
2913 continue;
2914 if (TYPE_ALIGN_UNIT (type) > al)
2915 al = TYPE_ALIGN_UNIT (type);
2917 return build_int_cst (integer_type_node, al);
2920 /* Return maximum possible vectorization factor for the target. */
2922 static int
2923 omp_max_vf (void)
2925 if (!optimize
2926 || optimize_debug
2927 || (!flag_tree_loop_vectorize
2928 && (global_options_set.x_flag_tree_loop_vectorize
2929 || global_options_set.x_flag_tree_vectorize)))
2930 return 1;
2932 int vs = targetm.vectorize.autovectorize_vector_sizes ();
2933 if (vs)
2935 vs = 1 << floor_log2 (vs);
2936 return vs;
2938 enum machine_mode vqimode = targetm.vectorize.preferred_simd_mode (QImode);
2939 if (GET_MODE_CLASS (vqimode) == MODE_VECTOR_INT)
2940 return GET_MODE_NUNITS (vqimode);
2941 return 1;
2944 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2945 privatization. */
2947 static bool
2948 lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, int &max_vf,
2949 tree &idx, tree &lane, tree &ivar, tree &lvar)
2951 if (max_vf == 0)
2953 max_vf = omp_max_vf ();
2954 if (max_vf > 1)
2956 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2957 OMP_CLAUSE_SAFELEN);
2958 if (c
2959 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c), max_vf) == -1)
2960 max_vf = tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c));
2962 if (max_vf > 1)
2964 idx = create_tmp_var (unsigned_type_node, NULL);
2965 lane = create_tmp_var (unsigned_type_node, NULL);
2968 if (max_vf == 1)
2969 return false;
2971 tree atype = build_array_type_nelts (TREE_TYPE (new_var), max_vf);
2972 tree avar = create_tmp_var_raw (atype, NULL);
2973 if (TREE_ADDRESSABLE (new_var))
2974 TREE_ADDRESSABLE (avar) = 1;
2975 DECL_ATTRIBUTES (avar)
2976 = tree_cons (get_identifier ("omp simd array"), NULL,
2977 DECL_ATTRIBUTES (avar));
2978 gimple_add_tmp_var (avar);
2979 ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, idx,
2980 NULL_TREE, NULL_TREE);
2981 lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, lane,
2982 NULL_TREE, NULL_TREE);
2983 if (DECL_P (new_var))
2985 SET_DECL_VALUE_EXPR (new_var, lvar);
2986 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
2988 return true;
2991 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
2992 from the receiver (aka child) side and initializers for REFERENCE_TYPE
2993 private variables. Initialization statements go in ILIST, while calls
2994 to destructors go in DLIST. */
2996 static void
2997 lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
2998 omp_context *ctx, struct omp_for_data *fd)
3000 tree c, dtor, copyin_seq, x, ptr;
3001 bool copyin_by_ref = false;
3002 bool lastprivate_firstprivate = false;
3003 bool reduction_omp_orig_ref = false;
3004 int pass;
3005 bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3006 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD);
3007 int max_vf = 0;
3008 tree lane = NULL_TREE, idx = NULL_TREE;
3009 tree ivar = NULL_TREE, lvar = NULL_TREE;
3010 gimple_seq llist[2] = { NULL, NULL };
3012 copyin_seq = NULL;
3014 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3015 with data sharing clauses referencing variable sized vars. That
3016 is unnecessarily hard to support and very unlikely to result in
3017 vectorized code anyway. */
3018 if (is_simd)
3019 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3020 switch (OMP_CLAUSE_CODE (c))
3022 case OMP_CLAUSE_REDUCTION:
3023 case OMP_CLAUSE_PRIVATE:
3024 case OMP_CLAUSE_FIRSTPRIVATE:
3025 case OMP_CLAUSE_LASTPRIVATE:
3026 case OMP_CLAUSE_LINEAR:
3027 if (is_variable_sized (OMP_CLAUSE_DECL (c)))
3028 max_vf = 1;
3029 break;
3030 default:
3031 continue;
3034 /* Do all the fixed sized types in the first pass, and the variable sized
3035 types in the second pass. This makes sure that the scalar arguments to
3036 the variable sized types are processed before we use them in the
3037 variable sized operations. */
3038 for (pass = 0; pass < 2; ++pass)
3040 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3042 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
3043 tree var, new_var;
3044 bool by_ref;
3045 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3047 switch (c_kind)
3049 case OMP_CLAUSE_PRIVATE:
3050 if (OMP_CLAUSE_PRIVATE_DEBUG (c))
3051 continue;
3052 break;
3053 case OMP_CLAUSE_SHARED:
3054 /* Ignore shared directives in teams construct. */
3055 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3056 continue;
3057 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
3059 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c)));
3060 continue;
3062 case OMP_CLAUSE_FIRSTPRIVATE:
3063 case OMP_CLAUSE_COPYIN:
3064 case OMP_CLAUSE_LINEAR:
3065 break;
3066 case OMP_CLAUSE_REDUCTION:
3067 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
3068 reduction_omp_orig_ref = true;
3069 break;
3070 case OMP_CLAUSE__LOOPTEMP_:
3071 /* Handle _looptemp_ clauses only on parallel. */
3072 if (fd)
3073 continue;
3074 break;
3075 case OMP_CLAUSE_LASTPRIVATE:
3076 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3078 lastprivate_firstprivate = true;
3079 if (pass != 0)
3080 continue;
3082 break;
3083 case OMP_CLAUSE_ALIGNED:
3084 if (pass == 0)
3085 continue;
3086 var = OMP_CLAUSE_DECL (c);
3087 if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
3088 && !is_global_var (var))
3090 new_var = maybe_lookup_decl (var, ctx);
3091 if (new_var == NULL_TREE)
3092 new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
3093 x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3094 x = build_call_expr_loc (clause_loc, x, 2, new_var,
3095 omp_clause_aligned_alignment (c));
3096 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3097 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
3098 gimplify_and_add (x, ilist);
3100 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
3101 && is_global_var (var))
3103 tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
3104 new_var = lookup_decl (var, ctx);
3105 t = maybe_lookup_decl_in_outer_ctx (var, ctx);
3106 t = build_fold_addr_expr_loc (clause_loc, t);
3107 t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3108 t = build_call_expr_loc (clause_loc, t2, 2, t,
3109 omp_clause_aligned_alignment (c));
3110 t = fold_convert_loc (clause_loc, ptype, t);
3111 x = create_tmp_var (ptype, NULL);
3112 t = build2 (MODIFY_EXPR, ptype, x, t);
3113 gimplify_and_add (t, ilist);
3114 t = build_simple_mem_ref_loc (clause_loc, x);
3115 SET_DECL_VALUE_EXPR (new_var, t);
3116 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3118 continue;
3119 default:
3120 continue;
3123 new_var = var = OMP_CLAUSE_DECL (c);
3124 if (c_kind != OMP_CLAUSE_COPYIN)
3125 new_var = lookup_decl (var, ctx);
3127 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
3129 if (pass != 0)
3130 continue;
3132 else if (is_variable_sized (var))
3134 /* For variable sized types, we need to allocate the
3135 actual storage here. Call alloca and store the
3136 result in the pointer decl that we created elsewhere. */
3137 if (pass == 0)
3138 continue;
3140 if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
3142 gimple stmt;
3143 tree tmp, atmp;
3145 ptr = DECL_VALUE_EXPR (new_var);
3146 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
3147 ptr = TREE_OPERAND (ptr, 0);
3148 gcc_assert (DECL_P (ptr));
3149 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
3151 /* void *tmp = __builtin_alloca */
3152 atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3153 stmt = gimple_build_call (atmp, 1, x);
3154 tmp = create_tmp_var_raw (ptr_type_node, NULL);
3155 gimple_add_tmp_var (tmp);
3156 gimple_call_set_lhs (stmt, tmp);
3158 gimple_seq_add_stmt (ilist, stmt);
3160 x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
3161 gimplify_assign (ptr, x, ilist);
3164 else if (is_reference (var))
3166 /* For references that are being privatized for Fortran,
3167 allocate new backing storage for the new pointer
3168 variable. This allows us to avoid changing all the
3169 code that expects a pointer to something that expects
3170 a direct variable. */
3171 if (pass == 0)
3172 continue;
3174 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
3175 if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
3177 x = build_receiver_ref (var, false, ctx);
3178 x = build_fold_addr_expr_loc (clause_loc, x);
3180 else if (TREE_CONSTANT (x))
3182 const char *name = NULL;
3183 if (DECL_NAME (var))
3184 name = IDENTIFIER_POINTER (DECL_NAME (new_var));
3186 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
3187 name);
3188 gimple_add_tmp_var (x);
3189 TREE_ADDRESSABLE (x) = 1;
3190 x = build_fold_addr_expr_loc (clause_loc, x);
3192 else
3194 tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3195 x = build_call_expr_loc (clause_loc, atmp, 1, x);
3198 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3199 gimplify_assign (new_var, x, ilist);
3201 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3203 else if (c_kind == OMP_CLAUSE_REDUCTION
3204 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3206 if (pass == 0)
3207 continue;
3209 else if (pass != 0)
3210 continue;
3212 switch (OMP_CLAUSE_CODE (c))
3214 case OMP_CLAUSE_SHARED:
3215 /* Ignore shared directives in teams construct. */
3216 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3217 continue;
3218 /* Shared global vars are just accessed directly. */
3219 if (is_global_var (new_var))
3220 break;
3221 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3222 needs to be delayed until after fixup_child_record_type so
3223 that we get the correct type during the dereference. */
3224 by_ref = use_pointer_for_field (var, ctx);
3225 x = build_receiver_ref (var, by_ref, ctx);
3226 SET_DECL_VALUE_EXPR (new_var, x);
3227 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3229 /* ??? If VAR is not passed by reference, and the variable
3230 hasn't been initialized yet, then we'll get a warning for
3231 the store into the omp_data_s structure. Ideally, we'd be
3232 able to notice this and not store anything at all, but
3233 we're generating code too early. Suppress the warning. */
3234 if (!by_ref)
3235 TREE_NO_WARNING (var) = 1;
3236 break;
3238 case OMP_CLAUSE_LASTPRIVATE:
3239 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3240 break;
3241 /* FALLTHRU */
3243 case OMP_CLAUSE_PRIVATE:
3244 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
3245 x = build_outer_var_ref (var, ctx);
3246 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
3248 if (is_task_ctx (ctx))
3249 x = build_receiver_ref (var, false, ctx);
3250 else
3251 x = build_outer_var_ref (var, ctx);
3253 else
3254 x = NULL;
3255 do_private:
3256 tree nx;
3257 nx = lang_hooks.decls.omp_clause_default_ctor (c, new_var, x);
3258 if (is_simd)
3260 tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
3261 if ((TREE_ADDRESSABLE (new_var) || nx || y
3262 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
3263 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3264 idx, lane, ivar, lvar))
3266 if (nx)
3267 x = lang_hooks.decls.omp_clause_default_ctor
3268 (c, unshare_expr (ivar), x);
3269 if (nx && x)
3270 gimplify_and_add (x, &llist[0]);
3271 if (y)
3273 y = lang_hooks.decls.omp_clause_dtor (c, ivar);
3274 if (y)
3276 gimple_seq tseq = NULL;
3278 dtor = y;
3279 gimplify_stmt (&dtor, &tseq);
3280 gimple_seq_add_seq (&llist[1], tseq);
3283 break;
3286 if (nx)
3287 gimplify_and_add (nx, ilist);
3288 /* FALLTHRU */
3290 do_dtor:
3291 x = lang_hooks.decls.omp_clause_dtor (c, new_var);
3292 if (x)
3294 gimple_seq tseq = NULL;
3296 dtor = x;
3297 gimplify_stmt (&dtor, &tseq);
3298 gimple_seq_add_seq (dlist, tseq);
3300 break;
3302 case OMP_CLAUSE_LINEAR:
3303 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
3304 goto do_firstprivate;
3305 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
3306 x = NULL;
3307 else
3308 x = build_outer_var_ref (var, ctx);
3309 goto do_private;
3311 case OMP_CLAUSE_FIRSTPRIVATE:
3312 if (is_task_ctx (ctx))
3314 if (is_reference (var) || is_variable_sized (var))
3315 goto do_dtor;
3316 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
3317 ctx))
3318 || use_pointer_for_field (var, NULL))
3320 x = build_receiver_ref (var, false, ctx);
3321 SET_DECL_VALUE_EXPR (new_var, x);
3322 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3323 goto do_dtor;
3326 do_firstprivate:
3327 x = build_outer_var_ref (var, ctx);
3328 if (is_simd)
3330 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
3331 && gimple_omp_for_combined_into_p (ctx->stmt))
3333 tree stept = POINTER_TYPE_P (TREE_TYPE (x))
3334 ? sizetype : TREE_TYPE (x);
3335 tree t = fold_convert (stept,
3336 OMP_CLAUSE_LINEAR_STEP (c));
3337 tree c = find_omp_clause (clauses,
3338 OMP_CLAUSE__LOOPTEMP_);
3339 gcc_assert (c);
3340 tree l = OMP_CLAUSE_DECL (c);
3341 if (fd->collapse == 1)
3343 tree n1 = fd->loop.n1;
3344 tree step = fd->loop.step;
3345 tree itype = TREE_TYPE (l);
3346 if (POINTER_TYPE_P (itype))
3347 itype = signed_type_for (itype);
3348 l = fold_build2 (MINUS_EXPR, itype, l, n1);
3349 if (TYPE_UNSIGNED (itype)
3350 && fd->loop.cond_code == GT_EXPR)
3351 l = fold_build2 (TRUNC_DIV_EXPR, itype,
3352 fold_build1 (NEGATE_EXPR,
3353 itype, l),
3354 fold_build1 (NEGATE_EXPR,
3355 itype, step));
3356 else
3357 l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
3359 t = fold_build2 (MULT_EXPR, stept,
3360 fold_convert (stept, l), t);
3361 if (POINTER_TYPE_P (TREE_TYPE (x)))
3362 x = fold_build2 (POINTER_PLUS_EXPR,
3363 TREE_TYPE (x), x, t);
3364 else
3365 x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, t);
3368 if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
3369 || TREE_ADDRESSABLE (new_var))
3370 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3371 idx, lane, ivar, lvar))
3373 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
3375 tree iv = create_tmp_var (TREE_TYPE (new_var), NULL);
3376 x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
3377 gimplify_and_add (x, ilist);
3378 gimple_stmt_iterator gsi
3379 = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
3380 gimple g
3381 = gimple_build_assign (unshare_expr (lvar), iv);
3382 gsi_insert_before_without_update (&gsi, g,
3383 GSI_SAME_STMT);
3384 tree stept = POINTER_TYPE_P (TREE_TYPE (x))
3385 ? sizetype : TREE_TYPE (x);
3386 tree t = fold_convert (stept,
3387 OMP_CLAUSE_LINEAR_STEP (c));
3388 enum tree_code code = PLUS_EXPR;
3389 if (POINTER_TYPE_P (TREE_TYPE (new_var)))
3390 code = POINTER_PLUS_EXPR;
3391 g = gimple_build_assign_with_ops (code, iv, iv, t);
3392 gsi_insert_before_without_update (&gsi, g,
3393 GSI_SAME_STMT);
3394 break;
3396 x = lang_hooks.decls.omp_clause_copy_ctor
3397 (c, unshare_expr (ivar), x);
3398 gimplify_and_add (x, &llist[0]);
3399 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
3400 if (x)
3402 gimple_seq tseq = NULL;
3404 dtor = x;
3405 gimplify_stmt (&dtor, &tseq);
3406 gimple_seq_add_seq (&llist[1], tseq);
3408 break;
3411 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
3412 gimplify_and_add (x, ilist);
3413 goto do_dtor;
3415 case OMP_CLAUSE__LOOPTEMP_:
3416 gcc_assert (is_parallel_ctx (ctx));
3417 x = build_outer_var_ref (var, ctx);
3418 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
3419 gimplify_and_add (x, ilist);
3420 break;
3422 case OMP_CLAUSE_COPYIN:
3423 by_ref = use_pointer_for_field (var, NULL);
3424 x = build_receiver_ref (var, by_ref, ctx);
3425 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
3426 append_to_statement_list (x, &copyin_seq);
3427 copyin_by_ref |= by_ref;
3428 break;
3430 case OMP_CLAUSE_REDUCTION:
3431 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3433 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
3434 gimple tseq;
3435 x = build_outer_var_ref (var, ctx);
3437 if (is_reference (var)
3438 && !useless_type_conversion_p (TREE_TYPE (placeholder),
3439 TREE_TYPE (x)))
3440 x = build_fold_addr_expr_loc (clause_loc, x);
3441 SET_DECL_VALUE_EXPR (placeholder, x);
3442 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
3443 tree new_vard = new_var;
3444 if (is_reference (var))
3446 gcc_assert (TREE_CODE (new_var) == MEM_REF);
3447 new_vard = TREE_OPERAND (new_var, 0);
3448 gcc_assert (DECL_P (new_vard));
3450 if (is_simd
3451 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3452 idx, lane, ivar, lvar))
3454 if (new_vard == new_var)
3456 gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
3457 SET_DECL_VALUE_EXPR (new_var, ivar);
3459 else
3461 SET_DECL_VALUE_EXPR (new_vard,
3462 build_fold_addr_expr (ivar));
3463 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
3465 x = lang_hooks.decls.omp_clause_default_ctor
3466 (c, unshare_expr (ivar),
3467 build_outer_var_ref (var, ctx));
3468 if (x)
3469 gimplify_and_add (x, &llist[0]);
3470 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
3472 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
3473 lower_omp (&tseq, ctx);
3474 gimple_seq_add_seq (&llist[0], tseq);
3476 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
3477 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
3478 lower_omp (&tseq, ctx);
3479 gimple_seq_add_seq (&llist[1], tseq);
3480 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
3481 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
3482 if (new_vard == new_var)
3483 SET_DECL_VALUE_EXPR (new_var, lvar);
3484 else
3485 SET_DECL_VALUE_EXPR (new_vard,
3486 build_fold_addr_expr (lvar));
3487 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
3488 if (x)
3490 tseq = NULL;
3491 dtor = x;
3492 gimplify_stmt (&dtor, &tseq);
3493 gimple_seq_add_seq (&llist[1], tseq);
3495 break;
3497 x = lang_hooks.decls.omp_clause_default_ctor
3498 (c, new_var, unshare_expr (x));
3499 if (x)
3500 gimplify_and_add (x, ilist);
3501 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
3503 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
3504 lower_omp (&tseq, ctx);
3505 gimple_seq_add_seq (ilist, tseq);
3507 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
3508 if (is_simd)
3510 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
3511 lower_omp (&tseq, ctx);
3512 gimple_seq_add_seq (dlist, tseq);
3513 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
3515 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
3516 goto do_dtor;
3518 else
3520 x = omp_reduction_init (c, TREE_TYPE (new_var));
3521 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
3522 if (is_simd
3523 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3524 idx, lane, ivar, lvar))
3526 enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
3527 tree ref = build_outer_var_ref (var, ctx);
3529 gimplify_assign (unshare_expr (ivar), x, &llist[0]);
3531 /* reduction(-:var) sums up the partial results, so it
3532 acts identically to reduction(+:var). */
3533 if (code == MINUS_EXPR)
3534 code = PLUS_EXPR;
3536 x = build2 (code, TREE_TYPE (ref), ref, ivar);
3537 ref = build_outer_var_ref (var, ctx);
3538 gimplify_assign (ref, x, &llist[1]);
3540 else
3542 gimplify_assign (new_var, x, ilist);
3543 if (is_simd)
3544 gimplify_assign (build_outer_var_ref (var, ctx),
3545 new_var, dlist);
3548 break;
3550 default:
3551 gcc_unreachable ();
3556 if (lane)
3558 tree uid = create_tmp_var (ptr_type_node, "simduid");
3559 /* Don't want uninit warnings on simduid, it is always uninitialized,
3560 but we use it not for the value, but for the DECL_UID only. */
3561 TREE_NO_WARNING (uid) = 1;
3562 gimple g
3563 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid);
3564 gimple_call_set_lhs (g, lane);
3565 gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
3566 gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
3567 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
3568 OMP_CLAUSE__SIMDUID__DECL (c) = uid;
3569 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
3570 gimple_omp_for_set_clauses (ctx->stmt, c);
3571 g = gimple_build_assign_with_ops (INTEGER_CST, lane,
3572 build_int_cst (unsigned_type_node, 0),
3573 NULL_TREE);
3574 gimple_seq_add_stmt (ilist, g);
3575 for (int i = 0; i < 2; i++)
3576 if (llist[i])
3578 tree vf = create_tmp_var (unsigned_type_node, NULL);
3579 g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
3580 gimple_call_set_lhs (g, vf);
3581 gimple_seq *seq = i == 0 ? ilist : dlist;
3582 gimple_seq_add_stmt (seq, g);
3583 tree t = build_int_cst (unsigned_type_node, 0);
3584 g = gimple_build_assign_with_ops (INTEGER_CST, idx, t, NULL_TREE);
3585 gimple_seq_add_stmt (seq, g);
3586 tree body = create_artificial_label (UNKNOWN_LOCATION);
3587 tree header = create_artificial_label (UNKNOWN_LOCATION);
3588 tree end = create_artificial_label (UNKNOWN_LOCATION);
3589 gimple_seq_add_stmt (seq, gimple_build_goto (header));
3590 gimple_seq_add_stmt (seq, gimple_build_label (body));
3591 gimple_seq_add_seq (seq, llist[i]);
3592 t = build_int_cst (unsigned_type_node, 1);
3593 g = gimple_build_assign_with_ops (PLUS_EXPR, idx, idx, t);
3594 gimple_seq_add_stmt (seq, g);
3595 gimple_seq_add_stmt (seq, gimple_build_label (header));
3596 g = gimple_build_cond (LT_EXPR, idx, vf, body, end);
3597 gimple_seq_add_stmt (seq, g);
3598 gimple_seq_add_stmt (seq, gimple_build_label (end));
3602 /* The copyin sequence is not to be executed by the main thread, since
3603 that would result in self-copies. Perhaps not visible to scalars,
3604 but it certainly is to C++ operator=. */
3605 if (copyin_seq)
3607 x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
3609 x = build2 (NE_EXPR, boolean_type_node, x,
3610 build_int_cst (TREE_TYPE (x), 0));
3611 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
3612 gimplify_and_add (x, ilist);
3615 /* If any copyin variable is passed by reference, we must ensure the
3616 master thread doesn't modify it before it is copied over in all
3617 threads. Similarly for variables in both firstprivate and
3618 lastprivate clauses we need to ensure the lastprivate copying
3619 happens after firstprivate copying in all threads. And similarly
3620 for UDRs if initializer expression refers to omp_orig. */
3621 if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref)
3623 /* Don't add any barrier for #pragma omp simd or
3624 #pragma omp distribute. */
3625 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
3626 || gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_FOR)
3627 gimple_seq_add_stmt (ilist, build_omp_barrier (NULL_TREE));
3630 /* If max_vf is non-zero, then we can use only a vectorization factor
3631 up to the max_vf we chose. So stick it into the safelen clause. */
3632 if (max_vf)
3634 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3635 OMP_CLAUSE_SAFELEN);
3636 if (c == NULL_TREE
3637 || compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
3638 max_vf) == 1)
3640 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
3641 OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
3642 max_vf);
3643 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
3644 gimple_omp_for_set_clauses (ctx->stmt, c);
3650 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3651 both parallel and workshare constructs. PREDICATE may be NULL if it's
3652 always true. */
3654 static void
3655 lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list,
3656 omp_context *ctx)
3658 tree x, c, label = NULL, orig_clauses = clauses;
3659 bool par_clauses = false;
3660 tree simduid = NULL, lastlane = NULL;
3662 /* Early exit if there are no lastprivate or linear clauses. */
3663 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
3664 if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
3665 || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
3666 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
3667 break;
3668 if (clauses == NULL)
3670 /* If this was a workshare clause, see if it had been combined
3671 with its parallel. In that case, look for the clauses on the
3672 parallel statement itself. */
3673 if (is_parallel_ctx (ctx))
3674 return;
3676 ctx = ctx->outer;
3677 if (ctx == NULL || !is_parallel_ctx (ctx))
3678 return;
3680 clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
3681 OMP_CLAUSE_LASTPRIVATE);
3682 if (clauses == NULL)
3683 return;
3684 par_clauses = true;
3687 if (predicate)
3689 gimple stmt;
3690 tree label_true, arm1, arm2;
3692 label = create_artificial_label (UNKNOWN_LOCATION);
3693 label_true = create_artificial_label (UNKNOWN_LOCATION);
3694 arm1 = TREE_OPERAND (predicate, 0);
3695 arm2 = TREE_OPERAND (predicate, 1);
3696 gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
3697 gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
3698 stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2,
3699 label_true, label);
3700 gimple_seq_add_stmt (stmt_list, stmt);
3701 gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
3704 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3705 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD)
3707 simduid = find_omp_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
3708 if (simduid)
3709 simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
3712 for (c = clauses; c ;)
3714 tree var, new_var;
3715 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3717 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
3718 || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
3719 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
3721 var = OMP_CLAUSE_DECL (c);
3722 new_var = lookup_decl (var, ctx);
3724 if (simduid && DECL_HAS_VALUE_EXPR_P (new_var))
3726 tree val = DECL_VALUE_EXPR (new_var);
3727 if (TREE_CODE (val) == ARRAY_REF
3728 && VAR_P (TREE_OPERAND (val, 0))
3729 && lookup_attribute ("omp simd array",
3730 DECL_ATTRIBUTES (TREE_OPERAND (val,
3731 0))))
3733 if (lastlane == NULL)
3735 lastlane = create_tmp_var (unsigned_type_node, NULL);
3736 gimple g
3737 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
3738 2, simduid,
3739 TREE_OPERAND (val, 1));
3740 gimple_call_set_lhs (g, lastlane);
3741 gimple_seq_add_stmt (stmt_list, g);
3743 new_var = build4 (ARRAY_REF, TREE_TYPE (val),
3744 TREE_OPERAND (val, 0), lastlane,
3745 NULL_TREE, NULL_TREE);
3749 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
3750 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
3752 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
3753 gimple_seq_add_seq (stmt_list,
3754 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
3755 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
3758 x = build_outer_var_ref (var, ctx);
3759 if (is_reference (var))
3760 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3761 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
3762 gimplify_and_add (x, stmt_list);
3764 c = OMP_CLAUSE_CHAIN (c);
3765 if (c == NULL && !par_clauses)
3767 /* If this was a workshare clause, see if it had been combined
3768 with its parallel. In that case, continue looking for the
3769 clauses also on the parallel statement itself. */
3770 if (is_parallel_ctx (ctx))
3771 break;
3773 ctx = ctx->outer;
3774 if (ctx == NULL || !is_parallel_ctx (ctx))
3775 break;
3777 c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
3778 OMP_CLAUSE_LASTPRIVATE);
3779 par_clauses = true;
3783 if (label)
3784 gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
3788 /* Generate code to implement the REDUCTION clauses. */
3790 static void
3791 lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx)
3793 gimple_seq sub_seq = NULL;
3794 gimple stmt;
3795 tree x, c;
3796 int count = 0;
3798 /* SIMD reductions are handled in lower_rec_input_clauses. */
3799 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3800 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_KIND_SIMD)
3801 return;
3803 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3804 update in that case, otherwise use a lock. */
3805 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
3806 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
3808 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3810 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3811 count = -1;
3812 break;
3814 count++;
3817 if (count == 0)
3818 return;
3820 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3822 tree var, ref, new_var;
3823 enum tree_code code;
3824 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3826 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
3827 continue;
3829 var = OMP_CLAUSE_DECL (c);
3830 new_var = lookup_decl (var, ctx);
3831 if (is_reference (var))
3832 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3833 ref = build_outer_var_ref (var, ctx);
3834 code = OMP_CLAUSE_REDUCTION_CODE (c);
3836 /* reduction(-:var) sums up the partial results, so it acts
3837 identically to reduction(+:var). */
3838 if (code == MINUS_EXPR)
3839 code = PLUS_EXPR;
3841 if (count == 1)
3843 tree addr = build_fold_addr_expr_loc (clause_loc, ref);
3845 addr = save_expr (addr);
3846 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
3847 x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var);
3848 x = build2 (OMP_ATOMIC, void_type_node, addr, x);
3849 gimplify_and_add (x, stmt_seqp);
3850 return;
3853 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3855 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
3857 if (is_reference (var)
3858 && !useless_type_conversion_p (TREE_TYPE (placeholder),
3859 TREE_TYPE (ref)))
3860 ref = build_fold_addr_expr_loc (clause_loc, ref);
3861 SET_DECL_VALUE_EXPR (placeholder, ref);
3862 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
3863 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
3864 gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
3865 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
3866 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
3868 else
3870 x = build2 (code, TREE_TYPE (ref), ref, new_var);
3871 ref = build_outer_var_ref (var, ctx);
3872 gimplify_assign (ref, x, &sub_seq);
3876 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START),
3878 gimple_seq_add_stmt (stmt_seqp, stmt);
3880 gimple_seq_add_seq (stmt_seqp, sub_seq);
3882 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
3884 gimple_seq_add_stmt (stmt_seqp, stmt);
3888 /* Generate code to implement the COPYPRIVATE clauses. */
3890 static void
3891 lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
3892 omp_context *ctx)
3894 tree c;
3896 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3898 tree var, new_var, ref, x;
3899 bool by_ref;
3900 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3902 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
3903 continue;
3905 var = OMP_CLAUSE_DECL (c);
3906 by_ref = use_pointer_for_field (var, NULL);
3908 ref = build_sender_ref (var, ctx);
3909 x = new_var = lookup_decl_in_outer_ctx (var, ctx);
3910 if (by_ref)
3912 x = build_fold_addr_expr_loc (clause_loc, new_var);
3913 x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
3915 gimplify_assign (ref, x, slist);
3917 ref = build_receiver_ref (var, false, ctx);
3918 if (by_ref)
3920 ref = fold_convert_loc (clause_loc,
3921 build_pointer_type (TREE_TYPE (new_var)),
3922 ref);
3923 ref = build_fold_indirect_ref_loc (clause_loc, ref);
3925 if (is_reference (var))
3927 ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
3928 ref = build_simple_mem_ref_loc (clause_loc, ref);
3929 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3931 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
3932 gimplify_and_add (x, rlist);
3937 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
3938 and REDUCTION from the sender (aka parent) side. */
3940 static void
3941 lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
3942 omp_context *ctx)
3944 tree c;
3946 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3948 tree val, ref, x, var;
3949 bool by_ref, do_in = false, do_out = false;
3950 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3952 switch (OMP_CLAUSE_CODE (c))
3954 case OMP_CLAUSE_PRIVATE:
3955 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
3956 break;
3957 continue;
3958 case OMP_CLAUSE_FIRSTPRIVATE:
3959 case OMP_CLAUSE_COPYIN:
3960 case OMP_CLAUSE_LASTPRIVATE:
3961 case OMP_CLAUSE_REDUCTION:
3962 case OMP_CLAUSE__LOOPTEMP_:
3963 break;
3964 default:
3965 continue;
3968 val = OMP_CLAUSE_DECL (c);
3969 var = lookup_decl_in_outer_ctx (val, ctx);
3971 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
3972 && is_global_var (var))
3973 continue;
3974 if (is_variable_sized (val))
3975 continue;
3976 by_ref = use_pointer_for_field (val, NULL);
3978 switch (OMP_CLAUSE_CODE (c))
3980 case OMP_CLAUSE_PRIVATE:
3981 case OMP_CLAUSE_FIRSTPRIVATE:
3982 case OMP_CLAUSE_COPYIN:
3983 case OMP_CLAUSE__LOOPTEMP_:
3984 do_in = true;
3985 break;
3987 case OMP_CLAUSE_LASTPRIVATE:
3988 if (by_ref || is_reference (val))
3990 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3991 continue;
3992 do_in = true;
3994 else
3996 do_out = true;
3997 if (lang_hooks.decls.omp_private_outer_ref (val))
3998 do_in = true;
4000 break;
4002 case OMP_CLAUSE_REDUCTION:
4003 do_in = true;
4004 do_out = !(by_ref || is_reference (val));
4005 break;
4007 default:
4008 gcc_unreachable ();
4011 if (do_in)
4013 ref = build_sender_ref (val, ctx);
4014 x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
4015 gimplify_assign (ref, x, ilist);
4016 if (is_task_ctx (ctx))
4017 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
4020 if (do_out)
4022 ref = build_sender_ref (val, ctx);
4023 gimplify_assign (var, ref, olist);
4028 /* Generate code to implement SHARED from the sender (aka parent)
4029 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4030 list things that got automatically shared. */
4032 static void
4033 lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
4035 tree var, ovar, nvar, f, x, record_type;
4037 if (ctx->record_type == NULL)
4038 return;
4040 record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
4041 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
4043 ovar = DECL_ABSTRACT_ORIGIN (f);
4044 nvar = maybe_lookup_decl (ovar, ctx);
4045 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar))
4046 continue;
4048 /* If CTX is a nested parallel directive. Find the immediately
4049 enclosing parallel or workshare construct that contains a
4050 mapping for OVAR. */
4051 var = lookup_decl_in_outer_ctx (ovar, ctx);
4053 if (use_pointer_for_field (ovar, ctx))
4055 x = build_sender_ref (ovar, ctx);
4056 var = build_fold_addr_expr (var);
4057 gimplify_assign (x, var, ilist);
4059 else
4061 x = build_sender_ref (ovar, ctx);
4062 gimplify_assign (x, var, ilist);
4064 if (!TREE_READONLY (var)
4065 /* We don't need to receive a new reference to a result
4066 or parm decl. In fact we may not store to it as we will
4067 invalidate any pending RSO and generate wrong gimple
4068 during inlining. */
4069 && !((TREE_CODE (var) == RESULT_DECL
4070 || TREE_CODE (var) == PARM_DECL)
4071 && DECL_BY_REFERENCE (var)))
4073 x = build_sender_ref (ovar, ctx);
4074 gimplify_assign (var, x, olist);
4081 /* A convenience function to build an empty GIMPLE_COND with just the
4082 condition. */
4084 static gimple
4085 gimple_build_cond_empty (tree cond)
4087 enum tree_code pred_code;
4088 tree lhs, rhs;
4090 gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs);
4091 return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE);
4095 /* Build the function calls to GOMP_parallel_start etc to actually
4096 generate the parallel operation. REGION is the parallel region
4097 being expanded. BB is the block where to insert the code. WS_ARGS
4098 will be set if this is a call to a combined parallel+workshare
4099 construct, it contains the list of additional arguments needed by
4100 the workshare construct. */
4102 static void
4103 expand_parallel_call (struct omp_region *region, basic_block bb,
4104 gimple entry_stmt, vec<tree, va_gc> *ws_args)
4106 tree t, t1, t2, val, cond, c, clauses, flags;
4107 gimple_stmt_iterator gsi;
4108 gimple stmt;
4109 enum built_in_function start_ix;
4110 int start_ix2;
4111 location_t clause_loc;
4112 vec<tree, va_gc> *args;
4114 clauses = gimple_omp_parallel_clauses (entry_stmt);
4116 /* Determine what flavor of GOMP_parallel we will be
4117 emitting. */
4118 start_ix = BUILT_IN_GOMP_PARALLEL;
4119 if (is_combined_parallel (region))
4121 switch (region->inner->type)
4123 case GIMPLE_OMP_FOR:
4124 gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
4125 start_ix2 = ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4126 + (region->inner->sched_kind
4127 == OMP_CLAUSE_SCHEDULE_RUNTIME
4128 ? 3 : region->inner->sched_kind));
4129 start_ix = (enum built_in_function)start_ix2;
4130 break;
4131 case GIMPLE_OMP_SECTIONS:
4132 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS;
4133 break;
4134 default:
4135 gcc_unreachable ();
4139 /* By default, the value of NUM_THREADS is zero (selected at run time)
4140 and there is no conditional. */
4141 cond = NULL_TREE;
4142 val = build_int_cst (unsigned_type_node, 0);
4143 flags = build_int_cst (unsigned_type_node, 0);
4145 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
4146 if (c)
4147 cond = OMP_CLAUSE_IF_EXPR (c);
4149 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS);
4150 if (c)
4152 val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
4153 clause_loc = OMP_CLAUSE_LOCATION (c);
4155 else
4156 clause_loc = gimple_location (entry_stmt);
4158 c = find_omp_clause (clauses, OMP_CLAUSE_PROC_BIND);
4159 if (c)
4160 flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c));
4162 /* Ensure 'val' is of the correct type. */
4163 val = fold_convert_loc (clause_loc, unsigned_type_node, val);
4165 /* If we found the clause 'if (cond)', build either
4166 (cond != 0) or (cond ? val : 1u). */
4167 if (cond)
4169 gimple_stmt_iterator gsi;
4171 cond = gimple_boolify (cond);
4173 if (integer_zerop (val))
4174 val = fold_build2_loc (clause_loc,
4175 EQ_EXPR, unsigned_type_node, cond,
4176 build_int_cst (TREE_TYPE (cond), 0));
4177 else
4179 basic_block cond_bb, then_bb, else_bb;
4180 edge e, e_then, e_else;
4181 tree tmp_then, tmp_else, tmp_join, tmp_var;
4183 tmp_var = create_tmp_var (TREE_TYPE (val), NULL);
4184 if (gimple_in_ssa_p (cfun))
4186 tmp_then = make_ssa_name (tmp_var, NULL);
4187 tmp_else = make_ssa_name (tmp_var, NULL);
4188 tmp_join = make_ssa_name (tmp_var, NULL);
4190 else
4192 tmp_then = tmp_var;
4193 tmp_else = tmp_var;
4194 tmp_join = tmp_var;
4197 e = split_block (bb, NULL);
4198 cond_bb = e->src;
4199 bb = e->dest;
4200 remove_edge (e);
4202 then_bb = create_empty_bb (cond_bb);
4203 else_bb = create_empty_bb (then_bb);
4204 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
4205 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
4207 stmt = gimple_build_cond_empty (cond);
4208 gsi = gsi_start_bb (cond_bb);
4209 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4211 gsi = gsi_start_bb (then_bb);
4212 stmt = gimple_build_assign (tmp_then, val);
4213 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4215 gsi = gsi_start_bb (else_bb);
4216 stmt = gimple_build_assign
4217 (tmp_else, build_int_cst (unsigned_type_node, 1));
4218 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4220 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
4221 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
4222 if (current_loops)
4224 add_bb_to_loop (then_bb, cond_bb->loop_father);
4225 add_bb_to_loop (else_bb, cond_bb->loop_father);
4227 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
4228 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
4230 if (gimple_in_ssa_p (cfun))
4232 gimple phi = create_phi_node (tmp_join, bb);
4233 add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION);
4234 add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION);
4237 val = tmp_join;
4240 gsi = gsi_start_bb (bb);
4241 val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE,
4242 false, GSI_CONTINUE_LINKING);
4245 gsi = gsi_last_bb (bb);
4246 t = gimple_omp_parallel_data_arg (entry_stmt);
4247 if (t == NULL)
4248 t1 = null_pointer_node;
4249 else
4250 t1 = build_fold_addr_expr (t);
4251 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
4253 vec_alloc (args, 4 + vec_safe_length (ws_args));
4254 args->quick_push (t2);
4255 args->quick_push (t1);
4256 args->quick_push (val);
4257 if (ws_args)
4258 args->splice (*ws_args);
4259 args->quick_push (flags);
4261 t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
4262 builtin_decl_explicit (start_ix), args);
4264 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4265 false, GSI_CONTINUE_LINKING);
4269 /* Build the function call to GOMP_task to actually
4270 generate the task operation. BB is the block where to insert the code. */
4272 static void
4273 expand_task_call (basic_block bb, gimple entry_stmt)
4275 tree t, t1, t2, t3, flags, cond, c, c2, clauses, depend;
4276 gimple_stmt_iterator gsi;
4277 location_t loc = gimple_location (entry_stmt);
4279 clauses = gimple_omp_task_clauses (entry_stmt);
4281 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
4282 if (c)
4283 cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (c));
4284 else
4285 cond = boolean_true_node;
4287 c = find_omp_clause (clauses, OMP_CLAUSE_UNTIED);
4288 c2 = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE);
4289 depend = find_omp_clause (clauses, OMP_CLAUSE_DEPEND);
4290 flags = build_int_cst (unsigned_type_node,
4291 (c ? 1 : 0) + (c2 ? 4 : 0) + (depend ? 8 : 0));
4293 c = find_omp_clause (clauses, OMP_CLAUSE_FINAL);
4294 if (c)
4296 c = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c));
4297 c = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, c,
4298 build_int_cst (unsigned_type_node, 2),
4299 build_int_cst (unsigned_type_node, 0));
4300 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, c);
4302 if (depend)
4303 depend = OMP_CLAUSE_DECL (depend);
4304 else
4305 depend = build_int_cst (ptr_type_node, 0);
4307 gsi = gsi_last_bb (bb);
4308 t = gimple_omp_task_data_arg (entry_stmt);
4309 if (t == NULL)
4310 t2 = null_pointer_node;
4311 else
4312 t2 = build_fold_addr_expr_loc (loc, t);
4313 t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt));
4314 t = gimple_omp_task_copy_fn (entry_stmt);
4315 if (t == NULL)
4316 t3 = null_pointer_node;
4317 else
4318 t3 = build_fold_addr_expr_loc (loc, t);
4320 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
4321 8, t1, t2, t3,
4322 gimple_omp_task_arg_size (entry_stmt),
4323 gimple_omp_task_arg_align (entry_stmt), cond, flags,
4324 depend);
4326 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
4327 false, GSI_CONTINUE_LINKING);
4331 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4332 catch handler and return it. This prevents programs from violating the
4333 structured block semantics with throws. */
4335 static gimple_seq
4336 maybe_catch_exception (gimple_seq body)
4338 gimple g;
4339 tree decl;
4341 if (!flag_exceptions)
4342 return body;
4344 if (lang_hooks.eh_protect_cleanup_actions != NULL)
4345 decl = lang_hooks.eh_protect_cleanup_actions ();
4346 else
4347 decl = builtin_decl_explicit (BUILT_IN_TRAP);
4349 g = gimple_build_eh_must_not_throw (decl);
4350 g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
4351 GIMPLE_TRY_CATCH);
4353 return gimple_seq_alloc_with_stmt (g);
4356 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4358 static tree
4359 vec2chain (vec<tree, va_gc> *v)
4361 tree chain = NULL_TREE, t;
4362 unsigned ix;
4364 FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t)
4366 DECL_CHAIN (t) = chain;
4367 chain = t;
4370 return chain;
4374 /* Remove barriers in REGION->EXIT's block. Note that this is only
4375 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4376 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4377 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4378 removed. */
4380 static void
4381 remove_exit_barrier (struct omp_region *region)
4383 gimple_stmt_iterator gsi;
4384 basic_block exit_bb;
4385 edge_iterator ei;
4386 edge e;
4387 gimple stmt;
4388 int any_addressable_vars = -1;
4390 exit_bb = region->exit;
4392 /* If the parallel region doesn't return, we don't have REGION->EXIT
4393 block at all. */
4394 if (! exit_bb)
4395 return;
4397 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4398 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4399 statements that can appear in between are extremely limited -- no
4400 memory operations at all. Here, we allow nothing at all, so the
4401 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4402 gsi = gsi_last_bb (exit_bb);
4403 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
4404 gsi_prev (&gsi);
4405 if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
4406 return;
4408 FOR_EACH_EDGE (e, ei, exit_bb->preds)
4410 gsi = gsi_last_bb (e->src);
4411 if (gsi_end_p (gsi))
4412 continue;
4413 stmt = gsi_stmt (gsi);
4414 if (gimple_code (stmt) == GIMPLE_OMP_RETURN
4415 && !gimple_omp_return_nowait_p (stmt))
4417 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4418 in many cases. If there could be tasks queued, the barrier
4419 might be needed to let the tasks run before some local
4420 variable of the parallel that the task uses as shared
4421 runs out of scope. The task can be spawned either
4422 from within current function (this would be easy to check)
4423 or from some function it calls and gets passed an address
4424 of such a variable. */
4425 if (any_addressable_vars < 0)
4427 gimple parallel_stmt = last_stmt (region->entry);
4428 tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt);
4429 tree local_decls, block, decl;
4430 unsigned ix;
4432 any_addressable_vars = 0;
4433 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl)
4434 if (TREE_ADDRESSABLE (decl))
4436 any_addressable_vars = 1;
4437 break;
4439 for (block = gimple_block (stmt);
4440 !any_addressable_vars
4441 && block
4442 && TREE_CODE (block) == BLOCK;
4443 block = BLOCK_SUPERCONTEXT (block))
4445 for (local_decls = BLOCK_VARS (block);
4446 local_decls;
4447 local_decls = DECL_CHAIN (local_decls))
4448 if (TREE_ADDRESSABLE (local_decls))
4450 any_addressable_vars = 1;
4451 break;
4453 if (block == gimple_block (parallel_stmt))
4454 break;
4457 if (!any_addressable_vars)
4458 gimple_omp_return_set_nowait (stmt);
4463 static void
4464 remove_exit_barriers (struct omp_region *region)
4466 if (region->type == GIMPLE_OMP_PARALLEL)
4467 remove_exit_barrier (region);
4469 if (region->inner)
4471 region = region->inner;
4472 remove_exit_barriers (region);
4473 while (region->next)
4475 region = region->next;
4476 remove_exit_barriers (region);
4481 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4482 calls. These can't be declared as const functions, but
4483 within one parallel body they are constant, so they can be
4484 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4485 which are declared const. Similarly for task body, except
4486 that in untied task omp_get_thread_num () can change at any task
4487 scheduling point. */
4489 static void
4490 optimize_omp_library_calls (gimple entry_stmt)
4492 basic_block bb;
4493 gimple_stmt_iterator gsi;
4494 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
4495 tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree);
4496 tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
4497 tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree);
4498 bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
4499 && find_omp_clause (gimple_omp_task_clauses (entry_stmt),
4500 OMP_CLAUSE_UNTIED) != NULL);
4502 FOR_EACH_BB (bb)
4503 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4505 gimple call = gsi_stmt (gsi);
4506 tree decl;
4508 if (is_gimple_call (call)
4509 && (decl = gimple_call_fndecl (call))
4510 && DECL_EXTERNAL (decl)
4511 && TREE_PUBLIC (decl)
4512 && DECL_INITIAL (decl) == NULL)
4514 tree built_in;
4516 if (DECL_NAME (decl) == thr_num_id)
4518 /* In #pragma omp task untied omp_get_thread_num () can change
4519 during the execution of the task region. */
4520 if (untied_task)
4521 continue;
4522 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
4524 else if (DECL_NAME (decl) == num_thr_id)
4525 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
4526 else
4527 continue;
4529 if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in)
4530 || gimple_call_num_args (call) != 0)
4531 continue;
4533 if (flag_exceptions && !TREE_NOTHROW (decl))
4534 continue;
4536 if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE
4537 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)),
4538 TREE_TYPE (TREE_TYPE (built_in))))
4539 continue;
4541 gimple_call_set_fndecl (call, built_in);
4546 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4547 regimplified. */
4549 static tree
4550 expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *)
4552 tree t = *tp;
4554 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4555 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t))
4556 return t;
4558 if (TREE_CODE (t) == ADDR_EXPR)
4559 recompute_tree_invariant_for_addr_expr (t);
4561 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
4562 return NULL_TREE;
4565 /* Prepend TO = FROM assignment before *GSI_P. */
4567 static void
4568 expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from)
4570 bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to);
4571 from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE,
4572 true, GSI_SAME_STMT);
4573 gimple stmt = gimple_build_assign (to, from);
4574 gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT);
4575 if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL)
4576 || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL))
4578 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
4579 gimple_regimplify_operands (stmt, &gsi);
4583 /* Expand the OpenMP parallel or task directive starting at REGION. */
4585 static void
4586 expand_omp_taskreg (struct omp_region *region)
4588 basic_block entry_bb, exit_bb, new_bb;
4589 struct function *child_cfun;
4590 tree child_fn, block, t;
4591 gimple_stmt_iterator gsi;
4592 gimple entry_stmt, stmt;
4593 edge e;
4594 vec<tree, va_gc> *ws_args;
4596 entry_stmt = last_stmt (region->entry);
4597 child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
4598 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
4600 entry_bb = region->entry;
4601 exit_bb = region->exit;
4603 if (is_combined_parallel (region))
4604 ws_args = region->ws_args;
4605 else
4606 ws_args = NULL;
4608 if (child_cfun->cfg)
4610 /* Due to inlining, it may happen that we have already outlined
4611 the region, in which case all we need to do is make the
4612 sub-graph unreachable and emit the parallel call. */
4613 edge entry_succ_e, exit_succ_e;
4614 gimple_stmt_iterator gsi;
4616 entry_succ_e = single_succ_edge (entry_bb);
4618 gsi = gsi_last_bb (entry_bb);
4619 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
4620 || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
4621 gsi_remove (&gsi, true);
4623 new_bb = entry_bb;
4624 if (exit_bb)
4626 exit_succ_e = single_succ_edge (exit_bb);
4627 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
4629 remove_edge_and_dominated_blocks (entry_succ_e);
4631 else
4633 unsigned srcidx, dstidx, num;
4635 /* If the parallel region needs data sent from the parent
4636 function, then the very first statement (except possible
4637 tree profile counter updates) of the parallel body
4638 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4639 &.OMP_DATA_O is passed as an argument to the child function,
4640 we need to replace it with the argument as seen by the child
4641 function.
4643 In most cases, this will end up being the identity assignment
4644 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4645 a function call that has been inlined, the original PARM_DECL
4646 .OMP_DATA_I may have been converted into a different local
4647 variable. In which case, we need to keep the assignment. */
4648 if (gimple_omp_taskreg_data_arg (entry_stmt))
4650 basic_block entry_succ_bb = single_succ (entry_bb);
4651 gimple_stmt_iterator gsi;
4652 tree arg, narg;
4653 gimple parcopy_stmt = NULL;
4655 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
4657 gimple stmt;
4659 gcc_assert (!gsi_end_p (gsi));
4660 stmt = gsi_stmt (gsi);
4661 if (gimple_code (stmt) != GIMPLE_ASSIGN)
4662 continue;
4664 if (gimple_num_ops (stmt) == 2)
4666 tree arg = gimple_assign_rhs1 (stmt);
4668 /* We're ignore the subcode because we're
4669 effectively doing a STRIP_NOPS. */
4671 if (TREE_CODE (arg) == ADDR_EXPR
4672 && TREE_OPERAND (arg, 0)
4673 == gimple_omp_taskreg_data_arg (entry_stmt))
4675 parcopy_stmt = stmt;
4676 break;
4681 gcc_assert (parcopy_stmt != NULL);
4682 arg = DECL_ARGUMENTS (child_fn);
4684 if (!gimple_in_ssa_p (cfun))
4686 if (gimple_assign_lhs (parcopy_stmt) == arg)
4687 gsi_remove (&gsi, true);
4688 else
4690 /* ?? Is setting the subcode really necessary ?? */
4691 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg));
4692 gimple_assign_set_rhs1 (parcopy_stmt, arg);
4695 else
4697 /* If we are in ssa form, we must load the value from the default
4698 definition of the argument. That should not be defined now,
4699 since the argument is not used uninitialized. */
4700 gcc_assert (ssa_default_def (cfun, arg) == NULL);
4701 narg = make_ssa_name (arg, gimple_build_nop ());
4702 set_ssa_default_def (cfun, arg, narg);
4703 /* ?? Is setting the subcode really necessary ?? */
4704 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (narg));
4705 gimple_assign_set_rhs1 (parcopy_stmt, narg);
4706 update_stmt (parcopy_stmt);
4710 /* Declare local variables needed in CHILD_CFUN. */
4711 block = DECL_INITIAL (child_fn);
4712 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
4713 /* The gimplifier could record temporaries in parallel/task block
4714 rather than in containing function's local_decls chain,
4715 which would mean cgraph missed finalizing them. Do it now. */
4716 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
4717 if (TREE_CODE (t) == VAR_DECL
4718 && TREE_STATIC (t)
4719 && !DECL_EXTERNAL (t))
4720 varpool_finalize_decl (t);
4721 DECL_SAVED_TREE (child_fn) = NULL;
4722 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4723 gimple_set_body (child_fn, NULL);
4724 TREE_USED (block) = 1;
4726 /* Reset DECL_CONTEXT on function arguments. */
4727 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
4728 DECL_CONTEXT (t) = child_fn;
4730 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4731 so that it can be moved to the child function. */
4732 gsi = gsi_last_bb (entry_bb);
4733 stmt = gsi_stmt (gsi);
4734 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
4735 || gimple_code (stmt) == GIMPLE_OMP_TASK));
4736 gsi_remove (&gsi, true);
4737 e = split_block (entry_bb, stmt);
4738 entry_bb = e->dest;
4739 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
4741 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4742 if (exit_bb)
4744 gsi = gsi_last_bb (exit_bb);
4745 gcc_assert (!gsi_end_p (gsi)
4746 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
4747 stmt = gimple_build_return (NULL);
4748 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
4749 gsi_remove (&gsi, true);
4752 /* Move the parallel region into CHILD_CFUN. */
4754 if (gimple_in_ssa_p (cfun))
4756 init_tree_ssa (child_cfun);
4757 init_ssa_operands (child_cfun);
4758 child_cfun->gimple_df->in_ssa_p = true;
4759 block = NULL_TREE;
4761 else
4762 block = gimple_block (entry_stmt);
4764 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
4765 if (exit_bb)
4766 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
4767 /* When the OMP expansion process cannot guarantee an up-to-date
4768 loop tree arrange for the child function to fixup loops. */
4769 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
4770 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
4772 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4773 num = vec_safe_length (child_cfun->local_decls);
4774 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
4776 t = (*child_cfun->local_decls)[srcidx];
4777 if (DECL_CONTEXT (t) == cfun->decl)
4778 continue;
4779 if (srcidx != dstidx)
4780 (*child_cfun->local_decls)[dstidx] = t;
4781 dstidx++;
4783 if (dstidx != num)
4784 vec_safe_truncate (child_cfun->local_decls, dstidx);
4786 /* Inform the callgraph about the new function. */
4787 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
4788 cgraph_add_new_function (child_fn, true);
4790 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4791 fixed in a following pass. */
4792 push_cfun (child_cfun);
4793 if (optimize)
4794 optimize_omp_library_calls (entry_stmt);
4795 rebuild_cgraph_edges ();
4797 /* Some EH regions might become dead, see PR34608. If
4798 pass_cleanup_cfg isn't the first pass to happen with the
4799 new child, these dead EH edges might cause problems.
4800 Clean them up now. */
4801 if (flag_exceptions)
4803 basic_block bb;
4804 bool changed = false;
4806 FOR_EACH_BB (bb)
4807 changed |= gimple_purge_dead_eh_edges (bb);
4808 if (changed)
4809 cleanup_tree_cfg ();
4811 if (gimple_in_ssa_p (cfun))
4812 update_ssa (TODO_update_ssa);
4813 pop_cfun ();
4816 /* Emit a library call to launch the children threads. */
4817 if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
4818 expand_parallel_call (region, new_bb, entry_stmt, ws_args);
4819 else
4820 expand_task_call (new_bb, entry_stmt);
4821 if (gimple_in_ssa_p (cfun))
4822 update_ssa (TODO_update_ssa_only_virtuals);
4826 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4827 of the combined collapse > 1 loop constructs, generate code like:
4828 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4829 if (cond3 is <)
4830 adj = STEP3 - 1;
4831 else
4832 adj = STEP3 + 1;
4833 count3 = (adj + N32 - N31) / STEP3;
4834 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4835 if (cond2 is <)
4836 adj = STEP2 - 1;
4837 else
4838 adj = STEP2 + 1;
4839 count2 = (adj + N22 - N21) / STEP2;
4840 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4841 if (cond1 is <)
4842 adj = STEP1 - 1;
4843 else
4844 adj = STEP1 + 1;
4845 count1 = (adj + N12 - N11) / STEP1;
4846 count = count1 * count2 * count3;
4847 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4848 count = 0;
4849 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4850 of the combined loop constructs, just initialize COUNTS array
4851 from the _looptemp_ clauses. */
4853 /* NOTE: It *could* be better to moosh all of the BBs together,
4854 creating one larger BB with all the computation and the unexpected
4855 jump at the end. I.e.
4857 bool zero3, zero2, zero1, zero;
4859 zero3 = N32 c3 N31;
4860 count3 = (N32 - N31) /[cl] STEP3;
4861 zero2 = N22 c2 N21;
4862 count2 = (N22 - N21) /[cl] STEP2;
4863 zero1 = N12 c1 N11;
4864 count1 = (N12 - N11) /[cl] STEP1;
4865 zero = zero3 || zero2 || zero1;
4866 count = count1 * count2 * count3;
4867 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4869 After all, we expect the zero=false, and thus we expect to have to
4870 evaluate all of the comparison expressions, so short-circuiting
4871 oughtn't be a win. Since the condition isn't protecting a
4872 denominator, we're not concerned about divide-by-zero, so we can
4873 fully evaluate count even if a numerator turned out to be wrong.
4875 It seems like putting this all together would create much better
4876 scheduling opportunities, and less pressure on the chip's branch
4877 predictor. */
4879 static void
4880 expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
4881 basic_block &entry_bb, tree *counts,
4882 basic_block &zero_iter_bb, int &first_zero_iter,
4883 basic_block &l2_dom_bb)
4885 tree t, type = TREE_TYPE (fd->loop.v);
4886 gimple stmt;
4887 edge e, ne;
4888 int i;
4890 /* Collapsed loops need work for expansion into SSA form. */
4891 gcc_assert (!gimple_in_ssa_p (cfun));
4893 if (gimple_omp_for_combined_into_p (fd->for_stmt)
4894 && TREE_CODE (fd->loop.n2) != INTEGER_CST)
4896 /* First two _looptemp_ clauses are for istart/iend, counts[0]
4897 isn't supposed to be handled, as the inner loop doesn't
4898 use it. */
4899 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
4900 OMP_CLAUSE__LOOPTEMP_);
4901 gcc_assert (innerc);
4902 for (i = 0; i < fd->collapse; i++)
4904 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
4905 OMP_CLAUSE__LOOPTEMP_);
4906 gcc_assert (innerc);
4907 if (i)
4908 counts[i] = OMP_CLAUSE_DECL (innerc);
4909 else
4910 counts[0] = NULL_TREE;
4912 return;
4915 for (i = 0; i < fd->collapse; i++)
4917 tree itype = TREE_TYPE (fd->loops[i].v);
4919 if (SSA_VAR_P (fd->loop.n2)
4920 && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
4921 fold_convert (itype, fd->loops[i].n1),
4922 fold_convert (itype, fd->loops[i].n2)))
4923 == NULL_TREE || !integer_onep (t)))
4925 tree n1, n2;
4926 n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1));
4927 n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE,
4928 true, GSI_SAME_STMT);
4929 n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
4930 n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE,
4931 true, GSI_SAME_STMT);
4932 stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
4933 NULL_TREE, NULL_TREE);
4934 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
4935 if (walk_tree (gimple_cond_lhs_ptr (stmt),
4936 expand_omp_regimplify_p, NULL, NULL)
4937 || walk_tree (gimple_cond_rhs_ptr (stmt),
4938 expand_omp_regimplify_p, NULL, NULL))
4940 *gsi = gsi_for_stmt (stmt);
4941 gimple_regimplify_operands (stmt, gsi);
4943 e = split_block (entry_bb, stmt);
4944 if (zero_iter_bb == NULL)
4946 first_zero_iter = i;
4947 zero_iter_bb = create_empty_bb (entry_bb);
4948 if (current_loops)
4949 add_bb_to_loop (zero_iter_bb, entry_bb->loop_father);
4950 *gsi = gsi_after_labels (zero_iter_bb);
4951 stmt = gimple_build_assign (fd->loop.n2,
4952 build_zero_cst (type));
4953 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
4954 set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb,
4955 entry_bb);
4957 ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
4958 ne->probability = REG_BR_PROB_BASE / 2000 - 1;
4959 e->flags = EDGE_TRUE_VALUE;
4960 e->probability = REG_BR_PROB_BASE - ne->probability;
4961 if (l2_dom_bb == NULL)
4962 l2_dom_bb = entry_bb;
4963 entry_bb = e->dest;
4964 *gsi = gsi_last_bb (entry_bb);
4967 if (POINTER_TYPE_P (itype))
4968 itype = signed_type_for (itype);
4969 t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
4970 ? -1 : 1));
4971 t = fold_build2 (PLUS_EXPR, itype,
4972 fold_convert (itype, fd->loops[i].step), t);
4973 t = fold_build2 (PLUS_EXPR, itype, t,
4974 fold_convert (itype, fd->loops[i].n2));
4975 t = fold_build2 (MINUS_EXPR, itype, t,
4976 fold_convert (itype, fd->loops[i].n1));
4977 /* ?? We could probably use CEIL_DIV_EXPR instead of
4978 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
4979 generate the same code in the end because generically we
4980 don't know that the values involved must be negative for
4981 GT?? */
4982 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
4983 t = fold_build2 (TRUNC_DIV_EXPR, itype,
4984 fold_build1 (NEGATE_EXPR, itype, t),
4985 fold_build1 (NEGATE_EXPR, itype,
4986 fold_convert (itype,
4987 fd->loops[i].step)));
4988 else
4989 t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
4990 fold_convert (itype, fd->loops[i].step));
4991 t = fold_convert (type, t);
4992 if (TREE_CODE (t) == INTEGER_CST)
4993 counts[i] = t;
4994 else
4996 counts[i] = create_tmp_reg (type, ".count");
4997 expand_omp_build_assign (gsi, counts[i], t);
4999 if (SSA_VAR_P (fd->loop.n2))
5001 if (i == 0)
5002 t = counts[0];
5003 else
5004 t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]);
5005 expand_omp_build_assign (gsi, fd->loop.n2, t);
5011 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5012 T = V;
5013 V3 = N31 + (T % count3) * STEP3;
5014 T = T / count3;
5015 V2 = N21 + (T % count2) * STEP2;
5016 T = T / count2;
5017 V1 = N11 + T * STEP1;
5018 if this loop doesn't have an inner loop construct combined with it.
5019 If it does have an inner loop construct combined with it and the
5020 iteration count isn't known constant, store values from counts array
5021 into its _looptemp_ temporaries instead. */
5023 static void
5024 expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
5025 tree *counts, gimple inner_stmt, tree startvar)
5027 int i;
5028 if (gimple_omp_for_combined_p (fd->for_stmt))
5030 /* If fd->loop.n2 is constant, then no propagation of the counts
5031 is needed, they are constant. */
5032 if (TREE_CODE (fd->loop.n2) == INTEGER_CST)
5033 return;
5035 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
5036 ? gimple_omp_parallel_clauses (inner_stmt)
5037 : gimple_omp_for_clauses (inner_stmt);
5038 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5039 isn't supposed to be handled, as the inner loop doesn't
5040 use it. */
5041 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
5042 gcc_assert (innerc);
5043 for (i = 0; i < fd->collapse; i++)
5045 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5046 OMP_CLAUSE__LOOPTEMP_);
5047 gcc_assert (innerc);
5048 if (i)
5050 tree tem = OMP_CLAUSE_DECL (innerc);
5051 tree t = fold_convert (TREE_TYPE (tem), counts[i]);
5052 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
5053 false, GSI_CONTINUE_LINKING);
5054 gimple stmt = gimple_build_assign (tem, t);
5055 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5058 return;
5061 tree type = TREE_TYPE (fd->loop.v);
5062 tree tem = create_tmp_reg (type, ".tem");
5063 gimple stmt = gimple_build_assign (tem, startvar);
5064 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5066 for (i = fd->collapse - 1; i >= 0; i--)
5068 tree vtype = TREE_TYPE (fd->loops[i].v), itype, t;
5069 itype = vtype;
5070 if (POINTER_TYPE_P (vtype))
5071 itype = signed_type_for (vtype);
5072 if (i != 0)
5073 t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]);
5074 else
5075 t = tem;
5076 t = fold_convert (itype, t);
5077 t = fold_build2 (MULT_EXPR, itype, t,
5078 fold_convert (itype, fd->loops[i].step));
5079 if (POINTER_TYPE_P (vtype))
5080 t = fold_build_pointer_plus (fd->loops[i].n1, t);
5081 else
5082 t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
5083 t = force_gimple_operand_gsi (gsi, t,
5084 DECL_P (fd->loops[i].v)
5085 && TREE_ADDRESSABLE (fd->loops[i].v),
5086 NULL_TREE, false,
5087 GSI_CONTINUE_LINKING);
5088 stmt = gimple_build_assign (fd->loops[i].v, t);
5089 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5090 if (i != 0)
5092 t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]);
5093 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
5094 false, GSI_CONTINUE_LINKING);
5095 stmt = gimple_build_assign (tem, t);
5096 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
5102 /* Helper function for expand_omp_for_*. Generate code like:
5103 L10:
5104 V3 += STEP3;
5105 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5106 L11:
5107 V3 = N31;
5108 V2 += STEP2;
5109 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5110 L12:
5111 V2 = N21;
5112 V1 += STEP1;
5113 goto BODY_BB; */
5115 static basic_block
5116 extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb,
5117 basic_block body_bb)
5119 basic_block last_bb, bb, collapse_bb = NULL;
5120 int i;
5121 gimple_stmt_iterator gsi;
5122 edge e;
5123 tree t;
5124 gimple stmt;
5126 last_bb = cont_bb;
5127 for (i = fd->collapse - 1; i >= 0; i--)
5129 tree vtype = TREE_TYPE (fd->loops[i].v);
5131 bb = create_empty_bb (last_bb);
5132 if (current_loops)
5133 add_bb_to_loop (bb, last_bb->loop_father);
5134 gsi = gsi_start_bb (bb);
5136 if (i < fd->collapse - 1)
5138 e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
5139 e->probability = REG_BR_PROB_BASE / 8;
5141 t = fd->loops[i + 1].n1;
5142 t = force_gimple_operand_gsi (&gsi, t,
5143 DECL_P (fd->loops[i + 1].v)
5144 && TREE_ADDRESSABLE (fd->loops[i
5145 + 1].v),
5146 NULL_TREE, false,
5147 GSI_CONTINUE_LINKING);
5148 stmt = gimple_build_assign (fd->loops[i + 1].v, t);
5149 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5151 else
5152 collapse_bb = bb;
5154 set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
5156 if (POINTER_TYPE_P (vtype))
5157 t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
5158 else
5159 t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step);
5160 t = force_gimple_operand_gsi (&gsi, t,
5161 DECL_P (fd->loops[i].v)
5162 && TREE_ADDRESSABLE (fd->loops[i].v),
5163 NULL_TREE, false, GSI_CONTINUE_LINKING);
5164 stmt = gimple_build_assign (fd->loops[i].v, t);
5165 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5167 if (i > 0)
5169 t = fd->loops[i].n2;
5170 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5171 false, GSI_CONTINUE_LINKING);
5172 tree v = fd->loops[i].v;
5173 if (DECL_P (v) && TREE_ADDRESSABLE (v))
5174 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
5175 false, GSI_CONTINUE_LINKING);
5176 t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
5177 stmt = gimple_build_cond_empty (t);
5178 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5179 e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
5180 e->probability = REG_BR_PROB_BASE * 7 / 8;
5182 else
5183 make_edge (bb, body_bb, EDGE_FALLTHRU);
5184 last_bb = bb;
5187 return collapse_bb;
5191 /* A subroutine of expand_omp_for. Generate code for a parallel
5192 loop with any schedule. Given parameters:
5194 for (V = N1; V cond N2; V += STEP) BODY;
5196 where COND is "<" or ">", we generate pseudocode
5198 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5199 if (more) goto L0; else goto L3;
5201 V = istart0;
5202 iend = iend0;
5204 BODY;
5205 V += STEP;
5206 if (V cond iend) goto L1; else goto L2;
5208 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5211 If this is a combined omp parallel loop, instead of the call to
5212 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5213 If this is gimple_omp_for_combined_p loop, then instead of assigning
5214 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5215 inner GIMPLE_OMP_FOR and V += STEP; and
5216 if (V cond iend) goto L1; else goto L2; are removed.
5218 For collapsed loops, given parameters:
5219 collapse(3)
5220 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5221 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5222 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5223 BODY;
5225 we generate pseudocode
5227 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5228 if (cond3 is <)
5229 adj = STEP3 - 1;
5230 else
5231 adj = STEP3 + 1;
5232 count3 = (adj + N32 - N31) / STEP3;
5233 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5234 if (cond2 is <)
5235 adj = STEP2 - 1;
5236 else
5237 adj = STEP2 + 1;
5238 count2 = (adj + N22 - N21) / STEP2;
5239 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5240 if (cond1 is <)
5241 adj = STEP1 - 1;
5242 else
5243 adj = STEP1 + 1;
5244 count1 = (adj + N12 - N11) / STEP1;
5245 count = count1 * count2 * count3;
5246 goto Z1;
5248 count = 0;
5250 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5251 if (more) goto L0; else goto L3;
5253 V = istart0;
5254 T = V;
5255 V3 = N31 + (T % count3) * STEP3;
5256 T = T / count3;
5257 V2 = N21 + (T % count2) * STEP2;
5258 T = T / count2;
5259 V1 = N11 + T * STEP1;
5260 iend = iend0;
5262 BODY;
5263 V += 1;
5264 if (V < iend) goto L10; else goto L2;
5265 L10:
5266 V3 += STEP3;
5267 if (V3 cond3 N32) goto L1; else goto L11;
5268 L11:
5269 V3 = N31;
5270 V2 += STEP2;
5271 if (V2 cond2 N22) goto L1; else goto L12;
5272 L12:
5273 V2 = N21;
5274 V1 += STEP1;
5275 goto L1;
5277 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5282 static void
5283 expand_omp_for_generic (struct omp_region *region,
5284 struct omp_for_data *fd,
5285 enum built_in_function start_fn,
5286 enum built_in_function next_fn,
5287 gimple inner_stmt)
5289 tree type, istart0, iend0, iend;
5290 tree t, vmain, vback, bias = NULL_TREE;
5291 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
5292 basic_block l2_bb = NULL, l3_bb = NULL;
5293 gimple_stmt_iterator gsi;
5294 gimple stmt;
5295 bool in_combined_parallel = is_combined_parallel (region);
5296 bool broken_loop = region->cont == NULL;
5297 edge e, ne;
5298 tree *counts = NULL;
5299 int i;
5301 gcc_assert (!broken_loop || !in_combined_parallel);
5302 gcc_assert (fd->iter_type == long_integer_type_node
5303 || !in_combined_parallel);
5305 type = TREE_TYPE (fd->loop.v);
5306 istart0 = create_tmp_var (fd->iter_type, ".istart0");
5307 iend0 = create_tmp_var (fd->iter_type, ".iend0");
5308 TREE_ADDRESSABLE (istart0) = 1;
5309 TREE_ADDRESSABLE (iend0) = 1;
5311 /* See if we need to bias by LLONG_MIN. */
5312 if (fd->iter_type == long_long_unsigned_type_node
5313 && TREE_CODE (type) == INTEGER_TYPE
5314 && !TYPE_UNSIGNED (type))
5316 tree n1, n2;
5318 if (fd->loop.cond_code == LT_EXPR)
5320 n1 = fd->loop.n1;
5321 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
5323 else
5325 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
5326 n2 = fd->loop.n1;
5328 if (TREE_CODE (n1) != INTEGER_CST
5329 || TREE_CODE (n2) != INTEGER_CST
5330 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
5331 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
5334 entry_bb = region->entry;
5335 cont_bb = region->cont;
5336 collapse_bb = NULL;
5337 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
5338 gcc_assert (broken_loop
5339 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
5340 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
5341 l1_bb = single_succ (l0_bb);
5342 if (!broken_loop)
5344 l2_bb = create_empty_bb (cont_bb);
5345 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb);
5346 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
5348 else
5349 l2_bb = NULL;
5350 l3_bb = BRANCH_EDGE (entry_bb)->dest;
5351 exit_bb = region->exit;
5353 gsi = gsi_last_bb (entry_bb);
5355 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5356 if (fd->collapse > 1)
5358 int first_zero_iter = -1;
5359 basic_block zero_iter_bb = NULL, l2_dom_bb = NULL;
5361 counts = XALLOCAVEC (tree, fd->collapse);
5362 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
5363 zero_iter_bb, first_zero_iter,
5364 l2_dom_bb);
5366 if (zero_iter_bb)
5368 /* Some counts[i] vars might be uninitialized if
5369 some loop has zero iterations. But the body shouldn't
5370 be executed in that case, so just avoid uninit warnings. */
5371 for (i = first_zero_iter; i < fd->collapse; i++)
5372 if (SSA_VAR_P (counts[i]))
5373 TREE_NO_WARNING (counts[i]) = 1;
5374 gsi_prev (&gsi);
5375 e = split_block (entry_bb, gsi_stmt (gsi));
5376 entry_bb = e->dest;
5377 make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU);
5378 gsi = gsi_last_bb (entry_bb);
5379 set_immediate_dominator (CDI_DOMINATORS, entry_bb,
5380 get_immediate_dominator (CDI_DOMINATORS,
5381 zero_iter_bb));
5384 if (in_combined_parallel)
5386 /* In a combined parallel loop, emit a call to
5387 GOMP_loop_foo_next. */
5388 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
5389 build_fold_addr_expr (istart0),
5390 build_fold_addr_expr (iend0));
5392 else
5394 tree t0, t1, t2, t3, t4;
5395 /* If this is not a combined parallel loop, emit a call to
5396 GOMP_loop_foo_start in ENTRY_BB. */
5397 t4 = build_fold_addr_expr (iend0);
5398 t3 = build_fold_addr_expr (istart0);
5399 t2 = fold_convert (fd->iter_type, fd->loop.step);
5400 t1 = fd->loop.n2;
5401 t0 = fd->loop.n1;
5402 if (gimple_omp_for_combined_into_p (fd->for_stmt))
5404 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
5405 OMP_CLAUSE__LOOPTEMP_);
5406 gcc_assert (innerc);
5407 t0 = OMP_CLAUSE_DECL (innerc);
5408 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5409 OMP_CLAUSE__LOOPTEMP_);
5410 gcc_assert (innerc);
5411 t1 = OMP_CLAUSE_DECL (innerc);
5413 if (POINTER_TYPE_P (TREE_TYPE (t0))
5414 && TYPE_PRECISION (TREE_TYPE (t0))
5415 != TYPE_PRECISION (fd->iter_type))
5417 /* Avoid casting pointers to integer of a different size. */
5418 tree itype = signed_type_for (type);
5419 t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
5420 t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
5422 else
5424 t1 = fold_convert (fd->iter_type, t1);
5425 t0 = fold_convert (fd->iter_type, t0);
5427 if (bias)
5429 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
5430 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
5432 if (fd->iter_type == long_integer_type_node)
5434 if (fd->chunk_size)
5436 t = fold_convert (fd->iter_type, fd->chunk_size);
5437 t = build_call_expr (builtin_decl_explicit (start_fn),
5438 6, t0, t1, t2, t, t3, t4);
5440 else
5441 t = build_call_expr (builtin_decl_explicit (start_fn),
5442 5, t0, t1, t2, t3, t4);
5444 else
5446 tree t5;
5447 tree c_bool_type;
5448 tree bfn_decl;
5450 /* The GOMP_loop_ull_*start functions have additional boolean
5451 argument, true for < loops and false for > loops.
5452 In Fortran, the C bool type can be different from
5453 boolean_type_node. */
5454 bfn_decl = builtin_decl_explicit (start_fn);
5455 c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl));
5456 t5 = build_int_cst (c_bool_type,
5457 fd->loop.cond_code == LT_EXPR ? 1 : 0);
5458 if (fd->chunk_size)
5460 tree bfn_decl = builtin_decl_explicit (start_fn);
5461 t = fold_convert (fd->iter_type, fd->chunk_size);
5462 t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
5464 else
5465 t = build_call_expr (builtin_decl_explicit (start_fn),
5466 6, t5, t0, t1, t2, t3, t4);
5469 if (TREE_TYPE (t) != boolean_type_node)
5470 t = fold_build2 (NE_EXPR, boolean_type_node,
5471 t, build_int_cst (TREE_TYPE (t), 0));
5472 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5473 true, GSI_SAME_STMT);
5474 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5476 /* Remove the GIMPLE_OMP_FOR statement. */
5477 gsi_remove (&gsi, true);
5479 /* Iteration setup for sequential loop goes in L0_BB. */
5480 tree startvar = fd->loop.v;
5481 tree endvar = NULL_TREE;
5483 if (gimple_omp_for_combined_p (fd->for_stmt))
5485 gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR
5486 && gimple_omp_for_kind (inner_stmt)
5487 == GF_OMP_FOR_KIND_SIMD);
5488 tree innerc = find_omp_clause (gimple_omp_for_clauses (inner_stmt),
5489 OMP_CLAUSE__LOOPTEMP_);
5490 gcc_assert (innerc);
5491 startvar = OMP_CLAUSE_DECL (innerc);
5492 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5493 OMP_CLAUSE__LOOPTEMP_);
5494 gcc_assert (innerc);
5495 endvar = OMP_CLAUSE_DECL (innerc);
5498 gsi = gsi_start_bb (l0_bb);
5499 t = istart0;
5500 if (bias)
5501 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
5502 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
5503 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
5504 t = fold_convert (TREE_TYPE (startvar), t);
5505 t = force_gimple_operand_gsi (&gsi, t,
5506 DECL_P (startvar)
5507 && TREE_ADDRESSABLE (startvar),
5508 NULL_TREE, false, GSI_CONTINUE_LINKING);
5509 stmt = gimple_build_assign (startvar, t);
5510 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5512 t = iend0;
5513 if (bias)
5514 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
5515 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
5516 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
5517 t = fold_convert (TREE_TYPE (startvar), t);
5518 iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5519 false, GSI_CONTINUE_LINKING);
5520 if (endvar)
5522 stmt = gimple_build_assign (endvar, iend);
5523 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5525 if (fd->collapse > 1)
5526 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
5528 if (!broken_loop)
5530 /* Code to control the increment and predicate for the sequential
5531 loop goes in the CONT_BB. */
5532 gsi = gsi_last_bb (cont_bb);
5533 stmt = gsi_stmt (gsi);
5534 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
5535 vmain = gimple_omp_continue_control_use (stmt);
5536 vback = gimple_omp_continue_control_def (stmt);
5538 if (!gimple_omp_for_combined_p (fd->for_stmt))
5540 if (POINTER_TYPE_P (type))
5541 t = fold_build_pointer_plus (vmain, fd->loop.step);
5542 else
5543 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
5544 t = force_gimple_operand_gsi (&gsi, t,
5545 DECL_P (vback)
5546 && TREE_ADDRESSABLE (vback),
5547 NULL_TREE, true, GSI_SAME_STMT);
5548 stmt = gimple_build_assign (vback, t);
5549 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5551 t = build2 (fd->loop.cond_code, boolean_type_node,
5552 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
5553 iend);
5554 stmt = gimple_build_cond_empty (t);
5555 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5558 /* Remove GIMPLE_OMP_CONTINUE. */
5559 gsi_remove (&gsi, true);
5561 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
5562 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, l1_bb);
5564 /* Emit code to get the next parallel iteration in L2_BB. */
5565 gsi = gsi_start_bb (l2_bb);
5567 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
5568 build_fold_addr_expr (istart0),
5569 build_fold_addr_expr (iend0));
5570 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5571 false, GSI_CONTINUE_LINKING);
5572 if (TREE_TYPE (t) != boolean_type_node)
5573 t = fold_build2 (NE_EXPR, boolean_type_node,
5574 t, build_int_cst (TREE_TYPE (t), 0));
5575 stmt = gimple_build_cond_empty (t);
5576 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5579 /* Add the loop cleanup function. */
5580 gsi = gsi_last_bb (exit_bb);
5581 if (gimple_omp_return_nowait_p (gsi_stmt (gsi)))
5582 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
5583 else if (gimple_omp_return_lhs (gsi_stmt (gsi)))
5584 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
5585 else
5586 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
5587 stmt = gimple_build_call (t, 0);
5588 if (gimple_omp_return_lhs (gsi_stmt (gsi)))
5589 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
5590 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
5591 gsi_remove (&gsi, true);
5593 /* Connect the new blocks. */
5594 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
5595 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
5597 if (!broken_loop)
5599 gimple_seq phis;
5601 e = find_edge (cont_bb, l3_bb);
5602 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
5604 phis = phi_nodes (l3_bb);
5605 for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi))
5607 gimple phi = gsi_stmt (gsi);
5608 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
5609 PHI_ARG_DEF_FROM_EDGE (phi, e));
5611 remove_edge (e);
5613 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
5614 if (current_loops)
5615 add_bb_to_loop (l2_bb, cont_bb->loop_father);
5616 e = find_edge (cont_bb, l1_bb);
5617 if (gimple_omp_for_combined_p (fd->for_stmt))
5619 remove_edge (e);
5620 e = NULL;
5622 else if (fd->collapse > 1)
5624 remove_edge (e);
5625 e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
5627 else
5628 e->flags = EDGE_TRUE_VALUE;
5629 if (e)
5631 e->probability = REG_BR_PROB_BASE * 7 / 8;
5632 find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8;
5634 else
5636 e = find_edge (cont_bb, l2_bb);
5637 e->flags = EDGE_FALLTHRU;
5639 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
5641 set_immediate_dominator (CDI_DOMINATORS, l2_bb,
5642 recompute_dominator (CDI_DOMINATORS, l2_bb));
5643 set_immediate_dominator (CDI_DOMINATORS, l3_bb,
5644 recompute_dominator (CDI_DOMINATORS, l3_bb));
5645 set_immediate_dominator (CDI_DOMINATORS, l0_bb,
5646 recompute_dominator (CDI_DOMINATORS, l0_bb));
5647 set_immediate_dominator (CDI_DOMINATORS, l1_bb,
5648 recompute_dominator (CDI_DOMINATORS, l1_bb));
5650 struct loop *outer_loop = alloc_loop ();
5651 outer_loop->header = l0_bb;
5652 outer_loop->latch = l2_bb;
5653 add_loop (outer_loop, l0_bb->loop_father);
5655 if (!gimple_omp_for_combined_p (fd->for_stmt))
5657 struct loop *loop = alloc_loop ();
5658 loop->header = l1_bb;
5659 /* The loop may have multiple latches. */
5660 add_loop (loop, outer_loop);
5666 /* A subroutine of expand_omp_for. Generate code for a parallel
5667 loop with static schedule and no specified chunk size. Given
5668 parameters:
5670 for (V = N1; V cond N2; V += STEP) BODY;
5672 where COND is "<" or ">", we generate pseudocode
5674 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5675 if (cond is <)
5676 adj = STEP - 1;
5677 else
5678 adj = STEP + 1;
5679 if ((__typeof (V)) -1 > 0 && cond is >)
5680 n = -(adj + N2 - N1) / -STEP;
5681 else
5682 n = (adj + N2 - N1) / STEP;
5683 q = n / nthreads;
5684 tt = n % nthreads;
5685 if (threadid < tt) goto L3; else goto L4;
5687 tt = 0;
5688 q = q + 1;
5690 s0 = q * threadid + tt;
5691 e0 = s0 + q;
5692 V = s0 * STEP + N1;
5693 if (s0 >= e0) goto L2; else goto L0;
5695 e = e0 * STEP + N1;
5697 BODY;
5698 V += STEP;
5699 if (V cond e) goto L1;
5703 static void
5704 expand_omp_for_static_nochunk (struct omp_region *region,
5705 struct omp_for_data *fd,
5706 gimple inner_stmt)
5708 tree n, q, s0, e0, e, t, tt, nthreads, threadid;
5709 tree type, itype, vmain, vback;
5710 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
5711 basic_block body_bb, cont_bb, collapse_bb = NULL;
5712 basic_block fin_bb;
5713 gimple_stmt_iterator gsi;
5714 gimple stmt;
5715 edge ep;
5716 enum built_in_function get_num_threads = BUILT_IN_OMP_GET_NUM_THREADS;
5717 enum built_in_function get_thread_num = BUILT_IN_OMP_GET_THREAD_NUM;
5718 bool broken_loop = region->cont == NULL;
5719 tree *counts = NULL;
5720 tree n1, n2, step;
5722 itype = type = TREE_TYPE (fd->loop.v);
5723 if (POINTER_TYPE_P (type))
5724 itype = signed_type_for (type);
5726 entry_bb = region->entry;
5727 cont_bb = region->cont;
5728 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
5729 fin_bb = BRANCH_EDGE (entry_bb)->dest;
5730 gcc_assert (broken_loop
5731 || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
5732 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
5733 body_bb = single_succ (seq_start_bb);
5734 if (!broken_loop)
5736 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
5737 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
5739 exit_bb = region->exit;
5741 /* Iteration space partitioning goes in ENTRY_BB. */
5742 gsi = gsi_last_bb (entry_bb);
5743 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5745 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
5747 get_num_threads = BUILT_IN_OMP_GET_NUM_TEAMS;
5748 get_thread_num = BUILT_IN_OMP_GET_TEAM_NUM;
5751 if (fd->collapse > 1)
5753 int first_zero_iter = -1;
5754 basic_block l2_dom_bb = NULL;
5756 counts = XALLOCAVEC (tree, fd->collapse);
5757 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
5758 fin_bb, first_zero_iter,
5759 l2_dom_bb);
5760 t = NULL_TREE;
5762 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
5763 t = integer_one_node;
5764 else
5765 t = fold_binary (fd->loop.cond_code, boolean_type_node,
5766 fold_convert (type, fd->loop.n1),
5767 fold_convert (type, fd->loop.n2));
5768 if (fd->collapse == 1
5769 && TYPE_UNSIGNED (type)
5770 && (t == NULL_TREE || !integer_onep (t)))
5772 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
5773 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
5774 true, GSI_SAME_STMT);
5775 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
5776 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
5777 true, GSI_SAME_STMT);
5778 stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
5779 NULL_TREE, NULL_TREE);
5780 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5781 if (walk_tree (gimple_cond_lhs_ptr (stmt),
5782 expand_omp_regimplify_p, NULL, NULL)
5783 || walk_tree (gimple_cond_rhs_ptr (stmt),
5784 expand_omp_regimplify_p, NULL, NULL))
5786 gsi = gsi_for_stmt (stmt);
5787 gimple_regimplify_operands (stmt, &gsi);
5789 ep = split_block (entry_bb, stmt);
5790 ep->flags = EDGE_TRUE_VALUE;
5791 entry_bb = ep->dest;
5792 ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
5793 ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
5794 ep->probability = REG_BR_PROB_BASE / 2000 - 1;
5795 if (gimple_in_ssa_p (cfun))
5797 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
5798 for (gsi = gsi_start_phis (fin_bb);
5799 !gsi_end_p (gsi); gsi_next (&gsi))
5801 gimple phi = gsi_stmt (gsi);
5802 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
5803 ep, UNKNOWN_LOCATION);
5806 gsi = gsi_last_bb (entry_bb);
5809 t = build_call_expr (builtin_decl_explicit (get_num_threads), 0);
5810 t = fold_convert (itype, t);
5811 nthreads = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5812 true, GSI_SAME_STMT);
5814 t = build_call_expr (builtin_decl_explicit (get_thread_num), 0);
5815 t = fold_convert (itype, t);
5816 threadid = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5817 true, GSI_SAME_STMT);
5819 n1 = fd->loop.n1;
5820 n2 = fd->loop.n2;
5821 step = fd->loop.step;
5822 if (gimple_omp_for_combined_into_p (fd->for_stmt))
5824 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
5825 OMP_CLAUSE__LOOPTEMP_);
5826 gcc_assert (innerc);
5827 n1 = OMP_CLAUSE_DECL (innerc);
5828 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5829 OMP_CLAUSE__LOOPTEMP_);
5830 gcc_assert (innerc);
5831 n2 = OMP_CLAUSE_DECL (innerc);
5833 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
5834 true, NULL_TREE, true, GSI_SAME_STMT);
5835 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
5836 true, NULL_TREE, true, GSI_SAME_STMT);
5837 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
5838 true, NULL_TREE, true, GSI_SAME_STMT);
5840 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
5841 t = fold_build2 (PLUS_EXPR, itype, step, t);
5842 t = fold_build2 (PLUS_EXPR, itype, t, n2);
5843 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
5844 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
5845 t = fold_build2 (TRUNC_DIV_EXPR, itype,
5846 fold_build1 (NEGATE_EXPR, itype, t),
5847 fold_build1 (NEGATE_EXPR, itype, step));
5848 else
5849 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
5850 t = fold_convert (itype, t);
5851 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5853 q = create_tmp_reg (itype, "q");
5854 t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads);
5855 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
5856 gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT);
5858 tt = create_tmp_reg (itype, "tt");
5859 t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads);
5860 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
5861 gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT);
5863 t = build2 (LT_EXPR, boolean_type_node, threadid, tt);
5864 stmt = gimple_build_cond_empty (t);
5865 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5867 second_bb = split_block (entry_bb, stmt)->dest;
5868 gsi = gsi_last_bb (second_bb);
5869 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5871 gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)),
5872 GSI_SAME_STMT);
5873 stmt = gimple_build_assign_with_ops (PLUS_EXPR, q, q,
5874 build_int_cst (itype, 1));
5875 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5877 third_bb = split_block (second_bb, stmt)->dest;
5878 gsi = gsi_last_bb (third_bb);
5879 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
5881 t = build2 (MULT_EXPR, itype, q, threadid);
5882 t = build2 (PLUS_EXPR, itype, t, tt);
5883 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5885 t = fold_build2 (PLUS_EXPR, itype, s0, q);
5886 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
5888 t = build2 (GE_EXPR, boolean_type_node, s0, e0);
5889 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5891 /* Remove the GIMPLE_OMP_FOR statement. */
5892 gsi_remove (&gsi, true);
5894 /* Setup code for sequential iteration goes in SEQ_START_BB. */
5895 gsi = gsi_start_bb (seq_start_bb);
5897 tree startvar = fd->loop.v;
5898 tree endvar = NULL_TREE;
5900 if (gimple_omp_for_combined_p (fd->for_stmt))
5902 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
5903 ? gimple_omp_parallel_clauses (inner_stmt)
5904 : gimple_omp_for_clauses (inner_stmt);
5905 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
5906 gcc_assert (innerc);
5907 startvar = OMP_CLAUSE_DECL (innerc);
5908 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
5909 OMP_CLAUSE__LOOPTEMP_);
5910 gcc_assert (innerc);
5911 endvar = OMP_CLAUSE_DECL (innerc);
5913 t = fold_convert (itype, s0);
5914 t = fold_build2 (MULT_EXPR, itype, t, step);
5915 if (POINTER_TYPE_P (type))
5916 t = fold_build_pointer_plus (n1, t);
5917 else
5918 t = fold_build2 (PLUS_EXPR, type, t, n1);
5919 t = fold_convert (TREE_TYPE (startvar), t);
5920 t = force_gimple_operand_gsi (&gsi, t,
5921 DECL_P (startvar)
5922 && TREE_ADDRESSABLE (startvar),
5923 NULL_TREE, false, GSI_CONTINUE_LINKING);
5924 stmt = gimple_build_assign (startvar, t);
5925 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5927 t = fold_convert (itype, e0);
5928 t = fold_build2 (MULT_EXPR, itype, t, step);
5929 if (POINTER_TYPE_P (type))
5930 t = fold_build_pointer_plus (n1, t);
5931 else
5932 t = fold_build2 (PLUS_EXPR, type, t, n1);
5933 t = fold_convert (TREE_TYPE (startvar), t);
5934 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5935 false, GSI_CONTINUE_LINKING);
5936 if (endvar)
5938 stmt = gimple_build_assign (endvar, e);
5939 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5941 if (fd->collapse > 1)
5942 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
5944 if (!broken_loop)
5946 /* The code controlling the sequential loop replaces the
5947 GIMPLE_OMP_CONTINUE. */
5948 gsi = gsi_last_bb (cont_bb);
5949 stmt = gsi_stmt (gsi);
5950 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
5951 vmain = gimple_omp_continue_control_use (stmt);
5952 vback = gimple_omp_continue_control_def (stmt);
5954 if (!gimple_omp_for_combined_p (fd->for_stmt))
5956 if (POINTER_TYPE_P (type))
5957 t = fold_build_pointer_plus (vmain, step);
5958 else
5959 t = fold_build2 (PLUS_EXPR, type, vmain, step);
5960 t = force_gimple_operand_gsi (&gsi, t,
5961 DECL_P (vback)
5962 && TREE_ADDRESSABLE (vback),
5963 NULL_TREE, true, GSI_SAME_STMT);
5964 stmt = gimple_build_assign (vback, t);
5965 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
5967 t = build2 (fd->loop.cond_code, boolean_type_node,
5968 DECL_P (vback) && TREE_ADDRESSABLE (vback)
5969 ? t : vback, e);
5970 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
5973 /* Remove the GIMPLE_OMP_CONTINUE statement. */
5974 gsi_remove (&gsi, true);
5976 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
5977 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
5980 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
5981 gsi = gsi_last_bb (exit_bb);
5982 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
5984 t = gimple_omp_return_lhs (gsi_stmt (gsi));
5985 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
5987 gsi_remove (&gsi, true);
5989 /* Connect all the blocks. */
5990 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
5991 ep->probability = REG_BR_PROB_BASE / 4 * 3;
5992 ep = find_edge (entry_bb, second_bb);
5993 ep->flags = EDGE_TRUE_VALUE;
5994 ep->probability = REG_BR_PROB_BASE / 4;
5995 find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
5996 find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
5998 if (!broken_loop)
6000 ep = find_edge (cont_bb, body_bb);
6001 if (gimple_omp_for_combined_p (fd->for_stmt))
6003 remove_edge (ep);
6004 ep = NULL;
6006 else if (fd->collapse > 1)
6008 remove_edge (ep);
6009 ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6011 else
6012 ep->flags = EDGE_TRUE_VALUE;
6013 find_edge (cont_bb, fin_bb)->flags
6014 = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
6017 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
6018 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
6019 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb);
6021 set_immediate_dominator (CDI_DOMINATORS, body_bb,
6022 recompute_dominator (CDI_DOMINATORS, body_bb));
6023 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
6024 recompute_dominator (CDI_DOMINATORS, fin_bb));
6026 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
6028 struct loop *loop = alloc_loop ();
6029 loop->header = body_bb;
6030 if (collapse_bb == NULL)
6031 loop->latch = cont_bb;
6032 add_loop (loop, body_bb->loop_father);
6037 /* A subroutine of expand_omp_for. Generate code for a parallel
6038 loop with static schedule and a specified chunk size. Given
6039 parameters:
6041 for (V = N1; V cond N2; V += STEP) BODY;
6043 where COND is "<" or ">", we generate pseudocode
6045 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6046 if (cond is <)
6047 adj = STEP - 1;
6048 else
6049 adj = STEP + 1;
6050 if ((__typeof (V)) -1 > 0 && cond is >)
6051 n = -(adj + N2 - N1) / -STEP;
6052 else
6053 n = (adj + N2 - N1) / STEP;
6054 trip = 0;
6055 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6056 here so that V is defined
6057 if the loop is not entered
6059 s0 = (trip * nthreads + threadid) * CHUNK;
6060 e0 = min(s0 + CHUNK, n);
6061 if (s0 < n) goto L1; else goto L4;
6063 V = s0 * STEP + N1;
6064 e = e0 * STEP + N1;
6066 BODY;
6067 V += STEP;
6068 if (V cond e) goto L2; else goto L3;
6070 trip += 1;
6071 goto L0;
6075 static void
6076 expand_omp_for_static_chunk (struct omp_region *region,
6077 struct omp_for_data *fd, gimple inner_stmt)
6079 tree n, s0, e0, e, t;
6080 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
6081 tree type, itype, v_main, v_back, v_extra;
6082 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
6083 basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb;
6084 gimple_stmt_iterator si;
6085 gimple stmt;
6086 edge se;
6087 enum built_in_function get_num_threads = BUILT_IN_OMP_GET_NUM_THREADS;
6088 enum built_in_function get_thread_num = BUILT_IN_OMP_GET_THREAD_NUM;
6089 bool broken_loop = region->cont == NULL;
6090 tree *counts = NULL;
6091 tree n1, n2, step;
6093 itype = type = TREE_TYPE (fd->loop.v);
6094 if (POINTER_TYPE_P (type))
6095 itype = signed_type_for (type);
6097 entry_bb = region->entry;
6098 se = split_block (entry_bb, last_stmt (entry_bb));
6099 entry_bb = se->src;
6100 iter_part_bb = se->dest;
6101 cont_bb = region->cont;
6102 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
6103 fin_bb = BRANCH_EDGE (iter_part_bb)->dest;
6104 gcc_assert (broken_loop
6105 || fin_bb == FALLTHRU_EDGE (cont_bb)->dest);
6106 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
6107 body_bb = single_succ (seq_start_bb);
6108 if (!broken_loop)
6110 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
6111 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6112 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
6114 exit_bb = region->exit;
6116 /* Trip and adjustment setup goes in ENTRY_BB. */
6117 si = gsi_last_bb (entry_bb);
6118 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_FOR);
6120 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
6122 get_num_threads = BUILT_IN_OMP_GET_NUM_TEAMS;
6123 get_thread_num = BUILT_IN_OMP_GET_TEAM_NUM;
6126 if (fd->collapse > 1)
6128 int first_zero_iter = -1;
6129 basic_block l2_dom_bb = NULL;
6131 counts = XALLOCAVEC (tree, fd->collapse);
6132 expand_omp_for_init_counts (fd, &si, entry_bb, counts,
6133 fin_bb, first_zero_iter,
6134 l2_dom_bb);
6135 t = NULL_TREE;
6137 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
6138 t = integer_one_node;
6139 else
6140 t = fold_binary (fd->loop.cond_code, boolean_type_node,
6141 fold_convert (type, fd->loop.n1),
6142 fold_convert (type, fd->loop.n2));
6143 if (fd->collapse == 1
6144 && TYPE_UNSIGNED (type)
6145 && (t == NULL_TREE || !integer_onep (t)))
6147 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
6148 n1 = force_gimple_operand_gsi (&si, n1, true, NULL_TREE,
6149 true, GSI_SAME_STMT);
6150 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
6151 n2 = force_gimple_operand_gsi (&si, n2, true, NULL_TREE,
6152 true, GSI_SAME_STMT);
6153 stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
6154 NULL_TREE, NULL_TREE);
6155 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
6156 if (walk_tree (gimple_cond_lhs_ptr (stmt),
6157 expand_omp_regimplify_p, NULL, NULL)
6158 || walk_tree (gimple_cond_rhs_ptr (stmt),
6159 expand_omp_regimplify_p, NULL, NULL))
6161 si = gsi_for_stmt (stmt);
6162 gimple_regimplify_operands (stmt, &si);
6164 se = split_block (entry_bb, stmt);
6165 se->flags = EDGE_TRUE_VALUE;
6166 entry_bb = se->dest;
6167 se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
6168 se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
6169 se->probability = REG_BR_PROB_BASE / 2000 - 1;
6170 if (gimple_in_ssa_p (cfun))
6172 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
6173 for (si = gsi_start_phis (fin_bb);
6174 !gsi_end_p (si); gsi_next (&si))
6176 gimple phi = gsi_stmt (si);
6177 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
6178 se, UNKNOWN_LOCATION);
6181 si = gsi_last_bb (entry_bb);
6184 t = build_call_expr (builtin_decl_explicit (get_num_threads), 0);
6185 t = fold_convert (itype, t);
6186 nthreads = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6187 true, GSI_SAME_STMT);
6189 t = build_call_expr (builtin_decl_explicit (get_thread_num), 0);
6190 t = fold_convert (itype, t);
6191 threadid = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6192 true, GSI_SAME_STMT);
6194 n1 = fd->loop.n1;
6195 n2 = fd->loop.n2;
6196 step = fd->loop.step;
6197 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6199 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6200 OMP_CLAUSE__LOOPTEMP_);
6201 gcc_assert (innerc);
6202 n1 = OMP_CLAUSE_DECL (innerc);
6203 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6204 OMP_CLAUSE__LOOPTEMP_);
6205 gcc_assert (innerc);
6206 n2 = OMP_CLAUSE_DECL (innerc);
6208 n1 = force_gimple_operand_gsi (&si, fold_convert (type, n1),
6209 true, NULL_TREE, true, GSI_SAME_STMT);
6210 n2 = force_gimple_operand_gsi (&si, fold_convert (itype, n2),
6211 true, NULL_TREE, true, GSI_SAME_STMT);
6212 step = force_gimple_operand_gsi (&si, fold_convert (itype, step),
6213 true, NULL_TREE, true, GSI_SAME_STMT);
6214 fd->chunk_size
6215 = force_gimple_operand_gsi (&si, fold_convert (itype, fd->chunk_size),
6216 true, NULL_TREE, true, GSI_SAME_STMT);
6218 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
6219 t = fold_build2 (PLUS_EXPR, itype, step, t);
6220 t = fold_build2 (PLUS_EXPR, itype, t, n2);
6221 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
6222 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
6223 t = fold_build2 (TRUNC_DIV_EXPR, itype,
6224 fold_build1 (NEGATE_EXPR, itype, t),
6225 fold_build1 (NEGATE_EXPR, itype, step));
6226 else
6227 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
6228 t = fold_convert (itype, t);
6229 n = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6230 true, GSI_SAME_STMT);
6232 trip_var = create_tmp_reg (itype, ".trip");
6233 if (gimple_in_ssa_p (cfun))
6235 trip_init = make_ssa_name (trip_var, NULL);
6236 trip_main = make_ssa_name (trip_var, NULL);
6237 trip_back = make_ssa_name (trip_var, NULL);
6239 else
6241 trip_init = trip_var;
6242 trip_main = trip_var;
6243 trip_back = trip_var;
6246 stmt = gimple_build_assign (trip_init, build_int_cst (itype, 0));
6247 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
6249 t = fold_build2 (MULT_EXPR, itype, threadid, fd->chunk_size);
6250 t = fold_build2 (MULT_EXPR, itype, t, step);
6251 if (POINTER_TYPE_P (type))
6252 t = fold_build_pointer_plus (n1, t);
6253 else
6254 t = fold_build2 (PLUS_EXPR, type, t, n1);
6255 v_extra = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6256 true, GSI_SAME_STMT);
6258 /* Remove the GIMPLE_OMP_FOR. */
6259 gsi_remove (&si, true);
6261 /* Iteration space partitioning goes in ITER_PART_BB. */
6262 si = gsi_last_bb (iter_part_bb);
6264 t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads);
6265 t = fold_build2 (PLUS_EXPR, itype, t, threadid);
6266 t = fold_build2 (MULT_EXPR, itype, t, fd->chunk_size);
6267 s0 = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6268 false, GSI_CONTINUE_LINKING);
6270 t = fold_build2 (PLUS_EXPR, itype, s0, fd->chunk_size);
6271 t = fold_build2 (MIN_EXPR, itype, t, n);
6272 e0 = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6273 false, GSI_CONTINUE_LINKING);
6275 t = build2 (LT_EXPR, boolean_type_node, s0, n);
6276 gsi_insert_after (&si, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING);
6278 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6279 si = gsi_start_bb (seq_start_bb);
6281 tree startvar = fd->loop.v;
6282 tree endvar = NULL_TREE;
6284 if (gimple_omp_for_combined_p (fd->for_stmt))
6286 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
6287 ? gimple_omp_parallel_clauses (inner_stmt)
6288 : gimple_omp_for_clauses (inner_stmt);
6289 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
6290 gcc_assert (innerc);
6291 startvar = OMP_CLAUSE_DECL (innerc);
6292 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6293 OMP_CLAUSE__LOOPTEMP_);
6294 gcc_assert (innerc);
6295 endvar = OMP_CLAUSE_DECL (innerc);
6298 t = fold_convert (itype, s0);
6299 t = fold_build2 (MULT_EXPR, itype, t, step);
6300 if (POINTER_TYPE_P (type))
6301 t = fold_build_pointer_plus (n1, t);
6302 else
6303 t = fold_build2 (PLUS_EXPR, type, t, n1);
6304 t = fold_convert (TREE_TYPE (startvar), t);
6305 t = force_gimple_operand_gsi (&si, t,
6306 DECL_P (startvar)
6307 && TREE_ADDRESSABLE (startvar),
6308 NULL_TREE, false, GSI_CONTINUE_LINKING);
6309 stmt = gimple_build_assign (startvar, t);
6310 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
6312 t = fold_convert (itype, e0);
6313 t = fold_build2 (MULT_EXPR, itype, t, step);
6314 if (POINTER_TYPE_P (type))
6315 t = fold_build_pointer_plus (n1, t);
6316 else
6317 t = fold_build2 (PLUS_EXPR, type, t, n1);
6318 t = fold_convert (TREE_TYPE (startvar), t);
6319 e = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6320 false, GSI_CONTINUE_LINKING);
6321 if (endvar)
6323 stmt = gimple_build_assign (endvar, e);
6324 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
6326 if (fd->collapse > 1)
6327 expand_omp_for_init_vars (fd, &si, counts, inner_stmt, startvar);
6329 if (!broken_loop)
6331 /* The code controlling the sequential loop goes in CONT_BB,
6332 replacing the GIMPLE_OMP_CONTINUE. */
6333 si = gsi_last_bb (cont_bb);
6334 stmt = gsi_stmt (si);
6335 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
6336 v_main = gimple_omp_continue_control_use (stmt);
6337 v_back = gimple_omp_continue_control_def (stmt);
6339 if (!gimple_omp_for_combined_p (fd->for_stmt))
6341 if (POINTER_TYPE_P (type))
6342 t = fold_build_pointer_plus (v_main, step);
6343 else
6344 t = fold_build2 (PLUS_EXPR, type, v_main, step);
6345 if (DECL_P (v_back) && TREE_ADDRESSABLE (v_back))
6346 t = force_gimple_operand_gsi (&si, t, true, NULL_TREE,
6347 true, GSI_SAME_STMT);
6348 stmt = gimple_build_assign (v_back, t);
6349 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
6351 t = build2 (fd->loop.cond_code, boolean_type_node,
6352 DECL_P (v_back) && TREE_ADDRESSABLE (v_back)
6353 ? t : v_back, e);
6354 gsi_insert_before (&si, gimple_build_cond_empty (t), GSI_SAME_STMT);
6357 /* Remove GIMPLE_OMP_CONTINUE. */
6358 gsi_remove (&si, true);
6360 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6361 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
6363 /* Trip update code goes into TRIP_UPDATE_BB. */
6364 si = gsi_start_bb (trip_update_bb);
6366 t = build_int_cst (itype, 1);
6367 t = build2 (PLUS_EXPR, itype, trip_main, t);
6368 stmt = gimple_build_assign (trip_back, t);
6369 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
6372 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6373 si = gsi_last_bb (exit_bb);
6374 if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
6376 t = gimple_omp_return_lhs (gsi_stmt (si));
6377 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT);
6379 gsi_remove (&si, true);
6381 /* Connect the new blocks. */
6382 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
6383 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
6385 if (!broken_loop)
6387 se = find_edge (cont_bb, body_bb);
6388 if (gimple_omp_for_combined_p (fd->for_stmt))
6390 remove_edge (se);
6391 se = NULL;
6393 else if (fd->collapse > 1)
6395 remove_edge (se);
6396 se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6398 else
6399 se->flags = EDGE_TRUE_VALUE;
6400 find_edge (cont_bb, trip_update_bb)->flags
6401 = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
6403 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb);
6406 if (gimple_in_ssa_p (cfun))
6408 gimple_stmt_iterator psi;
6409 gimple phi;
6410 edge re, ene;
6411 edge_var_map_vector *head;
6412 edge_var_map *vm;
6413 size_t i;
6415 gcc_assert (fd->collapse == 1 && !broken_loop);
6417 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6418 remove arguments of the phi nodes in fin_bb. We need to create
6419 appropriate phi nodes in iter_part_bb instead. */
6420 se = single_pred_edge (fin_bb);
6421 re = single_succ_edge (trip_update_bb);
6422 head = redirect_edge_var_map_vector (re);
6423 ene = single_succ_edge (entry_bb);
6425 psi = gsi_start_phis (fin_bb);
6426 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
6427 gsi_next (&psi), ++i)
6429 gimple nphi;
6430 source_location locus;
6432 phi = gsi_stmt (psi);
6433 t = gimple_phi_result (phi);
6434 gcc_assert (t == redirect_edge_var_map_result (vm));
6435 nphi = create_phi_node (t, iter_part_bb);
6437 t = PHI_ARG_DEF_FROM_EDGE (phi, se);
6438 locus = gimple_phi_arg_location_from_edge (phi, se);
6440 /* A special case -- fd->loop.v is not yet computed in
6441 iter_part_bb, we need to use v_extra instead. */
6442 if (t == fd->loop.v)
6443 t = v_extra;
6444 add_phi_arg (nphi, t, ene, locus);
6445 locus = redirect_edge_var_map_location (vm);
6446 add_phi_arg (nphi, redirect_edge_var_map_def (vm), re, locus);
6448 gcc_assert (!gsi_end_p (psi) && i == head->length ());
6449 redirect_edge_var_map_clear (re);
6450 while (1)
6452 psi = gsi_start_phis (fin_bb);
6453 if (gsi_end_p (psi))
6454 break;
6455 remove_phi_node (&psi, false);
6458 /* Make phi node for trip. */
6459 phi = create_phi_node (trip_main, iter_part_bb);
6460 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb),
6461 UNKNOWN_LOCATION);
6462 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb),
6463 UNKNOWN_LOCATION);
6466 if (!broken_loop)
6467 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
6468 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
6469 recompute_dominator (CDI_DOMINATORS, iter_part_bb));
6470 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
6471 recompute_dominator (CDI_DOMINATORS, fin_bb));
6472 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
6473 recompute_dominator (CDI_DOMINATORS, seq_start_bb));
6474 set_immediate_dominator (CDI_DOMINATORS, body_bb,
6475 recompute_dominator (CDI_DOMINATORS, body_bb));
6477 if (!broken_loop)
6479 struct loop *trip_loop = alloc_loop ();
6480 trip_loop->header = iter_part_bb;
6481 trip_loop->latch = trip_update_bb;
6482 add_loop (trip_loop, iter_part_bb->loop_father);
6484 if (!gimple_omp_for_combined_p (fd->for_stmt))
6486 struct loop *loop = alloc_loop ();
6487 loop->header = body_bb;
6488 loop->latch = cont_bb;
6489 add_loop (loop, trip_loop);
6495 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6496 loop. Given parameters:
6498 for (V = N1; V cond N2; V += STEP) BODY;
6500 where COND is "<" or ">", we generate pseudocode
6502 V = N1;
6503 goto L1;
6505 BODY;
6506 V += STEP;
6508 if (V cond N2) goto L0; else goto L2;
6511 For collapsed loops, given parameters:
6512 collapse(3)
6513 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6514 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6515 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6516 BODY;
6518 we generate pseudocode
6520 if (cond3 is <)
6521 adj = STEP3 - 1;
6522 else
6523 adj = STEP3 + 1;
6524 count3 = (adj + N32 - N31) / STEP3;
6525 if (cond2 is <)
6526 adj = STEP2 - 1;
6527 else
6528 adj = STEP2 + 1;
6529 count2 = (adj + N22 - N21) / STEP2;
6530 if (cond1 is <)
6531 adj = STEP1 - 1;
6532 else
6533 adj = STEP1 + 1;
6534 count1 = (adj + N12 - N11) / STEP1;
6535 count = count1 * count2 * count3;
6536 V = 0;
6537 V1 = N11;
6538 V2 = N21;
6539 V3 = N31;
6540 goto L1;
6542 BODY;
6543 V += 1;
6544 V3 += STEP3;
6545 V2 += (V3 cond3 N32) ? 0 : STEP2;
6546 V3 = (V3 cond3 N32) ? V3 : N31;
6547 V1 += (V2 cond2 N22) ? 0 : STEP1;
6548 V2 = (V2 cond2 N22) ? V2 : N21;
6550 if (V < count) goto L0; else goto L2;
6555 static void
6556 expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
6558 tree type, t;
6559 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb;
6560 gimple_stmt_iterator gsi;
6561 gimple stmt;
6562 bool broken_loop = region->cont == NULL;
6563 edge e, ne;
6564 tree *counts = NULL;
6565 int i;
6566 tree safelen = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6567 OMP_CLAUSE_SAFELEN);
6568 tree simduid = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6569 OMP_CLAUSE__SIMDUID_);
6570 tree n1, n2;
6572 type = TREE_TYPE (fd->loop.v);
6573 entry_bb = region->entry;
6574 cont_bb = region->cont;
6575 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6576 gcc_assert (broken_loop
6577 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
6578 l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
6579 if (!broken_loop)
6581 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
6582 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6583 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
6584 l2_bb = BRANCH_EDGE (entry_bb)->dest;
6586 else
6588 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
6589 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
6590 l2_bb = single_succ (l1_bb);
6592 exit_bb = region->exit;
6593 l2_dom_bb = NULL;
6595 gsi = gsi_last_bb (entry_bb);
6597 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6598 /* Not needed in SSA form right now. */
6599 gcc_assert (!gimple_in_ssa_p (cfun));
6600 if (fd->collapse > 1)
6602 int first_zero_iter = -1;
6603 basic_block zero_iter_bb = l2_bb;
6605 counts = XALLOCAVEC (tree, fd->collapse);
6606 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6607 zero_iter_bb, first_zero_iter,
6608 l2_dom_bb);
6610 if (l2_dom_bb == NULL)
6611 l2_dom_bb = l1_bb;
6613 n1 = fd->loop.n1;
6614 n2 = fd->loop.n2;
6615 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6617 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6618 OMP_CLAUSE__LOOPTEMP_);
6619 gcc_assert (innerc);
6620 n1 = OMP_CLAUSE_DECL (innerc);
6621 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6622 OMP_CLAUSE__LOOPTEMP_);
6623 gcc_assert (innerc);
6624 n2 = OMP_CLAUSE_DECL (innerc);
6625 expand_omp_build_assign (&gsi, fd->loop.v,
6626 fold_convert (type, n1));
6627 if (fd->collapse > 1)
6629 gsi_prev (&gsi);
6630 expand_omp_for_init_vars (fd, &gsi, counts, NULL, n1);
6631 gsi_next (&gsi);
6634 else
6636 expand_omp_build_assign (&gsi, fd->loop.v,
6637 fold_convert (type, fd->loop.n1));
6638 if (fd->collapse > 1)
6639 for (i = 0; i < fd->collapse; i++)
6641 tree itype = TREE_TYPE (fd->loops[i].v);
6642 if (POINTER_TYPE_P (itype))
6643 itype = signed_type_for (itype);
6644 t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1);
6645 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6649 /* Remove the GIMPLE_OMP_FOR statement. */
6650 gsi_remove (&gsi, true);
6652 if (!broken_loop)
6654 /* Code to control the increment goes in the CONT_BB. */
6655 gsi = gsi_last_bb (cont_bb);
6656 stmt = gsi_stmt (gsi);
6657 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
6659 if (POINTER_TYPE_P (type))
6660 t = fold_build_pointer_plus (fd->loop.v, fd->loop.step);
6661 else
6662 t = fold_build2 (PLUS_EXPR, type, fd->loop.v, fd->loop.step);
6663 expand_omp_build_assign (&gsi, fd->loop.v, t);
6665 if (fd->collapse > 1)
6667 i = fd->collapse - 1;
6668 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
6670 t = fold_convert (sizetype, fd->loops[i].step);
6671 t = fold_build_pointer_plus (fd->loops[i].v, t);
6673 else
6675 t = fold_convert (TREE_TYPE (fd->loops[i].v),
6676 fd->loops[i].step);
6677 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
6678 fd->loops[i].v, t);
6680 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6682 for (i = fd->collapse - 1; i > 0; i--)
6684 tree itype = TREE_TYPE (fd->loops[i].v);
6685 tree itype2 = TREE_TYPE (fd->loops[i - 1].v);
6686 if (POINTER_TYPE_P (itype2))
6687 itype2 = signed_type_for (itype2);
6688 t = build3 (COND_EXPR, itype2,
6689 build2 (fd->loops[i].cond_code, boolean_type_node,
6690 fd->loops[i].v,
6691 fold_convert (itype, fd->loops[i].n2)),
6692 build_int_cst (itype2, 0),
6693 fold_convert (itype2, fd->loops[i - 1].step));
6694 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i - 1].v)))
6695 t = fold_build_pointer_plus (fd->loops[i - 1].v, t);
6696 else
6697 t = fold_build2 (PLUS_EXPR, itype2, fd->loops[i - 1].v, t);
6698 expand_omp_build_assign (&gsi, fd->loops[i - 1].v, t);
6700 t = build3 (COND_EXPR, itype,
6701 build2 (fd->loops[i].cond_code, boolean_type_node,
6702 fd->loops[i].v,
6703 fold_convert (itype, fd->loops[i].n2)),
6704 fd->loops[i].v,
6705 fold_convert (itype, fd->loops[i].n1));
6706 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
6710 /* Remove GIMPLE_OMP_CONTINUE. */
6711 gsi_remove (&gsi, true);
6714 /* Emit the condition in L1_BB. */
6715 gsi = gsi_start_bb (l1_bb);
6717 t = fold_convert (type, n2);
6718 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6719 false, GSI_CONTINUE_LINKING);
6720 t = build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t);
6721 stmt = gimple_build_cond_empty (t);
6722 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6723 if (walk_tree (gimple_cond_lhs_ptr (stmt), expand_omp_regimplify_p,
6724 NULL, NULL)
6725 || walk_tree (gimple_cond_rhs_ptr (stmt), expand_omp_regimplify_p,
6726 NULL, NULL))
6728 gsi = gsi_for_stmt (stmt);
6729 gimple_regimplify_operands (stmt, &gsi);
6732 /* Remove GIMPLE_OMP_RETURN. */
6733 gsi = gsi_last_bb (exit_bb);
6734 gsi_remove (&gsi, true);
6736 /* Connect the new blocks. */
6737 remove_edge (FALLTHRU_EDGE (entry_bb));
6739 if (!broken_loop)
6741 remove_edge (BRANCH_EDGE (entry_bb));
6742 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
6744 e = BRANCH_EDGE (l1_bb);
6745 ne = FALLTHRU_EDGE (l1_bb);
6746 e->flags = EDGE_TRUE_VALUE;
6748 else
6750 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
6752 ne = single_succ_edge (l1_bb);
6753 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
6756 ne->flags = EDGE_FALSE_VALUE;
6757 e->probability = REG_BR_PROB_BASE * 7 / 8;
6758 ne->probability = REG_BR_PROB_BASE / 8;
6760 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
6761 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
6762 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
6764 if (!broken_loop)
6766 struct loop *loop = alloc_loop ();
6767 loop->header = l1_bb;
6768 loop->latch = e->dest;
6769 add_loop (loop, l1_bb->loop_father);
6770 if (safelen == NULL_TREE)
6771 loop->safelen = INT_MAX;
6772 else
6774 safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen);
6775 if (!tree_fits_uhwi_p (safelen)
6776 || tree_to_uhwi (safelen) > INT_MAX)
6777 loop->safelen = INT_MAX;
6778 else
6779 loop->safelen = tree_to_uhwi (safelen);
6780 if (loop->safelen == 1)
6781 loop->safelen = 0;
6783 if (simduid)
6785 loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
6786 cfun->has_simduid_loops = true;
6788 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6789 the loop. */
6790 if ((flag_tree_loop_vectorize
6791 || (!global_options_set.x_flag_tree_loop_vectorize
6792 && !global_options_set.x_flag_tree_vectorize))
6793 && loop->safelen > 1)
6795 loop->force_vect = true;
6796 cfun->has_force_vect_loops = true;
6802 /* Expand the OpenMP loop defined by REGION. */
6804 static void
6805 expand_omp_for (struct omp_region *region, gimple inner_stmt)
6807 struct omp_for_data fd;
6808 struct omp_for_data_loop *loops;
6810 loops
6811 = (struct omp_for_data_loop *)
6812 alloca (gimple_omp_for_collapse (last_stmt (region->entry))
6813 * sizeof (struct omp_for_data_loop));
6814 extract_omp_for_data (last_stmt (region->entry), &fd, loops);
6815 region->sched_kind = fd.sched_kind;
6817 gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
6818 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
6819 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
6820 if (region->cont)
6822 gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
6823 BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
6824 FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
6826 else
6827 /* If there isn't a continue then this is a degerate case where
6828 the introduction of abnormal edges during lowering will prevent
6829 original loops from being detected. Fix that up. */
6830 loops_state_set (LOOPS_NEED_FIXUP);
6832 if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_KIND_SIMD)
6833 expand_omp_simd (region, &fd);
6834 else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
6835 && !fd.have_ordered)
6837 if (fd.chunk_size == NULL)
6838 expand_omp_for_static_nochunk (region, &fd, inner_stmt);
6839 else
6840 expand_omp_for_static_chunk (region, &fd, inner_stmt);
6842 else
6844 int fn_index, start_ix, next_ix;
6846 gcc_assert (gimple_omp_for_kind (fd.for_stmt)
6847 == GF_OMP_FOR_KIND_FOR);
6848 if (fd.chunk_size == NULL
6849 && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
6850 fd.chunk_size = integer_zero_node;
6851 gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
6852 fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
6853 ? 3 : fd.sched_kind;
6854 fn_index += fd.have_ordered * 4;
6855 start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
6856 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
6857 if (fd.iter_type == long_long_unsigned_type_node)
6859 start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6860 - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
6861 next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6862 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
6864 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
6865 (enum built_in_function) next_ix, inner_stmt);
6868 if (gimple_in_ssa_p (cfun))
6869 update_ssa (TODO_update_ssa_only_virtuals);
6873 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
6875 v = GOMP_sections_start (n);
6877 switch (v)
6879 case 0:
6880 goto L2;
6881 case 1:
6882 section 1;
6883 goto L1;
6884 case 2:
6886 case n:
6888 default:
6889 abort ();
6892 v = GOMP_sections_next ();
6893 goto L0;
6895 reduction;
6897 If this is a combined parallel sections, replace the call to
6898 GOMP_sections_start with call to GOMP_sections_next. */
6900 static void
6901 expand_omp_sections (struct omp_region *region)
6903 tree t, u, vin = NULL, vmain, vnext, l2;
6904 vec<tree> label_vec;
6905 unsigned len;
6906 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
6907 gimple_stmt_iterator si, switch_si;
6908 gimple sections_stmt, stmt, cont;
6909 edge_iterator ei;
6910 edge e;
6911 struct omp_region *inner;
6912 unsigned i, casei;
6913 bool exit_reachable = region->cont != NULL;
6915 gcc_assert (region->exit != NULL);
6916 entry_bb = region->entry;
6917 l0_bb = single_succ (entry_bb);
6918 l1_bb = region->cont;
6919 l2_bb = region->exit;
6920 if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb)
6921 l2 = gimple_block_label (l2_bb);
6922 else
6924 /* This can happen if there are reductions. */
6925 len = EDGE_COUNT (l0_bb->succs);
6926 gcc_assert (len > 0);
6927 e = EDGE_SUCC (l0_bb, len - 1);
6928 si = gsi_last_bb (e->dest);
6929 l2 = NULL_TREE;
6930 if (gsi_end_p (si)
6931 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
6932 l2 = gimple_block_label (e->dest);
6933 else
6934 FOR_EACH_EDGE (e, ei, l0_bb->succs)
6936 si = gsi_last_bb (e->dest);
6937 if (gsi_end_p (si)
6938 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
6940 l2 = gimple_block_label (e->dest);
6941 break;
6945 if (exit_reachable)
6946 default_bb = create_empty_bb (l1_bb->prev_bb);
6947 else
6948 default_bb = create_empty_bb (l0_bb);
6950 /* We will build a switch() with enough cases for all the
6951 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
6952 and a default case to abort if something goes wrong. */
6953 len = EDGE_COUNT (l0_bb->succs);
6955 /* Use vec::quick_push on label_vec throughout, since we know the size
6956 in advance. */
6957 label_vec.create (len);
6959 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
6960 GIMPLE_OMP_SECTIONS statement. */
6961 si = gsi_last_bb (entry_bb);
6962 sections_stmt = gsi_stmt (si);
6963 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
6964 vin = gimple_omp_sections_control (sections_stmt);
6965 if (!is_combined_parallel (region))
6967 /* If we are not inside a combined parallel+sections region,
6968 call GOMP_sections_start. */
6969 t = build_int_cst (unsigned_type_node, len - 1);
6970 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
6971 stmt = gimple_build_call (u, 1, t);
6973 else
6975 /* Otherwise, call GOMP_sections_next. */
6976 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
6977 stmt = gimple_build_call (u, 0);
6979 gimple_call_set_lhs (stmt, vin);
6980 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
6981 gsi_remove (&si, true);
6983 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
6984 L0_BB. */
6985 switch_si = gsi_last_bb (l0_bb);
6986 gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH);
6987 if (exit_reachable)
6989 cont = last_stmt (l1_bb);
6990 gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE);
6991 vmain = gimple_omp_continue_control_use (cont);
6992 vnext = gimple_omp_continue_control_def (cont);
6994 else
6996 vmain = vin;
6997 vnext = NULL_TREE;
7000 t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2);
7001 label_vec.quick_push (t);
7002 i = 1;
7004 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7005 for (inner = region->inner, casei = 1;
7006 inner;
7007 inner = inner->next, i++, casei++)
7009 basic_block s_entry_bb, s_exit_bb;
7011 /* Skip optional reduction region. */
7012 if (inner->type == GIMPLE_OMP_ATOMIC_LOAD)
7014 --i;
7015 --casei;
7016 continue;
7019 s_entry_bb = inner->entry;
7020 s_exit_bb = inner->exit;
7022 t = gimple_block_label (s_entry_bb);
7023 u = build_int_cst (unsigned_type_node, casei);
7024 u = build_case_label (u, NULL, t);
7025 label_vec.quick_push (u);
7027 si = gsi_last_bb (s_entry_bb);
7028 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION);
7029 gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si)));
7030 gsi_remove (&si, true);
7031 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
7033 if (s_exit_bb == NULL)
7034 continue;
7036 si = gsi_last_bb (s_exit_bb);
7037 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
7038 gsi_remove (&si, true);
7040 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
7043 /* Error handling code goes in DEFAULT_BB. */
7044 t = gimple_block_label (default_bb);
7045 u = build_case_label (NULL, NULL, t);
7046 make_edge (l0_bb, default_bb, 0);
7047 if (current_loops)
7048 add_bb_to_loop (default_bb, current_loops->tree_root);
7050 stmt = gimple_build_switch (vmain, u, label_vec);
7051 gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT);
7052 gsi_remove (&switch_si, true);
7053 label_vec.release ();
7055 si = gsi_start_bb (default_bb);
7056 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
7057 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
7059 if (exit_reachable)
7061 tree bfn_decl;
7063 /* Code to get the next section goes in L1_BB. */
7064 si = gsi_last_bb (l1_bb);
7065 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
7067 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
7068 stmt = gimple_build_call (bfn_decl, 0);
7069 gimple_call_set_lhs (stmt, vnext);
7070 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
7071 gsi_remove (&si, true);
7073 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
7076 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7077 si = gsi_last_bb (l2_bb);
7078 if (gimple_omp_return_nowait_p (gsi_stmt (si)))
7079 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT);
7080 else if (gimple_omp_return_lhs (gsi_stmt (si)))
7081 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL);
7082 else
7083 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END);
7084 stmt = gimple_build_call (t, 0);
7085 if (gimple_omp_return_lhs (gsi_stmt (si)))
7086 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si)));
7087 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
7088 gsi_remove (&si, true);
7090 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
7094 /* Expand code for an OpenMP single directive. We've already expanded
7095 much of the code, here we simply place the GOMP_barrier call. */
7097 static void
7098 expand_omp_single (struct omp_region *region)
7100 basic_block entry_bb, exit_bb;
7101 gimple_stmt_iterator si;
7103 entry_bb = region->entry;
7104 exit_bb = region->exit;
7106 si = gsi_last_bb (entry_bb);
7107 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE);
7108 gsi_remove (&si, true);
7109 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7111 si = gsi_last_bb (exit_bb);
7112 if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
7114 tree t = gimple_omp_return_lhs (gsi_stmt (si));
7115 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT);
7117 gsi_remove (&si, true);
7118 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
7122 /* Generic expansion for OpenMP synchronization directives: master,
7123 ordered and critical. All we need to do here is remove the entry
7124 and exit markers for REGION. */
7126 static void
7127 expand_omp_synch (struct omp_region *region)
7129 basic_block entry_bb, exit_bb;
7130 gimple_stmt_iterator si;
7132 entry_bb = region->entry;
7133 exit_bb = region->exit;
7135 si = gsi_last_bb (entry_bb);
7136 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
7137 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
7138 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
7139 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
7140 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
7141 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS);
7142 gsi_remove (&si, true);
7143 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7145 if (exit_bb)
7147 si = gsi_last_bb (exit_bb);
7148 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
7149 gsi_remove (&si, true);
7150 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
7154 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7155 operation as a normal volatile load. */
7157 static bool
7158 expand_omp_atomic_load (basic_block load_bb, tree addr,
7159 tree loaded_val, int index)
7161 enum built_in_function tmpbase;
7162 gimple_stmt_iterator gsi;
7163 basic_block store_bb;
7164 location_t loc;
7165 gimple stmt;
7166 tree decl, call, type, itype;
7168 gsi = gsi_last_bb (load_bb);
7169 stmt = gsi_stmt (gsi);
7170 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
7171 loc = gimple_location (stmt);
7173 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7174 is smaller than word size, then expand_atomic_load assumes that the load
7175 is atomic. We could avoid the builtin entirely in this case. */
7177 tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
7178 decl = builtin_decl_explicit (tmpbase);
7179 if (decl == NULL_TREE)
7180 return false;
7182 type = TREE_TYPE (loaded_val);
7183 itype = TREE_TYPE (TREE_TYPE (decl));
7185 call = build_call_expr_loc (loc, decl, 2, addr,
7186 build_int_cst (NULL,
7187 gimple_omp_atomic_seq_cst_p (stmt)
7188 ? MEMMODEL_SEQ_CST
7189 : MEMMODEL_RELAXED));
7190 if (!useless_type_conversion_p (type, itype))
7191 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
7192 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
7194 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
7195 gsi_remove (&gsi, true);
7197 store_bb = single_succ (load_bb);
7198 gsi = gsi_last_bb (store_bb);
7199 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
7200 gsi_remove (&gsi, true);
7202 if (gimple_in_ssa_p (cfun))
7203 update_ssa (TODO_update_ssa_no_phi);
7205 return true;
7208 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7209 operation as a normal volatile store. */
7211 static bool
7212 expand_omp_atomic_store (basic_block load_bb, tree addr,
7213 tree loaded_val, tree stored_val, int index)
7215 enum built_in_function tmpbase;
7216 gimple_stmt_iterator gsi;
7217 basic_block store_bb = single_succ (load_bb);
7218 location_t loc;
7219 gimple stmt;
7220 tree decl, call, type, itype;
7221 enum machine_mode imode;
7222 bool exchange;
7224 gsi = gsi_last_bb (load_bb);
7225 stmt = gsi_stmt (gsi);
7226 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
7228 /* If the load value is needed, then this isn't a store but an exchange. */
7229 exchange = gimple_omp_atomic_need_value_p (stmt);
7231 gsi = gsi_last_bb (store_bb);
7232 stmt = gsi_stmt (gsi);
7233 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE);
7234 loc = gimple_location (stmt);
7236 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7237 is smaller than word size, then expand_atomic_store assumes that the store
7238 is atomic. We could avoid the builtin entirely in this case. */
7240 tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N);
7241 tmpbase = (enum built_in_function) ((int) tmpbase + index + 1);
7242 decl = builtin_decl_explicit (tmpbase);
7243 if (decl == NULL_TREE)
7244 return false;
7246 type = TREE_TYPE (stored_val);
7248 /* Dig out the type of the function's second argument. */
7249 itype = TREE_TYPE (decl);
7250 itype = TYPE_ARG_TYPES (itype);
7251 itype = TREE_CHAIN (itype);
7252 itype = TREE_VALUE (itype);
7253 imode = TYPE_MODE (itype);
7255 if (exchange && !can_atomic_exchange_p (imode, true))
7256 return false;
7258 if (!useless_type_conversion_p (itype, type))
7259 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
7260 call = build_call_expr_loc (loc, decl, 3, addr, stored_val,
7261 build_int_cst (NULL,
7262 gimple_omp_atomic_seq_cst_p (stmt)
7263 ? MEMMODEL_SEQ_CST
7264 : MEMMODEL_RELAXED));
7265 if (exchange)
7267 if (!useless_type_conversion_p (type, itype))
7268 call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
7269 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
7272 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
7273 gsi_remove (&gsi, true);
7275 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7276 gsi = gsi_last_bb (load_bb);
7277 gsi_remove (&gsi, true);
7279 if (gimple_in_ssa_p (cfun))
7280 update_ssa (TODO_update_ssa_no_phi);
7282 return true;
7285 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7286 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7287 size of the data type, and thus usable to find the index of the builtin
7288 decl. Returns false if the expression is not of the proper form. */
7290 static bool
7291 expand_omp_atomic_fetch_op (basic_block load_bb,
7292 tree addr, tree loaded_val,
7293 tree stored_val, int index)
7295 enum built_in_function oldbase, newbase, tmpbase;
7296 tree decl, itype, call;
7297 tree lhs, rhs;
7298 basic_block store_bb = single_succ (load_bb);
7299 gimple_stmt_iterator gsi;
7300 gimple stmt;
7301 location_t loc;
7302 enum tree_code code;
7303 bool need_old, need_new;
7304 enum machine_mode imode;
7305 bool seq_cst;
7307 /* We expect to find the following sequences:
7309 load_bb:
7310 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7312 store_bb:
7313 val = tmp OP something; (or: something OP tmp)
7314 GIMPLE_OMP_STORE (val)
7316 ???FIXME: Allow a more flexible sequence.
7317 Perhaps use data flow to pick the statements.
7321 gsi = gsi_after_labels (store_bb);
7322 stmt = gsi_stmt (gsi);
7323 loc = gimple_location (stmt);
7324 if (!is_gimple_assign (stmt))
7325 return false;
7326 gsi_next (&gsi);
7327 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
7328 return false;
7329 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
7330 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb));
7331 seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb));
7332 gcc_checking_assert (!need_old || !need_new);
7334 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
7335 return false;
7337 /* Check for one of the supported fetch-op operations. */
7338 code = gimple_assign_rhs_code (stmt);
7339 switch (code)
7341 case PLUS_EXPR:
7342 case POINTER_PLUS_EXPR:
7343 oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N;
7344 newbase = BUILT_IN_ATOMIC_ADD_FETCH_N;
7345 break;
7346 case MINUS_EXPR:
7347 oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N;
7348 newbase = BUILT_IN_ATOMIC_SUB_FETCH_N;
7349 break;
7350 case BIT_AND_EXPR:
7351 oldbase = BUILT_IN_ATOMIC_FETCH_AND_N;
7352 newbase = BUILT_IN_ATOMIC_AND_FETCH_N;
7353 break;
7354 case BIT_IOR_EXPR:
7355 oldbase = BUILT_IN_ATOMIC_FETCH_OR_N;
7356 newbase = BUILT_IN_ATOMIC_OR_FETCH_N;
7357 break;
7358 case BIT_XOR_EXPR:
7359 oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N;
7360 newbase = BUILT_IN_ATOMIC_XOR_FETCH_N;
7361 break;
7362 default:
7363 return false;
7366 /* Make sure the expression is of the proper form. */
7367 if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
7368 rhs = gimple_assign_rhs2 (stmt);
7369 else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
7370 && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
7371 rhs = gimple_assign_rhs1 (stmt);
7372 else
7373 return false;
7375 tmpbase = ((enum built_in_function)
7376 ((need_new ? newbase : oldbase) + index + 1));
7377 decl = builtin_decl_explicit (tmpbase);
7378 if (decl == NULL_TREE)
7379 return false;
7380 itype = TREE_TYPE (TREE_TYPE (decl));
7381 imode = TYPE_MODE (itype);
7383 /* We could test all of the various optabs involved, but the fact of the
7384 matter is that (with the exception of i486 vs i586 and xadd) all targets
7385 that support any atomic operaton optab also implements compare-and-swap.
7386 Let optabs.c take care of expanding any compare-and-swap loop. */
7387 if (!can_compare_and_swap_p (imode, true))
7388 return false;
7390 gsi = gsi_last_bb (load_bb);
7391 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
7393 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7394 It only requires that the operation happen atomically. Thus we can
7395 use the RELAXED memory model. */
7396 call = build_call_expr_loc (loc, decl, 3, addr,
7397 fold_convert_loc (loc, itype, rhs),
7398 build_int_cst (NULL,
7399 seq_cst ? MEMMODEL_SEQ_CST
7400 : MEMMODEL_RELAXED));
7402 if (need_old || need_new)
7404 lhs = need_old ? loaded_val : stored_val;
7405 call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
7406 call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call);
7408 else
7409 call = fold_convert_loc (loc, void_type_node, call);
7410 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
7411 gsi_remove (&gsi, true);
7413 gsi = gsi_last_bb (store_bb);
7414 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
7415 gsi_remove (&gsi, true);
7416 gsi = gsi_last_bb (store_bb);
7417 gsi_remove (&gsi, true);
7419 if (gimple_in_ssa_p (cfun))
7420 update_ssa (TODO_update_ssa_no_phi);
7422 return true;
7425 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7427 oldval = *addr;
7428 repeat:
7429 newval = rhs; // with oldval replacing *addr in rhs
7430 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7431 if (oldval != newval)
7432 goto repeat;
7434 INDEX is log2 of the size of the data type, and thus usable to find the
7435 index of the builtin decl. */
7437 static bool
7438 expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
7439 tree addr, tree loaded_val, tree stored_val,
7440 int index)
7442 tree loadedi, storedi, initial, new_storedi, old_vali;
7443 tree type, itype, cmpxchg, iaddr;
7444 gimple_stmt_iterator si;
7445 basic_block loop_header = single_succ (load_bb);
7446 gimple phi, stmt;
7447 edge e;
7448 enum built_in_function fncode;
7450 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7451 order to use the RELAXED memory model effectively. */
7452 fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7453 + index + 1);
7454 cmpxchg = builtin_decl_explicit (fncode);
7455 if (cmpxchg == NULL_TREE)
7456 return false;
7457 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
7458 itype = TREE_TYPE (TREE_TYPE (cmpxchg));
7460 if (!can_compare_and_swap_p (TYPE_MODE (itype), true))
7461 return false;
7463 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7464 si = gsi_last_bb (load_bb);
7465 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
7467 /* For floating-point values, we'll need to view-convert them to integers
7468 so that we can perform the atomic compare and swap. Simplify the
7469 following code by always setting up the "i"ntegral variables. */
7470 if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
7472 tree iaddr_val;
7474 iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode,
7475 true), NULL);
7476 iaddr_val
7477 = force_gimple_operand_gsi (&si,
7478 fold_convert (TREE_TYPE (iaddr), addr),
7479 false, NULL_TREE, true, GSI_SAME_STMT);
7480 stmt = gimple_build_assign (iaddr, iaddr_val);
7481 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7482 loadedi = create_tmp_var (itype, NULL);
7483 if (gimple_in_ssa_p (cfun))
7484 loadedi = make_ssa_name (loadedi, NULL);
7486 else
7488 iaddr = addr;
7489 loadedi = loaded_val;
7492 initial
7493 = force_gimple_operand_gsi (&si,
7494 build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)),
7495 iaddr,
7496 build_int_cst (TREE_TYPE (iaddr), 0)),
7497 true, NULL_TREE, true, GSI_SAME_STMT);
7499 /* Move the value to the LOADEDI temporary. */
7500 if (gimple_in_ssa_p (cfun))
7502 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
7503 phi = create_phi_node (loadedi, loop_header);
7504 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
7505 initial);
7507 else
7508 gsi_insert_before (&si,
7509 gimple_build_assign (loadedi, initial),
7510 GSI_SAME_STMT);
7511 if (loadedi != loaded_val)
7513 gimple_stmt_iterator gsi2;
7514 tree x;
7516 x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
7517 gsi2 = gsi_start_bb (loop_header);
7518 if (gimple_in_ssa_p (cfun))
7520 gimple stmt;
7521 x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
7522 true, GSI_SAME_STMT);
7523 stmt = gimple_build_assign (loaded_val, x);
7524 gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
7526 else
7528 x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
7529 force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
7530 true, GSI_SAME_STMT);
7533 gsi_remove (&si, true);
7535 si = gsi_last_bb (store_bb);
7536 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
7538 if (iaddr == addr)
7539 storedi = stored_val;
7540 else
7541 storedi =
7542 force_gimple_operand_gsi (&si,
7543 build1 (VIEW_CONVERT_EXPR, itype,
7544 stored_val), true, NULL_TREE, true,
7545 GSI_SAME_STMT);
7547 /* Build the compare&swap statement. */
7548 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
7549 new_storedi = force_gimple_operand_gsi (&si,
7550 fold_convert (TREE_TYPE (loadedi),
7551 new_storedi),
7552 true, NULL_TREE,
7553 true, GSI_SAME_STMT);
7555 if (gimple_in_ssa_p (cfun))
7556 old_vali = loadedi;
7557 else
7559 old_vali = create_tmp_var (TREE_TYPE (loadedi), NULL);
7560 stmt = gimple_build_assign (old_vali, loadedi);
7561 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7563 stmt = gimple_build_assign (loadedi, new_storedi);
7564 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7567 /* Note that we always perform the comparison as an integer, even for
7568 floating point. This allows the atomic operation to properly
7569 succeed even with NaNs and -0.0. */
7570 stmt = gimple_build_cond_empty
7571 (build2 (NE_EXPR, boolean_type_node,
7572 new_storedi, old_vali));
7573 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7575 /* Update cfg. */
7576 e = single_succ_edge (store_bb);
7577 e->flags &= ~EDGE_FALLTHRU;
7578 e->flags |= EDGE_FALSE_VALUE;
7580 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
7582 /* Copy the new value to loadedi (we already did that before the condition
7583 if we are not in SSA). */
7584 if (gimple_in_ssa_p (cfun))
7586 phi = gimple_seq_first_stmt (phi_nodes (loop_header));
7587 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
7590 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7591 gsi_remove (&si, true);
7593 struct loop *loop = alloc_loop ();
7594 loop->header = loop_header;
7595 loop->latch = store_bb;
7596 add_loop (loop, loop_header->loop_father);
7598 if (gimple_in_ssa_p (cfun))
7599 update_ssa (TODO_update_ssa_no_phi);
7601 return true;
7604 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7606 GOMP_atomic_start ();
7607 *addr = rhs;
7608 GOMP_atomic_end ();
7610 The result is not globally atomic, but works so long as all parallel
7611 references are within #pragma omp atomic directives. According to
7612 responses received from omp@openmp.org, appears to be within spec.
7613 Which makes sense, since that's how several other compilers handle
7614 this situation as well.
7615 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7616 expanding. STORED_VAL is the operand of the matching
7617 GIMPLE_OMP_ATOMIC_STORE.
7619 We replace
7620 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7621 loaded_val = *addr;
7623 and replace
7624 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7625 *addr = stored_val;
7628 static bool
7629 expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
7630 tree addr, tree loaded_val, tree stored_val)
7632 gimple_stmt_iterator si;
7633 gimple stmt;
7634 tree t;
7636 si = gsi_last_bb (load_bb);
7637 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
7639 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
7640 t = build_call_expr (t, 0);
7641 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
7643 stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr));
7644 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7645 gsi_remove (&si, true);
7647 si = gsi_last_bb (store_bb);
7648 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
7650 stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)),
7651 stored_val);
7652 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
7654 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
7655 t = build_call_expr (t, 0);
7656 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
7657 gsi_remove (&si, true);
7659 if (gimple_in_ssa_p (cfun))
7660 update_ssa (TODO_update_ssa_no_phi);
7661 return true;
7664 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7665 using expand_omp_atomic_fetch_op. If it failed, we try to
7666 call expand_omp_atomic_pipeline, and if it fails too, the
7667 ultimate fallback is wrapping the operation in a mutex
7668 (expand_omp_atomic_mutex). REGION is the atomic region built
7669 by build_omp_regions_1(). */
7671 static void
7672 expand_omp_atomic (struct omp_region *region)
7674 basic_block load_bb = region->entry, store_bb = region->exit;
7675 gimple load = last_stmt (load_bb), store = last_stmt (store_bb);
7676 tree loaded_val = gimple_omp_atomic_load_lhs (load);
7677 tree addr = gimple_omp_atomic_load_rhs (load);
7678 tree stored_val = gimple_omp_atomic_store_val (store);
7679 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
7680 HOST_WIDE_INT index;
7682 /* Make sure the type is one of the supported sizes. */
7683 index = tree_to_uhwi (TYPE_SIZE_UNIT (type));
7684 index = exact_log2 (index);
7685 if (index >= 0 && index <= 4)
7687 unsigned int align = TYPE_ALIGN_UNIT (type);
7689 /* __sync builtins require strict data alignment. */
7690 if (exact_log2 (align) >= index)
7692 /* Atomic load. */
7693 if (loaded_val == stored_val
7694 && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
7695 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
7696 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
7697 && expand_omp_atomic_load (load_bb, addr, loaded_val, index))
7698 return;
7700 /* Atomic store. */
7701 if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
7702 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
7703 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
7704 && store_bb == single_succ (load_bb)
7705 && first_stmt (store_bb) == store
7706 && expand_omp_atomic_store (load_bb, addr, loaded_val,
7707 stored_val, index))
7708 return;
7710 /* When possible, use specialized atomic update functions. */
7711 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
7712 && store_bb == single_succ (load_bb)
7713 && expand_omp_atomic_fetch_op (load_bb, addr,
7714 loaded_val, stored_val, index))
7715 return;
7717 /* If we don't have specialized __sync builtins, try and implement
7718 as a compare and swap loop. */
7719 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
7720 loaded_val, stored_val, index))
7721 return;
7725 /* The ultimate fallback is wrapping the operation in a mutex. */
7726 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
7730 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7732 static void
7733 expand_omp_target (struct omp_region *region)
7735 basic_block entry_bb, exit_bb, new_bb;
7736 struct function *child_cfun = NULL;
7737 tree child_fn = NULL_TREE, block, t;
7738 gimple_stmt_iterator gsi;
7739 gimple entry_stmt, stmt;
7740 edge e;
7742 entry_stmt = last_stmt (region->entry);
7743 new_bb = region->entry;
7744 int kind = gimple_omp_target_kind (entry_stmt);
7745 if (kind == GF_OMP_TARGET_KIND_REGION)
7747 child_fn = gimple_omp_target_child_fn (entry_stmt);
7748 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
7751 entry_bb = region->entry;
7752 exit_bb = region->exit;
7754 if (kind == GF_OMP_TARGET_KIND_REGION)
7756 unsigned srcidx, dstidx, num;
7758 /* If the target region needs data sent from the parent
7759 function, then the very first statement (except possible
7760 tree profile counter updates) of the parallel body
7761 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7762 &.OMP_DATA_O is passed as an argument to the child function,
7763 we need to replace it with the argument as seen by the child
7764 function.
7766 In most cases, this will end up being the identity assignment
7767 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7768 a function call that has been inlined, the original PARM_DECL
7769 .OMP_DATA_I may have been converted into a different local
7770 variable. In which case, we need to keep the assignment. */
7771 if (gimple_omp_target_data_arg (entry_stmt))
7773 basic_block entry_succ_bb = single_succ (entry_bb);
7774 gimple_stmt_iterator gsi;
7775 tree arg;
7776 gimple tgtcopy_stmt = NULL;
7777 tree sender
7778 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt), 0);
7780 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
7782 gcc_assert (!gsi_end_p (gsi));
7783 stmt = gsi_stmt (gsi);
7784 if (gimple_code (stmt) != GIMPLE_ASSIGN)
7785 continue;
7787 if (gimple_num_ops (stmt) == 2)
7789 tree arg = gimple_assign_rhs1 (stmt);
7791 /* We're ignoring the subcode because we're
7792 effectively doing a STRIP_NOPS. */
7794 if (TREE_CODE (arg) == ADDR_EXPR
7795 && TREE_OPERAND (arg, 0) == sender)
7797 tgtcopy_stmt = stmt;
7798 break;
7803 gcc_assert (tgtcopy_stmt != NULL);
7804 arg = DECL_ARGUMENTS (child_fn);
7806 gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg);
7807 gsi_remove (&gsi, true);
7810 /* Declare local variables needed in CHILD_CFUN. */
7811 block = DECL_INITIAL (child_fn);
7812 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
7813 /* The gimplifier could record temporaries in target block
7814 rather than in containing function's local_decls chain,
7815 which would mean cgraph missed finalizing them. Do it now. */
7816 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
7817 if (TREE_CODE (t) == VAR_DECL
7818 && TREE_STATIC (t)
7819 && !DECL_EXTERNAL (t))
7820 varpool_finalize_decl (t);
7821 DECL_SAVED_TREE (child_fn) = NULL;
7822 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7823 gimple_set_body (child_fn, NULL);
7824 TREE_USED (block) = 1;
7826 /* Reset DECL_CONTEXT on function arguments. */
7827 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
7828 DECL_CONTEXT (t) = child_fn;
7830 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7831 so that it can be moved to the child function. */
7832 gsi = gsi_last_bb (entry_bb);
7833 stmt = gsi_stmt (gsi);
7834 gcc_assert (stmt && gimple_code (stmt) == GIMPLE_OMP_TARGET
7835 && gimple_omp_target_kind (stmt)
7836 == GF_OMP_TARGET_KIND_REGION);
7837 gsi_remove (&gsi, true);
7838 e = split_block (entry_bb, stmt);
7839 entry_bb = e->dest;
7840 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7842 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7843 if (exit_bb)
7845 gsi = gsi_last_bb (exit_bb);
7846 gcc_assert (!gsi_end_p (gsi)
7847 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
7848 stmt = gimple_build_return (NULL);
7849 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
7850 gsi_remove (&gsi, true);
7853 /* Move the target region into CHILD_CFUN. */
7855 block = gimple_block (entry_stmt);
7857 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
7858 if (exit_bb)
7859 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
7860 /* When the OMP expansion process cannot guarantee an up-to-date
7861 loop tree arrange for the child function to fixup loops. */
7862 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
7863 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
7865 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7866 num = vec_safe_length (child_cfun->local_decls);
7867 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
7869 t = (*child_cfun->local_decls)[srcidx];
7870 if (DECL_CONTEXT (t) == cfun->decl)
7871 continue;
7872 if (srcidx != dstidx)
7873 (*child_cfun->local_decls)[dstidx] = t;
7874 dstidx++;
7876 if (dstidx != num)
7877 vec_safe_truncate (child_cfun->local_decls, dstidx);
7879 /* Inform the callgraph about the new function. */
7880 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
7881 cgraph_add_new_function (child_fn, true);
7883 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7884 fixed in a following pass. */
7885 push_cfun (child_cfun);
7886 rebuild_cgraph_edges ();
7888 /* Some EH regions might become dead, see PR34608. If
7889 pass_cleanup_cfg isn't the first pass to happen with the
7890 new child, these dead EH edges might cause problems.
7891 Clean them up now. */
7892 if (flag_exceptions)
7894 basic_block bb;
7895 bool changed = false;
7897 FOR_EACH_BB (bb)
7898 changed |= gimple_purge_dead_eh_edges (bb);
7899 if (changed)
7900 cleanup_tree_cfg ();
7902 pop_cfun ();
7905 /* Emit a library call to launch the target region, or do data
7906 transfers. */
7907 tree t1, t2, t3, t4, device, cond, c, clauses;
7908 enum built_in_function start_ix;
7909 location_t clause_loc;
7911 clauses = gimple_omp_target_clauses (entry_stmt);
7913 if (kind == GF_OMP_TARGET_KIND_REGION)
7914 start_ix = BUILT_IN_GOMP_TARGET;
7915 else if (kind == GF_OMP_TARGET_KIND_DATA)
7916 start_ix = BUILT_IN_GOMP_TARGET_DATA;
7917 else
7918 start_ix = BUILT_IN_GOMP_TARGET_UPDATE;
7920 /* By default, the value of DEVICE is -1 (let runtime library choose)
7921 and there is no conditional. */
7922 cond = NULL_TREE;
7923 device = build_int_cst (integer_type_node, -1);
7925 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
7926 if (c)
7927 cond = OMP_CLAUSE_IF_EXPR (c);
7929 c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE);
7930 if (c)
7932 device = OMP_CLAUSE_DEVICE_ID (c);
7933 clause_loc = OMP_CLAUSE_LOCATION (c);
7935 else
7936 clause_loc = gimple_location (entry_stmt);
7938 /* Ensure 'device' is of the correct type. */
7939 device = fold_convert_loc (clause_loc, integer_type_node, device);
7941 /* If we found the clause 'if (cond)', build
7942 (cond ? device : -2). */
7943 if (cond)
7945 cond = gimple_boolify (cond);
7947 basic_block cond_bb, then_bb, else_bb;
7948 edge e;
7949 tree tmp_var;
7951 tmp_var = create_tmp_var (TREE_TYPE (device), NULL);
7952 if (kind != GF_OMP_TARGET_KIND_REGION)
7954 gsi = gsi_last_bb (new_bb);
7955 gsi_prev (&gsi);
7956 e = split_block (new_bb, gsi_stmt (gsi));
7958 else
7959 e = split_block (new_bb, NULL);
7960 cond_bb = e->src;
7961 new_bb = e->dest;
7962 remove_edge (e);
7964 then_bb = create_empty_bb (cond_bb);
7965 else_bb = create_empty_bb (then_bb);
7966 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
7967 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
7969 stmt = gimple_build_cond_empty (cond);
7970 gsi = gsi_last_bb (cond_bb);
7971 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7973 gsi = gsi_start_bb (then_bb);
7974 stmt = gimple_build_assign (tmp_var, device);
7975 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7977 gsi = gsi_start_bb (else_bb);
7978 stmt = gimple_build_assign (tmp_var,
7979 build_int_cst (integer_type_node, -2));
7980 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7982 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
7983 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
7984 if (current_loops)
7986 add_bb_to_loop (then_bb, cond_bb->loop_father);
7987 add_bb_to_loop (else_bb, cond_bb->loop_father);
7989 make_edge (then_bb, new_bb, EDGE_FALLTHRU);
7990 make_edge (else_bb, new_bb, EDGE_FALLTHRU);
7992 device = tmp_var;
7995 gsi = gsi_last_bb (new_bb);
7996 t = gimple_omp_target_data_arg (entry_stmt);
7997 if (t == NULL)
7999 t1 = size_zero_node;
8000 t2 = build_zero_cst (ptr_type_node);
8001 t3 = t2;
8002 t4 = t2;
8004 else
8006 t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1))));
8007 t1 = size_binop (PLUS_EXPR, t1, size_int (1));
8008 t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0));
8009 t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1));
8010 t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2));
8013 gimple g;
8014 /* FIXME: This will be address of
8015 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8016 symbol, as soon as the linker plugin is able to create it for us. */
8017 tree openmp_target = build_zero_cst (ptr_type_node);
8018 if (kind == GF_OMP_TARGET_KIND_REGION)
8020 tree fnaddr = build_fold_addr_expr (child_fn);
8021 g = gimple_build_call (builtin_decl_explicit (start_ix), 7,
8022 device, fnaddr, openmp_target, t1, t2, t3, t4);
8024 else
8025 g = gimple_build_call (builtin_decl_explicit (start_ix), 6,
8026 device, openmp_target, t1, t2, t3, t4);
8027 gimple_set_location (g, gimple_location (entry_stmt));
8028 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
8029 if (kind != GF_OMP_TARGET_KIND_REGION)
8031 g = gsi_stmt (gsi);
8032 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET);
8033 gsi_remove (&gsi, true);
8035 if (kind == GF_OMP_TARGET_KIND_DATA && region->exit)
8037 gsi = gsi_last_bb (region->exit);
8038 g = gsi_stmt (gsi);
8039 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN);
8040 gsi_remove (&gsi, true);
8045 /* Expand the parallel region tree rooted at REGION. Expansion
8046 proceeds in depth-first order. Innermost regions are expanded
8047 first. This way, parallel regions that require a new function to
8048 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8049 internal dependencies in their body. */
8051 static void
8052 expand_omp (struct omp_region *region)
8054 while (region)
8056 location_t saved_location;
8057 gimple inner_stmt = NULL;
8059 /* First, determine whether this is a combined parallel+workshare
8060 region. */
8061 if (region->type == GIMPLE_OMP_PARALLEL)
8062 determine_parallel_type (region);
8064 if (region->type == GIMPLE_OMP_FOR
8065 && gimple_omp_for_combined_p (last_stmt (region->entry)))
8066 inner_stmt = last_stmt (region->inner->entry);
8068 if (region->inner)
8069 expand_omp (region->inner);
8071 saved_location = input_location;
8072 if (gimple_has_location (last_stmt (region->entry)))
8073 input_location = gimple_location (last_stmt (region->entry));
8075 switch (region->type)
8077 case GIMPLE_OMP_PARALLEL:
8078 case GIMPLE_OMP_TASK:
8079 expand_omp_taskreg (region);
8080 break;
8082 case GIMPLE_OMP_FOR:
8083 expand_omp_for (region, inner_stmt);
8084 break;
8086 case GIMPLE_OMP_SECTIONS:
8087 expand_omp_sections (region);
8088 break;
8090 case GIMPLE_OMP_SECTION:
8091 /* Individual omp sections are handled together with their
8092 parent GIMPLE_OMP_SECTIONS region. */
8093 break;
8095 case GIMPLE_OMP_SINGLE:
8096 expand_omp_single (region);
8097 break;
8099 case GIMPLE_OMP_MASTER:
8100 case GIMPLE_OMP_TASKGROUP:
8101 case GIMPLE_OMP_ORDERED:
8102 case GIMPLE_OMP_CRITICAL:
8103 case GIMPLE_OMP_TEAMS:
8104 expand_omp_synch (region);
8105 break;
8107 case GIMPLE_OMP_ATOMIC_LOAD:
8108 expand_omp_atomic (region);
8109 break;
8111 case GIMPLE_OMP_TARGET:
8112 expand_omp_target (region);
8113 break;
8115 default:
8116 gcc_unreachable ();
8119 input_location = saved_location;
8120 region = region->next;
8125 /* Helper for build_omp_regions. Scan the dominator tree starting at
8126 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8127 true, the function ends once a single tree is built (otherwise, whole
8128 forest of OMP constructs may be built). */
8130 static void
8131 build_omp_regions_1 (basic_block bb, struct omp_region *parent,
8132 bool single_tree)
8134 gimple_stmt_iterator gsi;
8135 gimple stmt;
8136 basic_block son;
8138 gsi = gsi_last_bb (bb);
8139 if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi)))
8141 struct omp_region *region;
8142 enum gimple_code code;
8144 stmt = gsi_stmt (gsi);
8145 code = gimple_code (stmt);
8146 if (code == GIMPLE_OMP_RETURN)
8148 /* STMT is the return point out of region PARENT. Mark it
8149 as the exit point and make PARENT the immediately
8150 enclosing region. */
8151 gcc_assert (parent);
8152 region = parent;
8153 region->exit = bb;
8154 parent = parent->outer;
8156 else if (code == GIMPLE_OMP_ATOMIC_STORE)
8158 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8159 GIMPLE_OMP_RETURN, but matches with
8160 GIMPLE_OMP_ATOMIC_LOAD. */
8161 gcc_assert (parent);
8162 gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD);
8163 region = parent;
8164 region->exit = bb;
8165 parent = parent->outer;
8168 else if (code == GIMPLE_OMP_CONTINUE)
8170 gcc_assert (parent);
8171 parent->cont = bb;
8173 else if (code == GIMPLE_OMP_SECTIONS_SWITCH)
8175 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8176 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8179 else if (code == GIMPLE_OMP_TARGET
8180 && gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_UPDATE)
8181 new_omp_region (bb, code, parent);
8182 else
8184 /* Otherwise, this directive becomes the parent for a new
8185 region. */
8186 region = new_omp_region (bb, code, parent);
8187 parent = region;
8191 if (single_tree && !parent)
8192 return;
8194 for (son = first_dom_son (CDI_DOMINATORS, bb);
8195 son;
8196 son = next_dom_son (CDI_DOMINATORS, son))
8197 build_omp_regions_1 (son, parent, single_tree);
8200 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8201 root_omp_region. */
8203 static void
8204 build_omp_regions_root (basic_block root)
8206 gcc_assert (root_omp_region == NULL);
8207 build_omp_regions_1 (root, NULL, true);
8208 gcc_assert (root_omp_region != NULL);
8211 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8213 void
8214 omp_expand_local (basic_block head)
8216 build_omp_regions_root (head);
8217 if (dump_file && (dump_flags & TDF_DETAILS))
8219 fprintf (dump_file, "\nOMP region tree\n\n");
8220 dump_omp_region (dump_file, root_omp_region, 0);
8221 fprintf (dump_file, "\n");
8224 remove_exit_barriers (root_omp_region);
8225 expand_omp (root_omp_region);
8227 free_omp_regions ();
8230 /* Scan the CFG and build a tree of OMP regions. Return the root of
8231 the OMP region tree. */
8233 static void
8234 build_omp_regions (void)
8236 gcc_assert (root_omp_region == NULL);
8237 calculate_dominance_info (CDI_DOMINATORS);
8238 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false);
8241 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8243 static unsigned int
8244 execute_expand_omp (void)
8246 build_omp_regions ();
8248 if (!root_omp_region)
8249 return 0;
8251 if (dump_file)
8253 fprintf (dump_file, "\nOMP region tree\n\n");
8254 dump_omp_region (dump_file, root_omp_region, 0);
8255 fprintf (dump_file, "\n");
8258 remove_exit_barriers (root_omp_region);
8260 expand_omp (root_omp_region);
8262 cleanup_tree_cfg ();
8264 free_omp_regions ();
8266 return 0;
8269 /* OMP expansion -- the default pass, run before creation of SSA form. */
8271 static bool
8272 gate_expand_omp (void)
8274 return ((flag_openmp != 0 || flag_openmp_simd != 0
8275 || flag_enable_cilkplus != 0) && !seen_error ());
8278 namespace {
8280 const pass_data pass_data_expand_omp =
8282 GIMPLE_PASS, /* type */
8283 "ompexp", /* name */
8284 OPTGROUP_NONE, /* optinfo_flags */
8285 true, /* has_gate */
8286 true, /* has_execute */
8287 TV_NONE, /* tv_id */
8288 PROP_gimple_any, /* properties_required */
8289 0, /* properties_provided */
8290 0, /* properties_destroyed */
8291 0, /* todo_flags_start */
8292 0, /* todo_flags_finish */
8295 class pass_expand_omp : public gimple_opt_pass
8297 public:
8298 pass_expand_omp (gcc::context *ctxt)
8299 : gimple_opt_pass (pass_data_expand_omp, ctxt)
8302 /* opt_pass methods: */
8303 bool gate () { return gate_expand_omp (); }
8304 unsigned int execute () { return execute_expand_omp (); }
8306 }; // class pass_expand_omp
8308 } // anon namespace
8310 gimple_opt_pass *
8311 make_pass_expand_omp (gcc::context *ctxt)
8313 return new pass_expand_omp (ctxt);
8316 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8318 /* If ctx is a worksharing context inside of a cancellable parallel
8319 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8320 and conditional branch to parallel's cancel_label to handle
8321 cancellation in the implicit barrier. */
8323 static void
8324 maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body)
8326 gimple omp_return = gimple_seq_last_stmt (*body);
8327 gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
8328 if (gimple_omp_return_nowait_p (omp_return))
8329 return;
8330 if (ctx->outer
8331 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL
8332 && ctx->outer->cancellable)
8334 tree lhs = create_tmp_var (boolean_type_node, NULL);
8335 gimple_omp_return_set_lhs (omp_return, lhs);
8336 tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
8337 gimple g = gimple_build_cond (NE_EXPR, lhs, boolean_false_node,
8338 ctx->outer->cancel_label, fallthru_label);
8339 gimple_seq_add_stmt (body, g);
8340 gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
8344 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8345 CTX is the enclosing OMP context for the current statement. */
8347 static void
8348 lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8350 tree block, control;
8351 gimple_stmt_iterator tgsi;
8352 gimple stmt, new_stmt, bind, t;
8353 gimple_seq ilist, dlist, olist, new_body;
8355 stmt = gsi_stmt (*gsi_p);
8357 push_gimplify_context ();
8359 dlist = NULL;
8360 ilist = NULL;
8361 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
8362 &ilist, &dlist, ctx, NULL);
8364 new_body = gimple_omp_body (stmt);
8365 gimple_omp_set_body (stmt, NULL);
8366 tgsi = gsi_start (new_body);
8367 for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
8369 omp_context *sctx;
8370 gimple sec_start;
8372 sec_start = gsi_stmt (tgsi);
8373 sctx = maybe_lookup_ctx (sec_start);
8374 gcc_assert (sctx);
8376 lower_omp (gimple_omp_body_ptr (sec_start), sctx);
8377 gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
8378 GSI_CONTINUE_LINKING);
8379 gimple_omp_set_body (sec_start, NULL);
8381 if (gsi_one_before_end_p (tgsi))
8383 gimple_seq l = NULL;
8384 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
8385 &l, ctx);
8386 gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
8387 gimple_omp_section_set_last (sec_start);
8390 gsi_insert_after (&tgsi, gimple_build_omp_return (false),
8391 GSI_CONTINUE_LINKING);
8394 block = make_node (BLOCK);
8395 bind = gimple_build_bind (NULL, new_body, block);
8397 olist = NULL;
8398 lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx);
8400 block = make_node (BLOCK);
8401 new_stmt = gimple_build_bind (NULL, NULL, block);
8402 gsi_replace (gsi_p, new_stmt, true);
8404 pop_gimplify_context (new_stmt);
8405 gimple_bind_append_vars (new_stmt, ctx->block_vars);
8406 BLOCK_VARS (block) = gimple_bind_vars (bind);
8407 if (BLOCK_VARS (block))
8408 TREE_USED (block) = 1;
8410 new_body = NULL;
8411 gimple_seq_add_seq (&new_body, ilist);
8412 gimple_seq_add_stmt (&new_body, stmt);
8413 gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
8414 gimple_seq_add_stmt (&new_body, bind);
8416 control = create_tmp_var (unsigned_type_node, ".section");
8417 t = gimple_build_omp_continue (control, control);
8418 gimple_omp_sections_set_control (stmt, control);
8419 gimple_seq_add_stmt (&new_body, t);
8421 gimple_seq_add_seq (&new_body, olist);
8422 if (ctx->cancellable)
8423 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
8424 gimple_seq_add_seq (&new_body, dlist);
8426 new_body = maybe_catch_exception (new_body);
8428 t = gimple_build_omp_return
8429 (!!find_omp_clause (gimple_omp_sections_clauses (stmt),
8430 OMP_CLAUSE_NOWAIT));
8431 gimple_seq_add_stmt (&new_body, t);
8432 maybe_add_implicit_barrier_cancel (ctx, &new_body);
8434 gimple_bind_set_body (new_stmt, new_body);
8438 /* A subroutine of lower_omp_single. Expand the simple form of
8439 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8441 if (GOMP_single_start ())
8442 BODY;
8443 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8445 FIXME. It may be better to delay expanding the logic of this until
8446 pass_expand_omp. The expanded logic may make the job more difficult
8447 to a synchronization analysis pass. */
8449 static void
8450 lower_omp_single_simple (gimple single_stmt, gimple_seq *pre_p)
8452 location_t loc = gimple_location (single_stmt);
8453 tree tlabel = create_artificial_label (loc);
8454 tree flabel = create_artificial_label (loc);
8455 gimple call, cond;
8456 tree lhs, decl;
8458 decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START);
8459 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)), NULL);
8460 call = gimple_build_call (decl, 0);
8461 gimple_call_set_lhs (call, lhs);
8462 gimple_seq_add_stmt (pre_p, call);
8464 cond = gimple_build_cond (EQ_EXPR, lhs,
8465 fold_convert_loc (loc, TREE_TYPE (lhs),
8466 boolean_true_node),
8467 tlabel, flabel);
8468 gimple_seq_add_stmt (pre_p, cond);
8469 gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
8470 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8471 gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
8475 /* A subroutine of lower_omp_single. Expand the simple form of
8476 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8478 #pragma omp single copyprivate (a, b, c)
8480 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8483 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8485 BODY;
8486 copyout.a = a;
8487 copyout.b = b;
8488 copyout.c = c;
8489 GOMP_single_copy_end (&copyout);
8491 else
8493 a = copyout_p->a;
8494 b = copyout_p->b;
8495 c = copyout_p->c;
8497 GOMP_barrier ();
8500 FIXME. It may be better to delay expanding the logic of this until
8501 pass_expand_omp. The expanded logic may make the job more difficult
8502 to a synchronization analysis pass. */
8504 static void
8505 lower_omp_single_copy (gimple single_stmt, gimple_seq *pre_p, omp_context *ctx)
8507 tree ptr_type, t, l0, l1, l2, bfn_decl;
8508 gimple_seq copyin_seq;
8509 location_t loc = gimple_location (single_stmt);
8511 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
8513 ptr_type = build_pointer_type (ctx->record_type);
8514 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
8516 l0 = create_artificial_label (loc);
8517 l1 = create_artificial_label (loc);
8518 l2 = create_artificial_label (loc);
8520 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
8521 t = build_call_expr_loc (loc, bfn_decl, 0);
8522 t = fold_convert_loc (loc, ptr_type, t);
8523 gimplify_assign (ctx->receiver_decl, t, pre_p);
8525 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
8526 build_int_cst (ptr_type, 0));
8527 t = build3 (COND_EXPR, void_type_node, t,
8528 build_and_jump (&l0), build_and_jump (&l1));
8529 gimplify_and_add (t, pre_p);
8531 gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
8533 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
8535 copyin_seq = NULL;
8536 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
8537 &copyin_seq, ctx);
8539 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
8540 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
8541 t = build_call_expr_loc (loc, bfn_decl, 1, t);
8542 gimplify_and_add (t, pre_p);
8544 t = build_and_jump (&l2);
8545 gimplify_and_add (t, pre_p);
8547 gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
8549 gimple_seq_add_seq (pre_p, copyin_seq);
8551 gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
8555 /* Expand code for an OpenMP single directive. */
8557 static void
8558 lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8560 tree block;
8561 gimple t, bind, single_stmt = gsi_stmt (*gsi_p);
8562 gimple_seq bind_body, bind_body_tail = NULL, dlist;
8564 push_gimplify_context ();
8566 block = make_node (BLOCK);
8567 bind = gimple_build_bind (NULL, NULL, block);
8568 gsi_replace (gsi_p, bind, true);
8569 bind_body = NULL;
8570 dlist = NULL;
8571 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
8572 &bind_body, &dlist, ctx, NULL);
8573 lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
8575 gimple_seq_add_stmt (&bind_body, single_stmt);
8577 if (ctx->record_type)
8578 lower_omp_single_copy (single_stmt, &bind_body, ctx);
8579 else
8580 lower_omp_single_simple (single_stmt, &bind_body);
8582 gimple_omp_set_body (single_stmt, NULL);
8584 gimple_seq_add_seq (&bind_body, dlist);
8586 bind_body = maybe_catch_exception (bind_body);
8588 t = gimple_build_omp_return
8589 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt),
8590 OMP_CLAUSE_NOWAIT));
8591 gimple_seq_add_stmt (&bind_body_tail, t);
8592 maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail);
8593 if (ctx->record_type)
8595 gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
8596 tree clobber = build_constructor (ctx->record_type, NULL);
8597 TREE_THIS_VOLATILE (clobber) = 1;
8598 gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
8599 clobber), GSI_SAME_STMT);
8601 gimple_seq_add_seq (&bind_body, bind_body_tail);
8602 gimple_bind_set_body (bind, bind_body);
8604 pop_gimplify_context (bind);
8606 gimple_bind_append_vars (bind, ctx->block_vars);
8607 BLOCK_VARS (block) = ctx->block_vars;
8608 if (BLOCK_VARS (block))
8609 TREE_USED (block) = 1;
8613 /* Expand code for an OpenMP master directive. */
8615 static void
8616 lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8618 tree block, lab = NULL, x, bfn_decl;
8619 gimple stmt = gsi_stmt (*gsi_p), bind;
8620 location_t loc = gimple_location (stmt);
8621 gimple_seq tseq;
8623 push_gimplify_context ();
8625 block = make_node (BLOCK);
8626 bind = gimple_build_bind (NULL, NULL, block);
8627 gsi_replace (gsi_p, bind, true);
8628 gimple_bind_add_stmt (bind, stmt);
8630 bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
8631 x = build_call_expr_loc (loc, bfn_decl, 0);
8632 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node);
8633 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
8634 tseq = NULL;
8635 gimplify_and_add (x, &tseq);
8636 gimple_bind_add_seq (bind, tseq);
8638 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8639 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
8640 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8641 gimple_omp_set_body (stmt, NULL);
8643 gimple_bind_add_stmt (bind, gimple_build_label (lab));
8645 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8647 pop_gimplify_context (bind);
8649 gimple_bind_append_vars (bind, ctx->block_vars);
8650 BLOCK_VARS (block) = ctx->block_vars;
8654 /* Expand code for an OpenMP taskgroup directive. */
8656 static void
8657 lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8659 gimple stmt = gsi_stmt (*gsi_p), bind, x;
8660 tree block = make_node (BLOCK);
8662 bind = gimple_build_bind (NULL, NULL, block);
8663 gsi_replace (gsi_p, bind, true);
8664 gimple_bind_add_stmt (bind, stmt);
8666 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
8668 gimple_bind_add_stmt (bind, x);
8670 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8671 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8672 gimple_omp_set_body (stmt, NULL);
8674 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8676 gimple_bind_append_vars (bind, ctx->block_vars);
8677 BLOCK_VARS (block) = ctx->block_vars;
8681 /* Expand code for an OpenMP ordered directive. */
8683 static void
8684 lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8686 tree block;
8687 gimple stmt = gsi_stmt (*gsi_p), bind, x;
8689 push_gimplify_context ();
8691 block = make_node (BLOCK);
8692 bind = gimple_build_bind (NULL, NULL, block);
8693 gsi_replace (gsi_p, bind, true);
8694 gimple_bind_add_stmt (bind, stmt);
8696 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
8698 gimple_bind_add_stmt (bind, x);
8700 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8701 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
8702 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8703 gimple_omp_set_body (stmt, NULL);
8705 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END), 0);
8706 gimple_bind_add_stmt (bind, x);
8708 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8710 pop_gimplify_context (bind);
8712 gimple_bind_append_vars (bind, ctx->block_vars);
8713 BLOCK_VARS (block) = gimple_bind_vars (bind);
8717 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8718 substitution of a couple of function calls. But in the NAMED case,
8719 requires that languages coordinate a symbol name. It is therefore
8720 best put here in common code. */
8722 static GTY((param1_is (tree), param2_is (tree)))
8723 splay_tree critical_name_mutexes;
8725 static void
8726 lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8728 tree block;
8729 tree name, lock, unlock;
8730 gimple stmt = gsi_stmt (*gsi_p), bind;
8731 location_t loc = gimple_location (stmt);
8732 gimple_seq tbody;
8734 name = gimple_omp_critical_name (stmt);
8735 if (name)
8737 tree decl;
8738 splay_tree_node n;
8740 if (!critical_name_mutexes)
8741 critical_name_mutexes
8742 = splay_tree_new_ggc (splay_tree_compare_pointers,
8743 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s,
8744 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s);
8746 n = splay_tree_lookup (critical_name_mutexes, (splay_tree_key) name);
8747 if (n == NULL)
8749 char *new_str;
8751 decl = create_tmp_var_raw (ptr_type_node, NULL);
8753 new_str = ACONCAT ((".gomp_critical_user_",
8754 IDENTIFIER_POINTER (name), NULL));
8755 DECL_NAME (decl) = get_identifier (new_str);
8756 TREE_PUBLIC (decl) = 1;
8757 TREE_STATIC (decl) = 1;
8758 DECL_COMMON (decl) = 1;
8759 DECL_ARTIFICIAL (decl) = 1;
8760 DECL_IGNORED_P (decl) = 1;
8761 varpool_finalize_decl (decl);
8763 splay_tree_insert (critical_name_mutexes, (splay_tree_key) name,
8764 (splay_tree_value) decl);
8766 else
8767 decl = (tree) n->value;
8769 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
8770 lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl));
8772 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
8773 unlock = build_call_expr_loc (loc, unlock, 1,
8774 build_fold_addr_expr_loc (loc, decl));
8776 else
8778 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
8779 lock = build_call_expr_loc (loc, lock, 0);
8781 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
8782 unlock = build_call_expr_loc (loc, unlock, 0);
8785 push_gimplify_context ();
8787 block = make_node (BLOCK);
8788 bind = gimple_build_bind (NULL, NULL, block);
8789 gsi_replace (gsi_p, bind, true);
8790 gimple_bind_add_stmt (bind, stmt);
8792 tbody = gimple_bind_body (bind);
8793 gimplify_and_add (lock, &tbody);
8794 gimple_bind_set_body (bind, tbody);
8796 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8797 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
8798 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
8799 gimple_omp_set_body (stmt, NULL);
8801 tbody = gimple_bind_body (bind);
8802 gimplify_and_add (unlock, &tbody);
8803 gimple_bind_set_body (bind, tbody);
8805 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
8807 pop_gimplify_context (bind);
8808 gimple_bind_append_vars (bind, ctx->block_vars);
8809 BLOCK_VARS (block) = gimple_bind_vars (bind);
8813 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8814 for a lastprivate clause. Given a loop control predicate of (V
8815 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8816 is appended to *DLIST, iterator initialization is appended to
8817 *BODY_P. */
8819 static void
8820 lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
8821 gimple_seq *dlist, struct omp_context *ctx)
8823 tree clauses, cond, vinit;
8824 enum tree_code cond_code;
8825 gimple_seq stmts;
8827 cond_code = fd->loop.cond_code;
8828 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
8830 /* When possible, use a strict equality expression. This can let VRP
8831 type optimizations deduce the value and remove a copy. */
8832 if (tree_fits_shwi_p (fd->loop.step))
8834 HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
8835 if (step == 1 || step == -1)
8836 cond_code = EQ_EXPR;
8839 cond = build2 (cond_code, boolean_type_node, fd->loop.v, fd->loop.n2);
8841 clauses = gimple_omp_for_clauses (fd->for_stmt);
8842 stmts = NULL;
8843 lower_lastprivate_clauses (clauses, cond, &stmts, ctx);
8844 if (!gimple_seq_empty_p (stmts))
8846 gimple_seq_add_seq (&stmts, *dlist);
8847 *dlist = stmts;
8849 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8850 vinit = fd->loop.n1;
8851 if (cond_code == EQ_EXPR
8852 && tree_fits_shwi_p (fd->loop.n2)
8853 && ! integer_zerop (fd->loop.n2))
8854 vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
8855 else
8856 vinit = unshare_expr (vinit);
8858 /* Initialize the iterator variable, so that threads that don't execute
8859 any iterations don't execute the lastprivate clauses by accident. */
8860 gimplify_assign (fd->loop.v, vinit, body_p);
8865 /* Lower code for an OpenMP loop directive. */
8867 static void
8868 lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
8870 tree *rhs_p, block;
8871 struct omp_for_data fd, *fdp = NULL;
8872 gimple stmt = gsi_stmt (*gsi_p), new_stmt;
8873 gimple_seq omp_for_body, body, dlist;
8874 size_t i;
8876 push_gimplify_context ();
8878 lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
8880 block = make_node (BLOCK);
8881 new_stmt = gimple_build_bind (NULL, NULL, block);
8882 /* Replace at gsi right away, so that 'stmt' is no member
8883 of a sequence anymore as we're going to add to to a different
8884 one below. */
8885 gsi_replace (gsi_p, new_stmt, true);
8887 /* Move declaration of temporaries in the loop body before we make
8888 it go away. */
8889 omp_for_body = gimple_omp_body (stmt);
8890 if (!gimple_seq_empty_p (omp_for_body)
8891 && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
8893 tree vars = gimple_bind_vars (gimple_seq_first_stmt (omp_for_body));
8894 gimple_bind_append_vars (new_stmt, vars);
8897 if (gimple_omp_for_combined_into_p (stmt))
8899 extract_omp_for_data (stmt, &fd, NULL);
8900 fdp = &fd;
8902 /* We need two temporaries with fd.loop.v type (istart/iend)
8903 and then (fd.collapse - 1) temporaries with the same
8904 type for count2 ... countN-1 vars if not constant. */
8905 size_t count = 2;
8906 tree type = fd.iter_type;
8907 if (fd.collapse > 1
8908 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
8909 count += fd.collapse - 1;
8910 bool parallel_for = gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR;
8911 tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
8912 tree clauses = *pc;
8913 if (parallel_for)
8914 outerc
8915 = find_omp_clause (gimple_omp_parallel_clauses (ctx->outer->stmt),
8916 OMP_CLAUSE__LOOPTEMP_);
8917 for (i = 0; i < count; i++)
8919 tree temp;
8920 if (parallel_for)
8922 gcc_assert (outerc);
8923 temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
8924 outerc = find_omp_clause (OMP_CLAUSE_CHAIN (outerc),
8925 OMP_CLAUSE__LOOPTEMP_);
8927 else
8928 temp = create_tmp_var (type, NULL);
8929 *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
8930 OMP_CLAUSE_DECL (*pc) = temp;
8931 pc = &OMP_CLAUSE_CHAIN (*pc);
8933 *pc = clauses;
8936 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
8937 dlist = NULL;
8938 body = NULL;
8939 lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
8940 fdp);
8941 gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt));
8943 lower_omp (gimple_omp_body_ptr (stmt), ctx);
8945 /* Lower the header expressions. At this point, we can assume that
8946 the header is of the form:
8948 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
8950 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
8951 using the .omp_data_s mapping, if needed. */
8952 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
8954 rhs_p = gimple_omp_for_initial_ptr (stmt, i);
8955 if (!is_gimple_min_invariant (*rhs_p))
8956 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
8958 rhs_p = gimple_omp_for_final_ptr (stmt, i);
8959 if (!is_gimple_min_invariant (*rhs_p))
8960 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
8962 rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
8963 if (!is_gimple_min_invariant (*rhs_p))
8964 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
8967 /* Once lowered, extract the bounds and clauses. */
8968 extract_omp_for_data (stmt, &fd, NULL);
8970 lower_omp_for_lastprivate (&fd, &body, &dlist, ctx);
8972 gimple_seq_add_stmt (&body, stmt);
8973 gimple_seq_add_seq (&body, gimple_omp_body (stmt));
8975 gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
8976 fd.loop.v));
8978 /* After the loop, add exit clauses. */
8979 lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx);
8981 if (ctx->cancellable)
8982 gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
8984 gimple_seq_add_seq (&body, dlist);
8986 body = maybe_catch_exception (body);
8988 /* Region exit marker goes at the end of the loop body. */
8989 gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait));
8990 maybe_add_implicit_barrier_cancel (ctx, &body);
8991 pop_gimplify_context (new_stmt);
8993 gimple_bind_append_vars (new_stmt, ctx->block_vars);
8994 BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
8995 if (BLOCK_VARS (block))
8996 TREE_USED (block) = 1;
8998 gimple_bind_set_body (new_stmt, body);
8999 gimple_omp_set_body (stmt, NULL);
9000 gimple_omp_for_set_pre_body (stmt, NULL);
9003 /* Callback for walk_stmts. Check if the current statement only contains
9004 GIMPLE_OMP_FOR or GIMPLE_OMP_PARALLEL. */
9006 static tree
9007 check_combined_parallel (gimple_stmt_iterator *gsi_p,
9008 bool *handled_ops_p,
9009 struct walk_stmt_info *wi)
9011 int *info = (int *) wi->info;
9012 gimple stmt = gsi_stmt (*gsi_p);
9014 *handled_ops_p = true;
9015 switch (gimple_code (stmt))
9017 WALK_SUBSTMTS;
9019 case GIMPLE_OMP_FOR:
9020 case GIMPLE_OMP_SECTIONS:
9021 *info = *info == 0 ? 1 : -1;
9022 break;
9023 default:
9024 *info = -1;
9025 break;
9027 return NULL;
9030 struct omp_taskcopy_context
9032 /* This field must be at the beginning, as we do "inheritance": Some
9033 callback functions for tree-inline.c (e.g., omp_copy_decl)
9034 receive a copy_body_data pointer that is up-casted to an
9035 omp_context pointer. */
9036 copy_body_data cb;
9037 omp_context *ctx;
9040 static tree
9041 task_copyfn_copy_decl (tree var, copy_body_data *cb)
9043 struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
9045 if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
9046 return create_tmp_var (TREE_TYPE (var), NULL);
9048 return var;
9051 static tree
9052 task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
9054 tree name, new_fields = NULL, type, f;
9056 type = lang_hooks.types.make_type (RECORD_TYPE);
9057 name = DECL_NAME (TYPE_NAME (orig_type));
9058 name = build_decl (gimple_location (tcctx->ctx->stmt),
9059 TYPE_DECL, name, type);
9060 TYPE_NAME (type) = name;
9062 for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
9064 tree new_f = copy_node (f);
9065 DECL_CONTEXT (new_f) = type;
9066 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
9067 TREE_CHAIN (new_f) = new_fields;
9068 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
9069 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
9070 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
9071 &tcctx->cb, NULL);
9072 new_fields = new_f;
9073 *pointer_map_insert (tcctx->cb.decl_map, f) = new_f;
9075 TYPE_FIELDS (type) = nreverse (new_fields);
9076 layout_type (type);
9077 return type;
9080 /* Create task copyfn. */
9082 static void
9083 create_task_copyfn (gimple task_stmt, omp_context *ctx)
9085 struct function *child_cfun;
9086 tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
9087 tree record_type, srecord_type, bind, list;
9088 bool record_needs_remap = false, srecord_needs_remap = false;
9089 splay_tree_node n;
9090 struct omp_taskcopy_context tcctx;
9091 location_t loc = gimple_location (task_stmt);
9093 child_fn = gimple_omp_task_copy_fn (task_stmt);
9094 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
9095 gcc_assert (child_cfun->cfg == NULL);
9096 DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
9098 /* Reset DECL_CONTEXT on function arguments. */
9099 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
9100 DECL_CONTEXT (t) = child_fn;
9102 /* Populate the function. */
9103 push_gimplify_context ();
9104 push_cfun (child_cfun);
9106 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
9107 TREE_SIDE_EFFECTS (bind) = 1;
9108 list = NULL;
9109 DECL_SAVED_TREE (child_fn) = bind;
9110 DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
9112 /* Remap src and dst argument types if needed. */
9113 record_type = ctx->record_type;
9114 srecord_type = ctx->srecord_type;
9115 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
9116 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
9118 record_needs_remap = true;
9119 break;
9121 for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
9122 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
9124 srecord_needs_remap = true;
9125 break;
9128 if (record_needs_remap || srecord_needs_remap)
9130 memset (&tcctx, '\0', sizeof (tcctx));
9131 tcctx.cb.src_fn = ctx->cb.src_fn;
9132 tcctx.cb.dst_fn = child_fn;
9133 tcctx.cb.src_node = cgraph_get_node (tcctx.cb.src_fn);
9134 gcc_checking_assert (tcctx.cb.src_node);
9135 tcctx.cb.dst_node = tcctx.cb.src_node;
9136 tcctx.cb.src_cfun = ctx->cb.src_cfun;
9137 tcctx.cb.copy_decl = task_copyfn_copy_decl;
9138 tcctx.cb.eh_lp_nr = 0;
9139 tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
9140 tcctx.cb.decl_map = pointer_map_create ();
9141 tcctx.ctx = ctx;
9143 if (record_needs_remap)
9144 record_type = task_copyfn_remap_type (&tcctx, record_type);
9145 if (srecord_needs_remap)
9146 srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
9148 else
9149 tcctx.cb.decl_map = NULL;
9151 arg = DECL_ARGUMENTS (child_fn);
9152 TREE_TYPE (arg) = build_pointer_type (record_type);
9153 sarg = DECL_CHAIN (arg);
9154 TREE_TYPE (sarg) = build_pointer_type (srecord_type);
9156 /* First pass: initialize temporaries used in record_type and srecord_type
9157 sizes and field offsets. */
9158 if (tcctx.cb.decl_map)
9159 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
9160 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
9162 tree *p;
9164 decl = OMP_CLAUSE_DECL (c);
9165 p = (tree *) pointer_map_contains (tcctx.cb.decl_map, decl);
9166 if (p == NULL)
9167 continue;
9168 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9169 sf = (tree) n->value;
9170 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9171 src = build_simple_mem_ref_loc (loc, sarg);
9172 src = omp_build_component_ref (src, sf);
9173 t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
9174 append_to_statement_list (t, &list);
9177 /* Second pass: copy shared var pointers and copy construct non-VLA
9178 firstprivate vars. */
9179 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
9180 switch (OMP_CLAUSE_CODE (c))
9182 case OMP_CLAUSE_SHARED:
9183 decl = OMP_CLAUSE_DECL (c);
9184 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9185 if (n == NULL)
9186 break;
9187 f = (tree) n->value;
9188 if (tcctx.cb.decl_map)
9189 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9190 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9191 sf = (tree) n->value;
9192 if (tcctx.cb.decl_map)
9193 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9194 src = build_simple_mem_ref_loc (loc, sarg);
9195 src = omp_build_component_ref (src, sf);
9196 dst = build_simple_mem_ref_loc (loc, arg);
9197 dst = omp_build_component_ref (dst, f);
9198 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
9199 append_to_statement_list (t, &list);
9200 break;
9201 case OMP_CLAUSE_FIRSTPRIVATE:
9202 decl = OMP_CLAUSE_DECL (c);
9203 if (is_variable_sized (decl))
9204 break;
9205 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9206 if (n == NULL)
9207 break;
9208 f = (tree) n->value;
9209 if (tcctx.cb.decl_map)
9210 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9211 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9212 if (n != NULL)
9214 sf = (tree) n->value;
9215 if (tcctx.cb.decl_map)
9216 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9217 src = build_simple_mem_ref_loc (loc, sarg);
9218 src = omp_build_component_ref (src, sf);
9219 if (use_pointer_for_field (decl, NULL) || is_reference (decl))
9220 src = build_simple_mem_ref_loc (loc, src);
9222 else
9223 src = decl;
9224 dst = build_simple_mem_ref_loc (loc, arg);
9225 dst = omp_build_component_ref (dst, f);
9226 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
9227 append_to_statement_list (t, &list);
9228 break;
9229 case OMP_CLAUSE_PRIVATE:
9230 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
9231 break;
9232 decl = OMP_CLAUSE_DECL (c);
9233 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9234 f = (tree) n->value;
9235 if (tcctx.cb.decl_map)
9236 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9237 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
9238 if (n != NULL)
9240 sf = (tree) n->value;
9241 if (tcctx.cb.decl_map)
9242 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9243 src = build_simple_mem_ref_loc (loc, sarg);
9244 src = omp_build_component_ref (src, sf);
9245 if (use_pointer_for_field (decl, NULL))
9246 src = build_simple_mem_ref_loc (loc, src);
9248 else
9249 src = decl;
9250 dst = build_simple_mem_ref_loc (loc, arg);
9251 dst = omp_build_component_ref (dst, f);
9252 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
9253 append_to_statement_list (t, &list);
9254 break;
9255 default:
9256 break;
9259 /* Last pass: handle VLA firstprivates. */
9260 if (tcctx.cb.decl_map)
9261 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
9262 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
9264 tree ind, ptr, df;
9266 decl = OMP_CLAUSE_DECL (c);
9267 if (!is_variable_sized (decl))
9268 continue;
9269 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
9270 if (n == NULL)
9271 continue;
9272 f = (tree) n->value;
9273 f = *(tree *) pointer_map_contains (tcctx.cb.decl_map, f);
9274 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
9275 ind = DECL_VALUE_EXPR (decl);
9276 gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
9277 gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
9278 n = splay_tree_lookup (ctx->sfield_map,
9279 (splay_tree_key) TREE_OPERAND (ind, 0));
9280 sf = (tree) n->value;
9281 sf = *(tree *) pointer_map_contains (tcctx.cb.decl_map, sf);
9282 src = build_simple_mem_ref_loc (loc, sarg);
9283 src = omp_build_component_ref (src, sf);
9284 src = build_simple_mem_ref_loc (loc, src);
9285 dst = build_simple_mem_ref_loc (loc, arg);
9286 dst = omp_build_component_ref (dst, f);
9287 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
9288 append_to_statement_list (t, &list);
9289 n = splay_tree_lookup (ctx->field_map,
9290 (splay_tree_key) TREE_OPERAND (ind, 0));
9291 df = (tree) n->value;
9292 df = *(tree *) pointer_map_contains (tcctx.cb.decl_map, df);
9293 ptr = build_simple_mem_ref_loc (loc, arg);
9294 ptr = omp_build_component_ref (ptr, df);
9295 t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
9296 build_fold_addr_expr_loc (loc, dst));
9297 append_to_statement_list (t, &list);
9300 t = build1 (RETURN_EXPR, void_type_node, NULL);
9301 append_to_statement_list (t, &list);
9303 if (tcctx.cb.decl_map)
9304 pointer_map_destroy (tcctx.cb.decl_map);
9305 pop_gimplify_context (NULL);
9306 BIND_EXPR_BODY (bind) = list;
9307 pop_cfun ();
9310 static void
9311 lower_depend_clauses (gimple stmt, gimple_seq *iseq, gimple_seq *oseq)
9313 tree c, clauses;
9314 gimple g;
9315 size_t n_in = 0, n_out = 0, idx = 2, i;
9317 clauses = find_omp_clause (gimple_omp_task_clauses (stmt),
9318 OMP_CLAUSE_DEPEND);
9319 gcc_assert (clauses);
9320 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9321 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
9322 switch (OMP_CLAUSE_DEPEND_KIND (c))
9324 case OMP_CLAUSE_DEPEND_IN:
9325 n_in++;
9326 break;
9327 case OMP_CLAUSE_DEPEND_OUT:
9328 case OMP_CLAUSE_DEPEND_INOUT:
9329 n_out++;
9330 break;
9331 default:
9332 gcc_unreachable ();
9334 tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2);
9335 tree array = create_tmp_var (type, NULL);
9336 tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
9337 NULL_TREE);
9338 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out));
9339 gimple_seq_add_stmt (iseq, g);
9340 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
9341 NULL_TREE);
9342 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out));
9343 gimple_seq_add_stmt (iseq, g);
9344 for (i = 0; i < 2; i++)
9346 if ((i ? n_in : n_out) == 0)
9347 continue;
9348 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
9349 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
9350 && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i))
9352 tree t = OMP_CLAUSE_DECL (c);
9353 t = fold_convert (ptr_type_node, t);
9354 gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
9355 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
9356 NULL_TREE, NULL_TREE);
9357 g = gimple_build_assign (r, t);
9358 gimple_seq_add_stmt (iseq, g);
9361 tree *p = gimple_omp_task_clauses_ptr (stmt);
9362 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
9363 OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
9364 OMP_CLAUSE_CHAIN (c) = *p;
9365 *p = c;
9366 tree clobber = build_constructor (type, NULL);
9367 TREE_THIS_VOLATILE (clobber) = 1;
9368 g = gimple_build_assign (array, clobber);
9369 gimple_seq_add_stmt (oseq, g);
9372 /* Lower the OpenMP parallel or task directive in the current statement
9373 in GSI_P. CTX holds context information for the directive. */
9375 static void
9376 lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9378 tree clauses;
9379 tree child_fn, t;
9380 gimple stmt = gsi_stmt (*gsi_p);
9381 gimple par_bind, bind, dep_bind = NULL;
9382 gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body;
9383 location_t loc = gimple_location (stmt);
9385 clauses = gimple_omp_taskreg_clauses (stmt);
9386 par_bind = gimple_seq_first_stmt (gimple_omp_body (stmt));
9387 par_body = gimple_bind_body (par_bind);
9388 child_fn = ctx->cb.dst_fn;
9389 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
9390 && !gimple_omp_parallel_combined_p (stmt))
9392 struct walk_stmt_info wi;
9393 int ws_num = 0;
9395 memset (&wi, 0, sizeof (wi));
9396 wi.info = &ws_num;
9397 wi.val_only = true;
9398 walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
9399 if (ws_num == 1)
9400 gimple_omp_parallel_set_combined_p (stmt, true);
9402 gimple_seq dep_ilist = NULL;
9403 gimple_seq dep_olist = NULL;
9404 if (gimple_code (stmt) == GIMPLE_OMP_TASK
9405 && find_omp_clause (clauses, OMP_CLAUSE_DEPEND))
9407 push_gimplify_context ();
9408 dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
9409 lower_depend_clauses (stmt, &dep_ilist, &dep_olist);
9412 if (ctx->srecord_type)
9413 create_task_copyfn (stmt, ctx);
9415 push_gimplify_context ();
9417 par_olist = NULL;
9418 par_ilist = NULL;
9419 par_rlist = NULL;
9420 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
9421 lower_omp (&par_body, ctx);
9422 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL)
9423 lower_reduction_clauses (clauses, &par_rlist, ctx);
9425 /* Declare all the variables created by mapping and the variables
9426 declared in the scope of the parallel body. */
9427 record_vars_into (ctx->block_vars, child_fn);
9428 record_vars_into (gimple_bind_vars (par_bind), child_fn);
9430 if (ctx->record_type)
9432 ctx->sender_decl
9433 = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
9434 : ctx->record_type, ".omp_data_o");
9435 DECL_NAMELESS (ctx->sender_decl) = 1;
9436 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
9437 gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
9440 olist = NULL;
9441 ilist = NULL;
9442 lower_send_clauses (clauses, &ilist, &olist, ctx);
9443 lower_send_shared_vars (&ilist, &olist, ctx);
9445 if (ctx->record_type)
9447 tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL);
9448 TREE_THIS_VOLATILE (clobber) = 1;
9449 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
9450 clobber));
9453 /* Once all the expansions are done, sequence all the different
9454 fragments inside gimple_omp_body. */
9456 new_body = NULL;
9458 if (ctx->record_type)
9460 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
9461 /* fixup_child_record_type might have changed receiver_decl's type. */
9462 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
9463 gimple_seq_add_stmt (&new_body,
9464 gimple_build_assign (ctx->receiver_decl, t));
9467 gimple_seq_add_seq (&new_body, par_ilist);
9468 gimple_seq_add_seq (&new_body, par_body);
9469 gimple_seq_add_seq (&new_body, par_rlist);
9470 if (ctx->cancellable)
9471 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
9472 gimple_seq_add_seq (&new_body, par_olist);
9473 new_body = maybe_catch_exception (new_body);
9474 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
9475 gimple_omp_set_body (stmt, new_body);
9477 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
9478 gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
9479 gimple_bind_add_seq (bind, ilist);
9480 gimple_bind_add_stmt (bind, stmt);
9481 gimple_bind_add_seq (bind, olist);
9483 pop_gimplify_context (NULL);
9485 if (dep_bind)
9487 gimple_bind_add_seq (dep_bind, dep_ilist);
9488 gimple_bind_add_stmt (dep_bind, bind);
9489 gimple_bind_add_seq (dep_bind, dep_olist);
9490 pop_gimplify_context (dep_bind);
9494 /* Lower the OpenMP target directive in the current statement
9495 in GSI_P. CTX holds context information for the directive. */
9497 static void
9498 lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9500 tree clauses;
9501 tree child_fn, t, c;
9502 gimple stmt = gsi_stmt (*gsi_p);
9503 gimple tgt_bind = NULL, bind;
9504 gimple_seq tgt_body = NULL, olist, ilist, new_body;
9505 location_t loc = gimple_location (stmt);
9506 int kind = gimple_omp_target_kind (stmt);
9507 unsigned int map_cnt = 0;
9509 clauses = gimple_omp_target_clauses (stmt);
9510 if (kind == GF_OMP_TARGET_KIND_REGION)
9512 tgt_bind = gimple_seq_first_stmt (gimple_omp_body (stmt));
9513 tgt_body = gimple_bind_body (tgt_bind);
9515 else if (kind == GF_OMP_TARGET_KIND_DATA)
9516 tgt_body = gimple_omp_body (stmt);
9517 child_fn = ctx->cb.dst_fn;
9519 push_gimplify_context ();
9521 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
9522 switch (OMP_CLAUSE_CODE (c))
9524 tree var, x;
9526 default:
9527 break;
9528 case OMP_CLAUSE_MAP:
9529 case OMP_CLAUSE_TO:
9530 case OMP_CLAUSE_FROM:
9531 var = OMP_CLAUSE_DECL (c);
9532 if (!DECL_P (var))
9534 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
9535 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
9536 map_cnt++;
9537 continue;
9540 if (DECL_SIZE (var)
9541 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
9543 tree var2 = DECL_VALUE_EXPR (var);
9544 gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
9545 var2 = TREE_OPERAND (var2, 0);
9546 gcc_assert (DECL_P (var2));
9547 var = var2;
9550 if (!maybe_lookup_field (var, ctx))
9551 continue;
9553 if (kind == GF_OMP_TARGET_KIND_REGION)
9555 x = build_receiver_ref (var, true, ctx);
9556 tree new_var = lookup_decl (var, ctx);
9557 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
9558 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
9559 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
9560 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
9561 x = build_simple_mem_ref (x);
9562 SET_DECL_VALUE_EXPR (new_var, x);
9563 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
9565 map_cnt++;
9568 if (kind == GF_OMP_TARGET_KIND_REGION)
9570 target_nesting_level++;
9571 lower_omp (&tgt_body, ctx);
9572 target_nesting_level--;
9574 else if (kind == GF_OMP_TARGET_KIND_DATA)
9575 lower_omp (&tgt_body, ctx);
9577 if (kind == GF_OMP_TARGET_KIND_REGION)
9579 /* Declare all the variables created by mapping and the variables
9580 declared in the scope of the target body. */
9581 record_vars_into (ctx->block_vars, child_fn);
9582 record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
9585 olist = NULL;
9586 ilist = NULL;
9587 if (ctx->record_type)
9589 ctx->sender_decl
9590 = create_tmp_var (ctx->record_type, ".omp_data_arr");
9591 DECL_NAMELESS (ctx->sender_decl) = 1;
9592 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
9593 t = make_tree_vec (3);
9594 TREE_VEC_ELT (t, 0) = ctx->sender_decl;
9595 TREE_VEC_ELT (t, 1)
9596 = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt),
9597 ".omp_data_sizes");
9598 DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
9599 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
9600 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
9601 TREE_VEC_ELT (t, 2)
9602 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node,
9603 map_cnt),
9604 ".omp_data_kinds");
9605 DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
9606 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
9607 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
9608 gimple_omp_target_set_data_arg (stmt, t);
9610 vec<constructor_elt, va_gc> *vsize;
9611 vec<constructor_elt, va_gc> *vkind;
9612 vec_alloc (vsize, map_cnt);
9613 vec_alloc (vkind, map_cnt);
9614 unsigned int map_idx = 0;
9616 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
9617 switch (OMP_CLAUSE_CODE (c))
9619 tree ovar, nc;
9621 default:
9622 break;
9623 case OMP_CLAUSE_MAP:
9624 case OMP_CLAUSE_TO:
9625 case OMP_CLAUSE_FROM:
9626 nc = c;
9627 ovar = OMP_CLAUSE_DECL (c);
9628 if (!DECL_P (ovar))
9630 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
9631 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
9633 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c))
9634 == get_base_address (ovar));
9635 nc = OMP_CLAUSE_CHAIN (c);
9636 ovar = OMP_CLAUSE_DECL (nc);
9638 else
9640 tree x = build_sender_ref (ovar, ctx);
9641 tree v
9642 = build_fold_addr_expr_with_type (ovar, ptr_type_node);
9643 gimplify_assign (x, v, &ilist);
9644 nc = NULL_TREE;
9647 else
9649 if (DECL_SIZE (ovar)
9650 && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST)
9652 tree ovar2 = DECL_VALUE_EXPR (ovar);
9653 gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
9654 ovar2 = TREE_OPERAND (ovar2, 0);
9655 gcc_assert (DECL_P (ovar2));
9656 ovar = ovar2;
9658 if (!maybe_lookup_field (ovar, ctx))
9659 continue;
9662 if (nc)
9664 tree var = lookup_decl_in_outer_ctx (ovar, ctx);
9665 tree x = build_sender_ref (ovar, ctx);
9666 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
9667 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
9668 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
9669 && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
9671 gcc_assert (kind == GF_OMP_TARGET_KIND_REGION);
9672 tree avar
9673 = create_tmp_var (TREE_TYPE (TREE_TYPE (x)), NULL);
9674 mark_addressable (avar);
9675 gimplify_assign (avar, build_fold_addr_expr (var), &ilist);
9676 avar = build_fold_addr_expr (avar);
9677 gimplify_assign (x, avar, &ilist);
9679 else if (is_gimple_reg (var))
9681 gcc_assert (kind == GF_OMP_TARGET_KIND_REGION);
9682 tree avar = create_tmp_var (TREE_TYPE (var), NULL);
9683 mark_addressable (avar);
9684 if (OMP_CLAUSE_MAP_KIND (c) != OMP_CLAUSE_MAP_ALLOC
9685 && OMP_CLAUSE_MAP_KIND (c) != OMP_CLAUSE_MAP_FROM)
9686 gimplify_assign (avar, var, &ilist);
9687 avar = build_fold_addr_expr (avar);
9688 gimplify_assign (x, avar, &ilist);
9689 if ((OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_FROM
9690 || OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_TOFROM)
9691 && !TYPE_READONLY (TREE_TYPE (var)))
9693 x = build_sender_ref (ovar, ctx);
9694 x = build_simple_mem_ref (x);
9695 gimplify_assign (var, x, &olist);
9698 else
9700 var = build_fold_addr_expr (var);
9701 gimplify_assign (x, var, &ilist);
9704 tree s = OMP_CLAUSE_SIZE (c);
9705 if (s == NULL_TREE)
9706 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
9707 s = fold_convert (size_type_node, s);
9708 tree purpose = size_int (map_idx++);
9709 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
9710 if (TREE_CODE (s) != INTEGER_CST)
9711 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
9713 unsigned char tkind = 0;
9714 switch (OMP_CLAUSE_CODE (c))
9716 case OMP_CLAUSE_MAP:
9717 tkind = OMP_CLAUSE_MAP_KIND (c);
9718 break;
9719 case OMP_CLAUSE_TO:
9720 tkind = OMP_CLAUSE_MAP_TO;
9721 break;
9722 case OMP_CLAUSE_FROM:
9723 tkind = OMP_CLAUSE_MAP_FROM;
9724 break;
9725 default:
9726 gcc_unreachable ();
9728 unsigned int talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
9729 if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
9730 talign = DECL_ALIGN_UNIT (ovar);
9731 talign = ceil_log2 (talign);
9732 tkind |= talign << 3;
9733 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
9734 build_int_cst (unsigned_char_type_node,
9735 tkind));
9736 if (nc && nc != c)
9737 c = nc;
9740 gcc_assert (map_idx == map_cnt);
9742 DECL_INITIAL (TREE_VEC_ELT (t, 1))
9743 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
9744 DECL_INITIAL (TREE_VEC_ELT (t, 2))
9745 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
9746 if (!TREE_STATIC (TREE_VEC_ELT (t, 1)))
9748 gimple_seq initlist = NULL;
9749 force_gimple_operand (build1 (DECL_EXPR, void_type_node,
9750 TREE_VEC_ELT (t, 1)),
9751 &initlist, true, NULL_TREE);
9752 gimple_seq_add_seq (&ilist, initlist);
9755 tree clobber = build_constructor (ctx->record_type, NULL);
9756 TREE_THIS_VOLATILE (clobber) = 1;
9757 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
9758 clobber));
9761 /* Once all the expansions are done, sequence all the different
9762 fragments inside gimple_omp_body. */
9764 new_body = NULL;
9766 if (ctx->record_type && kind == GF_OMP_TARGET_KIND_REGION)
9768 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
9769 /* fixup_child_record_type might have changed receiver_decl's type. */
9770 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
9771 gimple_seq_add_stmt (&new_body,
9772 gimple_build_assign (ctx->receiver_decl, t));
9775 if (kind == GF_OMP_TARGET_KIND_REGION)
9777 gimple_seq_add_seq (&new_body, tgt_body);
9778 new_body = maybe_catch_exception (new_body);
9780 else if (kind == GF_OMP_TARGET_KIND_DATA)
9781 new_body = tgt_body;
9782 if (kind != GF_OMP_TARGET_KIND_UPDATE)
9784 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
9785 gimple_omp_set_body (stmt, new_body);
9788 bind = gimple_build_bind (NULL, NULL,
9789 tgt_bind ? gimple_bind_block (tgt_bind)
9790 : NULL_TREE);
9791 gsi_replace (gsi_p, bind, true);
9792 gimple_bind_add_seq (bind, ilist);
9793 gimple_bind_add_stmt (bind, stmt);
9794 gimple_bind_add_seq (bind, olist);
9796 pop_gimplify_context (NULL);
9799 /* Expand code for an OpenMP teams directive. */
9801 static void
9802 lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9804 gimple teams_stmt = gsi_stmt (*gsi_p);
9805 push_gimplify_context ();
9807 tree block = make_node (BLOCK);
9808 gimple bind = gimple_build_bind (NULL, NULL, block);
9809 gsi_replace (gsi_p, bind, true);
9810 gimple_seq bind_body = NULL;
9811 gimple_seq dlist = NULL;
9812 gimple_seq olist = NULL;
9814 tree num_teams = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
9815 OMP_CLAUSE_NUM_TEAMS);
9816 if (num_teams == NULL_TREE)
9817 num_teams = build_int_cst (unsigned_type_node, 0);
9818 else
9820 num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams);
9821 num_teams = fold_convert (unsigned_type_node, num_teams);
9822 gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
9824 tree thread_limit = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
9825 OMP_CLAUSE_THREAD_LIMIT);
9826 if (thread_limit == NULL_TREE)
9827 thread_limit = build_int_cst (unsigned_type_node, 0);
9828 else
9830 thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
9831 thread_limit = fold_convert (unsigned_type_node, thread_limit);
9832 gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
9833 fb_rvalue);
9836 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
9837 &bind_body, &dlist, ctx, NULL);
9838 lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
9839 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx);
9840 gimple_seq_add_stmt (&bind_body, teams_stmt);
9842 location_t loc = gimple_location (teams_stmt);
9843 tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS);
9844 gimple call = gimple_build_call (decl, 2, num_teams, thread_limit);
9845 gimple_set_location (call, loc);
9846 gimple_seq_add_stmt (&bind_body, call);
9848 gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
9849 gimple_omp_set_body (teams_stmt, NULL);
9850 gimple_seq_add_seq (&bind_body, olist);
9851 gimple_seq_add_seq (&bind_body, dlist);
9852 gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
9853 gimple_bind_set_body (bind, bind_body);
9855 pop_gimplify_context (bind);
9857 gimple_bind_append_vars (bind, ctx->block_vars);
9858 BLOCK_VARS (block) = ctx->block_vars;
9859 if (BLOCK_VARS (block))
9860 TREE_USED (block) = 1;
9864 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
9865 regimplified. If DATA is non-NULL, lower_omp_1 is outside
9866 of OpenMP context, but with task_shared_vars set. */
9868 static tree
9869 lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
9870 void *data)
9872 tree t = *tp;
9874 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
9875 if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t))
9876 return t;
9878 if (task_shared_vars
9879 && DECL_P (t)
9880 && bitmap_bit_p (task_shared_vars, DECL_UID (t)))
9881 return t;
9883 /* If a global variable has been privatized, TREE_CONSTANT on
9884 ADDR_EXPR might be wrong. */
9885 if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
9886 recompute_tree_invariant_for_addr_expr (t);
9888 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
9889 return NULL_TREE;
9892 static void
9893 lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
9895 gimple stmt = gsi_stmt (*gsi_p);
9896 struct walk_stmt_info wi;
9898 if (gimple_has_location (stmt))
9899 input_location = gimple_location (stmt);
9901 if (task_shared_vars)
9902 memset (&wi, '\0', sizeof (wi));
9904 /* If we have issued syntax errors, avoid doing any heavy lifting.
9905 Just replace the OpenMP directives with a NOP to avoid
9906 confusing RTL expansion. */
9907 if (seen_error () && is_gimple_omp (stmt))
9909 gsi_replace (gsi_p, gimple_build_nop (), true);
9910 return;
9913 switch (gimple_code (stmt))
9915 case GIMPLE_COND:
9916 if ((ctx || task_shared_vars)
9917 && (walk_tree (gimple_cond_lhs_ptr (stmt), lower_omp_regimplify_p,
9918 ctx ? NULL : &wi, NULL)
9919 || walk_tree (gimple_cond_rhs_ptr (stmt), lower_omp_regimplify_p,
9920 ctx ? NULL : &wi, NULL)))
9921 gimple_regimplify_operands (stmt, gsi_p);
9922 break;
9923 case GIMPLE_CATCH:
9924 lower_omp (gimple_catch_handler_ptr (stmt), ctx);
9925 break;
9926 case GIMPLE_EH_FILTER:
9927 lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
9928 break;
9929 case GIMPLE_TRY:
9930 lower_omp (gimple_try_eval_ptr (stmt), ctx);
9931 lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
9932 break;
9933 case GIMPLE_TRANSACTION:
9934 lower_omp (gimple_transaction_body_ptr (stmt), ctx);
9935 break;
9936 case GIMPLE_BIND:
9937 lower_omp (gimple_bind_body_ptr (stmt), ctx);
9938 break;
9939 case GIMPLE_OMP_PARALLEL:
9940 case GIMPLE_OMP_TASK:
9941 ctx = maybe_lookup_ctx (stmt);
9942 gcc_assert (ctx);
9943 if (ctx->cancellable)
9944 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
9945 lower_omp_taskreg (gsi_p, ctx);
9946 break;
9947 case GIMPLE_OMP_FOR:
9948 ctx = maybe_lookup_ctx (stmt);
9949 gcc_assert (ctx);
9950 if (ctx->cancellable)
9951 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
9952 lower_omp_for (gsi_p, ctx);
9953 break;
9954 case GIMPLE_OMP_SECTIONS:
9955 ctx = maybe_lookup_ctx (stmt);
9956 gcc_assert (ctx);
9957 if (ctx->cancellable)
9958 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
9959 lower_omp_sections (gsi_p, ctx);
9960 break;
9961 case GIMPLE_OMP_SINGLE:
9962 ctx = maybe_lookup_ctx (stmt);
9963 gcc_assert (ctx);
9964 lower_omp_single (gsi_p, ctx);
9965 break;
9966 case GIMPLE_OMP_MASTER:
9967 ctx = maybe_lookup_ctx (stmt);
9968 gcc_assert (ctx);
9969 lower_omp_master (gsi_p, ctx);
9970 break;
9971 case GIMPLE_OMP_TASKGROUP:
9972 ctx = maybe_lookup_ctx (stmt);
9973 gcc_assert (ctx);
9974 lower_omp_taskgroup (gsi_p, ctx);
9975 break;
9976 case GIMPLE_OMP_ORDERED:
9977 ctx = maybe_lookup_ctx (stmt);
9978 gcc_assert (ctx);
9979 lower_omp_ordered (gsi_p, ctx);
9980 break;
9981 case GIMPLE_OMP_CRITICAL:
9982 ctx = maybe_lookup_ctx (stmt);
9983 gcc_assert (ctx);
9984 lower_omp_critical (gsi_p, ctx);
9985 break;
9986 case GIMPLE_OMP_ATOMIC_LOAD:
9987 if ((ctx || task_shared_vars)
9988 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt),
9989 lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
9990 gimple_regimplify_operands (stmt, gsi_p);
9991 break;
9992 case GIMPLE_OMP_TARGET:
9993 ctx = maybe_lookup_ctx (stmt);
9994 gcc_assert (ctx);
9995 lower_omp_target (gsi_p, ctx);
9996 break;
9997 case GIMPLE_OMP_TEAMS:
9998 ctx = maybe_lookup_ctx (stmt);
9999 gcc_assert (ctx);
10000 lower_omp_teams (gsi_p, ctx);
10001 break;
10002 case GIMPLE_CALL:
10003 tree fndecl;
10004 fndecl = gimple_call_fndecl (stmt);
10005 if (fndecl
10006 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
10007 switch (DECL_FUNCTION_CODE (fndecl))
10009 case BUILT_IN_GOMP_BARRIER:
10010 if (ctx == NULL)
10011 break;
10012 /* FALLTHRU */
10013 case BUILT_IN_GOMP_CANCEL:
10014 case BUILT_IN_GOMP_CANCELLATION_POINT:
10015 omp_context *cctx;
10016 cctx = ctx;
10017 if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
10018 cctx = cctx->outer;
10019 gcc_assert (gimple_call_lhs (stmt) == NULL_TREE);
10020 if (!cctx->cancellable)
10022 if (DECL_FUNCTION_CODE (fndecl)
10023 == BUILT_IN_GOMP_CANCELLATION_POINT)
10025 stmt = gimple_build_nop ();
10026 gsi_replace (gsi_p, stmt, false);
10028 break;
10030 tree lhs;
10031 lhs = create_tmp_var (boolean_type_node, NULL);
10032 if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
10034 fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
10035 gimple_call_set_fndecl (stmt, fndecl);
10036 gimple_call_set_fntype (stmt, TREE_TYPE (fndecl));
10038 gimple_call_set_lhs (stmt, lhs);
10039 tree fallthru_label;
10040 fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
10041 gimple g;
10042 g = gimple_build_label (fallthru_label);
10043 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
10044 g = gimple_build_cond (NE_EXPR, lhs, boolean_false_node,
10045 cctx->cancel_label, fallthru_label);
10046 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
10047 break;
10048 default:
10049 break;
10051 /* FALLTHRU */
10052 default:
10053 if ((ctx || task_shared_vars)
10054 && walk_gimple_op (stmt, lower_omp_regimplify_p,
10055 ctx ? NULL : &wi))
10056 gimple_regimplify_operands (stmt, gsi_p);
10057 break;
10061 static void
10062 lower_omp (gimple_seq *body, omp_context *ctx)
10064 location_t saved_location = input_location;
10065 gimple_stmt_iterator gsi;
10066 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
10067 lower_omp_1 (&gsi, ctx);
10068 /* Inside target region we haven't called fold_stmt during gimplification,
10069 because it can break code by adding decl references that weren't in the
10070 source. Call fold_stmt now. */
10071 if (target_nesting_level)
10072 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
10073 fold_stmt (&gsi);
10074 input_location = saved_location;
10077 /* Main entry point. */
10079 static unsigned int
10080 execute_lower_omp (void)
10082 gimple_seq body;
10084 /* This pass always runs, to provide PROP_gimple_lomp.
10085 But there is nothing to do unless -fopenmp is given. */
10086 if (flag_openmp == 0 && flag_openmp_simd == 0 && flag_enable_cilkplus == 0)
10087 return 0;
10089 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
10090 delete_omp_context);
10092 body = gimple_body (current_function_decl);
10093 scan_omp (&body, NULL);
10094 gcc_assert (taskreg_nesting_level == 0);
10096 if (all_contexts->root)
10098 if (task_shared_vars)
10099 push_gimplify_context ();
10100 lower_omp (&body, NULL);
10101 if (task_shared_vars)
10102 pop_gimplify_context (NULL);
10105 if (all_contexts)
10107 splay_tree_delete (all_contexts);
10108 all_contexts = NULL;
10110 BITMAP_FREE (task_shared_vars);
10111 return 0;
10114 namespace {
10116 const pass_data pass_data_lower_omp =
10118 GIMPLE_PASS, /* type */
10119 "omplower", /* name */
10120 OPTGROUP_NONE, /* optinfo_flags */
10121 false, /* has_gate */
10122 true, /* has_execute */
10123 TV_NONE, /* tv_id */
10124 PROP_gimple_any, /* properties_required */
10125 PROP_gimple_lomp, /* properties_provided */
10126 0, /* properties_destroyed */
10127 0, /* todo_flags_start */
10128 0, /* todo_flags_finish */
10131 class pass_lower_omp : public gimple_opt_pass
10133 public:
10134 pass_lower_omp (gcc::context *ctxt)
10135 : gimple_opt_pass (pass_data_lower_omp, ctxt)
10138 /* opt_pass methods: */
10139 unsigned int execute () { return execute_lower_omp (); }
10141 }; // class pass_lower_omp
10143 } // anon namespace
10145 gimple_opt_pass *
10146 make_pass_lower_omp (gcc::context *ctxt)
10148 return new pass_lower_omp (ctxt);
10151 /* The following is a utility to diagnose OpenMP structured block violations.
10152 It is not part of the "omplower" pass, as that's invoked too late. It
10153 should be invoked by the respective front ends after gimplification. */
10155 static splay_tree all_labels;
10157 /* Check for mismatched contexts and generate an error if needed. Return
10158 true if an error is detected. */
10160 static bool
10161 diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
10162 gimple branch_ctx, gimple label_ctx)
10164 if (label_ctx == branch_ctx)
10165 return false;
10169 Previously we kept track of the label's entire context in diagnose_sb_[12]
10170 so we could traverse it and issue a correct "exit" or "enter" error
10171 message upon a structured block violation.
10173 We built the context by building a list with tree_cons'ing, but there is
10174 no easy counterpart in gimple tuples. It seems like far too much work
10175 for issuing exit/enter error messages. If someone really misses the
10176 distinct error message... patches welcome.
10179 #if 0
10180 /* Try to avoid confusing the user by producing and error message
10181 with correct "exit" or "enter" verbiage. We prefer "exit"
10182 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10183 if (branch_ctx == NULL)
10184 exit_p = false;
10185 else
10187 while (label_ctx)
10189 if (TREE_VALUE (label_ctx) == branch_ctx)
10191 exit_p = false;
10192 break;
10194 label_ctx = TREE_CHAIN (label_ctx);
10198 if (exit_p)
10199 error ("invalid exit from OpenMP structured block");
10200 else
10201 error ("invalid entry to OpenMP structured block");
10202 #endif
10204 bool cilkplus_block = false;
10205 if (flag_enable_cilkplus)
10207 if ((branch_ctx
10208 && gimple_code (branch_ctx) == GIMPLE_OMP_FOR
10209 && gimple_omp_for_kind (branch_ctx) == GF_OMP_FOR_KIND_CILKSIMD)
10210 || (gimple_code (label_ctx) == GIMPLE_OMP_FOR
10211 && gimple_omp_for_kind (label_ctx) == GF_OMP_FOR_KIND_CILKSIMD))
10212 cilkplus_block = true;
10215 /* If it's obvious we have an invalid entry, be specific about the error. */
10216 if (branch_ctx == NULL)
10218 if (cilkplus_block)
10219 error ("invalid entry to Cilk Plus structured block");
10220 else
10221 error ("invalid entry to OpenMP structured block");
10223 else
10225 /* Otherwise, be vague and lazy, but efficient. */
10226 if (cilkplus_block)
10227 error ("invalid branch to/from a Cilk Plus structured block");
10228 else
10229 error ("invalid branch to/from an OpenMP structured block");
10232 gsi_replace (gsi_p, gimple_build_nop (), false);
10233 return true;
10236 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10237 where each label is found. */
10239 static tree
10240 diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10241 struct walk_stmt_info *wi)
10243 gimple context = (gimple) wi->info;
10244 gimple inner_context;
10245 gimple stmt = gsi_stmt (*gsi_p);
10247 *handled_ops_p = true;
10249 switch (gimple_code (stmt))
10251 WALK_SUBSTMTS;
10253 case GIMPLE_OMP_PARALLEL:
10254 case GIMPLE_OMP_TASK:
10255 case GIMPLE_OMP_SECTIONS:
10256 case GIMPLE_OMP_SINGLE:
10257 case GIMPLE_OMP_SECTION:
10258 case GIMPLE_OMP_MASTER:
10259 case GIMPLE_OMP_ORDERED:
10260 case GIMPLE_OMP_CRITICAL:
10261 case GIMPLE_OMP_TARGET:
10262 case GIMPLE_OMP_TEAMS:
10263 case GIMPLE_OMP_TASKGROUP:
10264 /* The minimal context here is just the current OMP construct. */
10265 inner_context = stmt;
10266 wi->info = inner_context;
10267 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
10268 wi->info = context;
10269 break;
10271 case GIMPLE_OMP_FOR:
10272 inner_context = stmt;
10273 wi->info = inner_context;
10274 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10275 walk them. */
10276 walk_gimple_seq (gimple_omp_for_pre_body (stmt),
10277 diagnose_sb_1, NULL, wi);
10278 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
10279 wi->info = context;
10280 break;
10282 case GIMPLE_LABEL:
10283 splay_tree_insert (all_labels, (splay_tree_key) gimple_label_label (stmt),
10284 (splay_tree_value) context);
10285 break;
10287 default:
10288 break;
10291 return NULL_TREE;
10294 /* Pass 2: Check each branch and see if its context differs from that of
10295 the destination label's context. */
10297 static tree
10298 diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
10299 struct walk_stmt_info *wi)
10301 gimple context = (gimple) wi->info;
10302 splay_tree_node n;
10303 gimple stmt = gsi_stmt (*gsi_p);
10305 *handled_ops_p = true;
10307 switch (gimple_code (stmt))
10309 WALK_SUBSTMTS;
10311 case GIMPLE_OMP_PARALLEL:
10312 case GIMPLE_OMP_TASK:
10313 case GIMPLE_OMP_SECTIONS:
10314 case GIMPLE_OMP_SINGLE:
10315 case GIMPLE_OMP_SECTION:
10316 case GIMPLE_OMP_MASTER:
10317 case GIMPLE_OMP_ORDERED:
10318 case GIMPLE_OMP_CRITICAL:
10319 case GIMPLE_OMP_TARGET:
10320 case GIMPLE_OMP_TEAMS:
10321 case GIMPLE_OMP_TASKGROUP:
10322 wi->info = stmt;
10323 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
10324 wi->info = context;
10325 break;
10327 case GIMPLE_OMP_FOR:
10328 wi->info = stmt;
10329 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10330 walk them. */
10331 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
10332 diagnose_sb_2, NULL, wi);
10333 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
10334 wi->info = context;
10335 break;
10337 case GIMPLE_COND:
10339 tree lab = gimple_cond_true_label (stmt);
10340 if (lab)
10342 n = splay_tree_lookup (all_labels,
10343 (splay_tree_key) lab);
10344 diagnose_sb_0 (gsi_p, context,
10345 n ? (gimple) n->value : NULL);
10347 lab = gimple_cond_false_label (stmt);
10348 if (lab)
10350 n = splay_tree_lookup (all_labels,
10351 (splay_tree_key) lab);
10352 diagnose_sb_0 (gsi_p, context,
10353 n ? (gimple) n->value : NULL);
10356 break;
10358 case GIMPLE_GOTO:
10360 tree lab = gimple_goto_dest (stmt);
10361 if (TREE_CODE (lab) != LABEL_DECL)
10362 break;
10364 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
10365 diagnose_sb_0 (gsi_p, context, n ? (gimple) n->value : NULL);
10367 break;
10369 case GIMPLE_SWITCH:
10371 unsigned int i;
10372 for (i = 0; i < gimple_switch_num_labels (stmt); ++i)
10374 tree lab = CASE_LABEL (gimple_switch_label (stmt, i));
10375 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
10376 if (n && diagnose_sb_0 (gsi_p, context, (gimple) n->value))
10377 break;
10380 break;
10382 case GIMPLE_RETURN:
10383 diagnose_sb_0 (gsi_p, context, NULL);
10384 break;
10386 default:
10387 break;
10390 return NULL_TREE;
10393 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10394 codes. */
10395 bool
10396 make_gimple_omp_edges (basic_block bb, struct omp_region **region)
10398 gimple last = last_stmt (bb);
10399 enum gimple_code code = gimple_code (last);
10400 struct omp_region *cur_region = *region;
10401 bool fallthru = false;
10403 switch (code)
10405 case GIMPLE_OMP_PARALLEL:
10406 case GIMPLE_OMP_TASK:
10407 case GIMPLE_OMP_FOR:
10408 case GIMPLE_OMP_SINGLE:
10409 case GIMPLE_OMP_TEAMS:
10410 case GIMPLE_OMP_MASTER:
10411 case GIMPLE_OMP_TASKGROUP:
10412 case GIMPLE_OMP_ORDERED:
10413 case GIMPLE_OMP_CRITICAL:
10414 case GIMPLE_OMP_SECTION:
10415 cur_region = new_omp_region (bb, code, cur_region);
10416 fallthru = true;
10417 break;
10419 case GIMPLE_OMP_TARGET:
10420 cur_region = new_omp_region (bb, code, cur_region);
10421 fallthru = true;
10422 if (gimple_omp_target_kind (last) == GF_OMP_TARGET_KIND_UPDATE)
10423 cur_region = cur_region->outer;
10424 break;
10426 case GIMPLE_OMP_SECTIONS:
10427 cur_region = new_omp_region (bb, code, cur_region);
10428 fallthru = true;
10429 break;
10431 case GIMPLE_OMP_SECTIONS_SWITCH:
10432 fallthru = false;
10433 break;
10435 case GIMPLE_OMP_ATOMIC_LOAD:
10436 case GIMPLE_OMP_ATOMIC_STORE:
10437 fallthru = true;
10438 break;
10440 case GIMPLE_OMP_RETURN:
10441 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10442 somewhere other than the next block. This will be
10443 created later. */
10444 cur_region->exit = bb;
10445 fallthru = cur_region->type != GIMPLE_OMP_SECTION;
10446 cur_region = cur_region->outer;
10447 break;
10449 case GIMPLE_OMP_CONTINUE:
10450 cur_region->cont = bb;
10451 switch (cur_region->type)
10453 case GIMPLE_OMP_FOR:
10454 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10455 succs edges as abnormal to prevent splitting
10456 them. */
10457 single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL;
10458 /* Make the loopback edge. */
10459 make_edge (bb, single_succ (cur_region->entry),
10460 EDGE_ABNORMAL);
10462 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10463 corresponds to the case that the body of the loop
10464 is not executed at all. */
10465 make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL);
10466 make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL);
10467 fallthru = false;
10468 break;
10470 case GIMPLE_OMP_SECTIONS:
10471 /* Wire up the edges into and out of the nested sections. */
10473 basic_block switch_bb = single_succ (cur_region->entry);
10475 struct omp_region *i;
10476 for (i = cur_region->inner; i ; i = i->next)
10478 gcc_assert (i->type == GIMPLE_OMP_SECTION);
10479 make_edge (switch_bb, i->entry, 0);
10480 make_edge (i->exit, bb, EDGE_FALLTHRU);
10483 /* Make the loopback edge to the block with
10484 GIMPLE_OMP_SECTIONS_SWITCH. */
10485 make_edge (bb, switch_bb, 0);
10487 /* Make the edge from the switch to exit. */
10488 make_edge (switch_bb, bb->next_bb, 0);
10489 fallthru = false;
10491 break;
10493 default:
10494 gcc_unreachable ();
10496 break;
10498 default:
10499 gcc_unreachable ();
10502 if (*region != cur_region)
10503 *region = cur_region;
10505 return fallthru;
10508 static unsigned int
10509 diagnose_omp_structured_block_errors (void)
10511 struct walk_stmt_info wi;
10512 gimple_seq body = gimple_body (current_function_decl);
10514 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
10516 memset (&wi, 0, sizeof (wi));
10517 walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
10519 memset (&wi, 0, sizeof (wi));
10520 wi.want_locations = true;
10521 walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
10523 gimple_set_body (current_function_decl, body);
10525 splay_tree_delete (all_labels);
10526 all_labels = NULL;
10528 return 0;
10531 static bool
10532 gate_diagnose_omp_blocks (void)
10534 return flag_openmp || flag_enable_cilkplus;
10537 namespace {
10539 const pass_data pass_data_diagnose_omp_blocks =
10541 GIMPLE_PASS, /* type */
10542 "*diagnose_omp_blocks", /* name */
10543 OPTGROUP_NONE, /* optinfo_flags */
10544 true, /* has_gate */
10545 true, /* has_execute */
10546 TV_NONE, /* tv_id */
10547 PROP_gimple_any, /* properties_required */
10548 0, /* properties_provided */
10549 0, /* properties_destroyed */
10550 0, /* todo_flags_start */
10551 0, /* todo_flags_finish */
10554 class pass_diagnose_omp_blocks : public gimple_opt_pass
10556 public:
10557 pass_diagnose_omp_blocks (gcc::context *ctxt)
10558 : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
10561 /* opt_pass methods: */
10562 bool gate () { return gate_diagnose_omp_blocks (); }
10563 unsigned int execute () {
10564 return diagnose_omp_structured_block_errors ();
10567 }; // class pass_diagnose_omp_blocks
10569 } // anon namespace
10571 gimple_opt_pass *
10572 make_pass_diagnose_omp_blocks (gcc::context *ctxt)
10574 return new pass_diagnose_omp_blocks (ctxt);
10577 #include "gt-omp-low.h"