OpenACC cache directive maintenance.
[official-gcc.git] / gcc / omp-low.c
blob1c9d94257c6afcf5ec8f7727e30f37e96d678c91
1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
13 version.
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 for more details.
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
24 #include "config.h"
25 #include "system.h"
26 #include "coretypes.h"
27 #include "tm.h"
28 #include "tree.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
31 #include "rtl.h"
32 #include "predict.h"
33 #include "vec.h"
34 #include "hashtab.h"
35 #include "hash-set.h"
36 #include "machmode.h"
37 #include "hard-reg-set.h"
38 #include "input.h"
39 #include "function.h"
40 #include "dominance.h"
41 #include "cfg.h"
42 #include "cfganal.h"
43 #include "basic-block.h"
44 #include "tree-ssa-alias.h"
45 #include "internal-fn.h"
46 #include "gimple-fold.h"
47 #include "gimple-expr.h"
48 #include "is-a.h"
49 #include "gimple.h"
50 #include "gimplify.h"
51 #include "gimple-iterator.h"
52 #include "gimplify-me.h"
53 #include "gimple-walk.h"
54 #include "tree-iterator.h"
55 #include "tree-inline.h"
56 #include "langhooks.h"
57 #include "diagnostic-core.h"
58 #include "gimple-ssa.h"
59 #include "hash-map.h"
60 #include "plugin-api.h"
61 #include "ipa-ref.h"
62 #include "cgraph.h"
63 #include "tree-cfg.h"
64 #include "tree-phinodes.h"
65 #include "ssa-iterators.h"
66 #include "tree-ssanames.h"
67 #include "tree-into-ssa.h"
68 #include "expr.h"
69 #include "tree-dfa.h"
70 #include "tree-ssa.h"
71 #include "flags.h"
72 #include "expr.h"
73 #include "tree-pass.h"
74 #include "except.h"
75 #include "splay-tree.h"
76 #include "optabs.h"
77 #include "cfgloop.h"
78 #include "target.h"
79 #include "common/common-target.h"
80 #include "omp-low.h"
81 #include "gimple-low.h"
82 #include "tree-cfgcleanup.h"
83 #include "pretty-print.h"
84 #include "alloc-pool.h"
85 #include "ipa-prop.h"
86 #include "tree-nested.h"
87 #include "tree-eh.h"
88 #include "cilk.h"
89 #include "lto-section-names.h"
92 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
93 phases. The first phase scans the function looking for OMP statements
94 and then for variables that must be replaced to satisfy data sharing
95 clauses. The second phase expands code for the constructs, as well as
96 re-gimplifying things when variables have been replaced with complex
97 expressions.
99 Final code generation is done by pass_expand_omp. The flowgraph is
100 scanned for parallel regions which are then moved to a new
101 function, to be invoked by the thread library. */
103 /* Parallel region information. Every parallel and workshare
104 directive is enclosed between two markers, the OMP_* directive
105 and a corresponding OMP_RETURN statement. */
107 struct omp_region
109 /* The enclosing region. */
110 struct omp_region *outer;
112 /* First child region. */
113 struct omp_region *inner;
115 /* Next peer region. */
116 struct omp_region *next;
118 /* Block containing the omp directive as its last stmt. */
119 basic_block entry;
121 /* Block containing the OMP_RETURN as its last stmt. */
122 basic_block exit;
124 /* Block containing the OMP_CONTINUE as its last stmt. */
125 basic_block cont;
127 /* If this is a combined parallel+workshare region, this is a list
128 of additional arguments needed by the combined parallel+workshare
129 library call. */
130 vec<tree, va_gc> *ws_args;
132 /* The code for the omp directive of this region. */
133 enum gimple_code type;
135 /* Schedule kind, only used for OMP_FOR type regions. */
136 enum omp_clause_schedule_kind sched_kind;
138 /* True if this is a combined parallel+workshare region. */
139 bool is_combined_parallel;
142 /* Context structure. Used to store information about each parallel
143 directive in the code. */
145 typedef struct omp_context
147 /* This field must be at the beginning, as we do "inheritance": Some
148 callback functions for tree-inline.c (e.g., omp_copy_decl)
149 receive a copy_body_data pointer that is up-casted to an
150 omp_context pointer. */
151 copy_body_data cb;
153 /* The tree of contexts corresponding to the encountered constructs. */
154 struct omp_context *outer;
155 gimple stmt;
157 /* Map variables to fields in a structure that allows communication
158 between sending and receiving threads. */
159 splay_tree field_map;
160 tree record_type;
161 tree sender_decl;
162 tree receiver_decl;
164 /* These are used just by task contexts, if task firstprivate fn is
165 needed. srecord_type is used to communicate from the thread
166 that encountered the task construct to task firstprivate fn,
167 record_type is allocated by GOMP_task, initialized by task firstprivate
168 fn and passed to the task body fn. */
169 splay_tree sfield_map;
170 tree srecord_type;
172 /* A chain of variables to add to the top-level block surrounding the
173 construct. In the case of a parallel, this is in the child function. */
174 tree block_vars;
176 /* A map of reduction pointer variables. For accelerators, each
177 reduction variable is replaced with an array. Each thread, in turn,
178 is assigned to a slot on that array. */
179 splay_tree reduction_map;
181 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
182 barriers should jump to during omplower pass. */
183 tree cancel_label;
185 /* What to do with variables with implicitly determined sharing
186 attributes. */
187 enum omp_clause_default_kind default_kind;
189 /* Nesting depth of this context. Used to beautify error messages re
190 invalid gotos. The outermost ctx is depth 1, with depth 0 being
191 reserved for the main body of the function. */
192 int depth;
194 /* True if this parallel directive is nested within another. */
195 bool is_nested;
197 /* True if this construct can be cancelled. */
198 bool cancellable;
199 } omp_context;
201 /* A structure holding the elements of:
202 for (V = N1; V cond N2; V += STEP) [...] */
204 struct omp_for_data_loop
206 tree v, n1, n2, step;
207 enum tree_code cond_code;
210 /* A structure describing the main elements of a parallel loop. */
212 struct omp_for_data
214 struct omp_for_data_loop loop;
215 tree chunk_size;
216 gimple for_stmt;
217 tree pre, iter_type;
218 int collapse;
219 bool have_nowait, have_ordered;
220 enum omp_clause_schedule_kind sched_kind;
221 struct omp_for_data_loop *loops;
225 static splay_tree all_contexts;
226 static int taskreg_nesting_level;
227 static int target_nesting_level;
228 static struct omp_region *root_omp_region;
229 static bitmap task_shared_vars;
230 static vec<omp_context *> taskreg_contexts;
232 static void scan_omp (gimple_seq *, omp_context *);
233 static tree scan_omp_1_op (tree *, int *, void *);
235 #define WALK_SUBSTMTS \
236 case GIMPLE_BIND: \
237 case GIMPLE_TRY: \
238 case GIMPLE_CATCH: \
239 case GIMPLE_EH_FILTER: \
240 case GIMPLE_TRANSACTION: \
241 /* The sub-statements for these should be walked. */ \
242 *handled_ops_p = false; \
243 break;
245 /* Helper function to get the reduction array name */
246 static const char *
247 omp_get_id (tree node)
249 const char *id = IDENTIFIER_POINTER (DECL_NAME (node));
250 int len = strlen ("omp$") + strlen (id);
251 char *temp_name = (char *)alloca (len+1);
252 snprintf (temp_name, len+1, "gfc$%s", id);
253 return IDENTIFIER_POINTER(get_identifier (temp_name));
256 /* Determine the number of threads OpenACC threads used to determine the
257 size of the array of partial reductions. Currently, this is num_gangs
258 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
259 because it is independed of the device used. */
261 static tree
262 oacc_max_threads (omp_context *ctx)
264 tree nthreads, vector_length, gangs, clauses;
266 gangs = fold_convert (sizetype, integer_one_node);
267 vector_length = gangs;
269 /* The reduction clause may be nested inside a loop directive.
270 Scan for the innermost vector_length clause. */
271 for (omp_context *oc = ctx; oc; oc = oc->outer)
273 if (gimple_code (oc->stmt) != GIMPLE_OACC_PARALLEL)
274 continue;
276 clauses = gimple_oacc_parallel_clauses (oc->stmt);
278 vector_length = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH);
279 if (vector_length)
280 vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length),
281 sizetype,
282 OMP_CLAUSE_VECTOR_LENGTH_EXPR
283 (vector_length));
284 else
285 vector_length = fold_convert (sizetype, integer_one_node);
287 gangs = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS);
288 if (gangs)
289 gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (gangs), sizetype,
290 OMP_CLAUSE_NUM_GANGS_EXPR (gangs));
291 else
292 gangs = fold_convert (sizetype, integer_one_node);
294 break;
297 nthreads = fold_build2 (MULT_EXPR, sizetype, gangs, vector_length);
299 return nthreads;
302 /* Holds a decl for __OPENMP_TARGET__. */
303 static GTY(()) tree offload_symbol_decl;
305 /* Holds offload tables with decls. */
306 vec<tree, va_gc> *offload_funcs, *offload_vars;
308 /* Get the __OPENMP_TARGET__ symbol. */
309 static tree
310 get_offload_symbol_decl (void)
312 if (!offload_symbol_decl)
314 tree decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
315 get_identifier ("__OPENMP_TARGET__"),
316 ptr_type_node);
317 TREE_ADDRESSABLE (decl) = 1;
318 TREE_PUBLIC (decl) = 1;
319 DECL_EXTERNAL (decl) = 1;
320 DECL_WEAK (decl) = 1;
321 DECL_ATTRIBUTES (decl)
322 = tree_cons (get_identifier ("weak"),
323 NULL_TREE, DECL_ATTRIBUTES (decl));
324 offload_symbol_decl = decl;
326 return offload_symbol_decl;
329 /* Convenience function for calling scan_omp_1_op on tree operands. */
331 static inline tree
332 scan_omp_op (tree *tp, omp_context *ctx)
334 struct walk_stmt_info wi;
336 memset (&wi, 0, sizeof (wi));
337 wi.info = ctx;
338 wi.want_locations = true;
340 return walk_tree (tp, scan_omp_1_op, &wi, NULL);
343 static void lower_omp (gimple_seq *, omp_context *);
344 static tree lookup_decl_in_outer_ctx (tree, omp_context *);
345 static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
347 /* Find an OpenMP clause of type KIND within CLAUSES. */
349 tree
350 find_omp_clause (tree clauses, enum omp_clause_code kind)
352 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
353 if (OMP_CLAUSE_CODE (clauses) == kind)
354 return clauses;
356 return NULL_TREE;
359 /* Return true if CTX is for an omp parallel. */
361 static inline bool
362 is_parallel_ctx (omp_context *ctx)
364 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL;
368 /* Return true if CTX is for an omp task. */
370 static inline bool
371 is_task_ctx (omp_context *ctx)
373 return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
377 /* Return true if CTX is for an omp parallel or omp task. */
379 static inline bool
380 is_taskreg_ctx (omp_context *ctx)
382 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL
383 || gimple_code (ctx->stmt) == GIMPLE_OMP_TASK;
387 /* Return true if REGION is a combined parallel+workshare region. */
389 static inline bool
390 is_combined_parallel (struct omp_region *region)
392 return region->is_combined_parallel;
396 /* Extract the header elements of parallel loop FOR_STMT and store
397 them into *FD. */
399 static void
400 extract_omp_for_data (gimple for_stmt, struct omp_for_data *fd,
401 struct omp_for_data_loop *loops)
403 tree t, var, *collapse_iter, *collapse_count;
404 tree count = NULL_TREE, iter_type = long_integer_type_node;
405 struct omp_for_data_loop *loop;
406 int i;
407 struct omp_for_data_loop dummy_loop;
408 location_t loc = gimple_location (for_stmt);
409 bool simd = gimple_omp_for_kind (for_stmt) & GF_OMP_FOR_SIMD;
410 bool distribute = gimple_omp_for_kind (for_stmt)
411 == GF_OMP_FOR_KIND_DISTRIBUTE;
413 fd->for_stmt = for_stmt;
414 fd->pre = NULL;
415 fd->collapse = gimple_omp_for_collapse (for_stmt);
416 if (fd->collapse > 1)
417 fd->loops = loops;
418 else
419 fd->loops = &fd->loop;
421 fd->have_nowait = (gimple_omp_for_kind (for_stmt) != GF_OMP_FOR_KIND_FOR);
422 fd->have_ordered = false;
423 fd->sched_kind = /* TODO: OACC_LOOP */ OMP_CLAUSE_SCHEDULE_STATIC;
424 fd->chunk_size = NULL_TREE;
425 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_CILKFOR)
426 fd->sched_kind = OMP_CLAUSE_SCHEDULE_CILKFOR;
427 collapse_iter = NULL;
428 collapse_count = NULL;
430 for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t))
431 switch (OMP_CLAUSE_CODE (t))
433 case OMP_CLAUSE_NOWAIT:
434 fd->have_nowait = true;
435 break;
436 case OMP_CLAUSE_ORDERED:
437 fd->have_ordered = true;
438 break;
439 case OMP_CLAUSE_SCHEDULE:
440 gcc_assert (!distribute);
441 fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t);
442 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t);
443 break;
444 case OMP_CLAUSE_DIST_SCHEDULE:
445 gcc_assert (distribute);
446 fd->chunk_size = OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t);
447 break;
448 case OMP_CLAUSE_COLLAPSE:
449 if (fd->collapse > 1)
451 collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t);
452 collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t);
454 break;
455 default:
456 break;
459 /* FIXME: for now map schedule(auto) to schedule(static).
460 There should be analysis to determine whether all iterations
461 are approximately the same amount of work (then schedule(static)
462 is best) or if it varies (then schedule(dynamic,N) is better). */
463 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO)
465 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
466 gcc_assert (fd->chunk_size == NULL);
468 gcc_assert (fd->collapse == 1 || collapse_iter != NULL);
469 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
470 gcc_assert (fd->chunk_size == NULL);
471 else if (fd->chunk_size == NULL)
473 /* We only need to compute a default chunk size for ordered
474 static loops and dynamic loops. */
475 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
476 || fd->have_ordered)
477 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
478 ? integer_zero_node : integer_one_node;
481 for (i = 0; i < fd->collapse; i++)
483 if (fd->collapse == 1)
484 loop = &fd->loop;
485 else if (loops != NULL)
486 loop = loops + i;
487 else
488 loop = &dummy_loop;
490 loop->v = gimple_omp_for_index (for_stmt, i);
491 gcc_assert (SSA_VAR_P (loop->v));
492 gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
493 || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE);
494 var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v;
495 loop->n1 = gimple_omp_for_initial (for_stmt, i);
497 loop->cond_code = gimple_omp_for_cond (for_stmt, i);
498 loop->n2 = gimple_omp_for_final (for_stmt, i);
499 switch (loop->cond_code)
501 case LT_EXPR:
502 case GT_EXPR:
503 break;
504 case NE_EXPR:
505 gcc_assert (gimple_omp_for_kind (for_stmt)
506 == GF_OMP_FOR_KIND_CILKSIMD
507 || (gimple_omp_for_kind (for_stmt)
508 == GF_OMP_FOR_KIND_CILKFOR));
509 break;
510 case LE_EXPR:
511 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
512 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, 1);
513 else
514 loop->n2 = fold_build2_loc (loc,
515 PLUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
516 build_int_cst (TREE_TYPE (loop->n2), 1));
517 loop->cond_code = LT_EXPR;
518 break;
519 case GE_EXPR:
520 if (POINTER_TYPE_P (TREE_TYPE (loop->n2)))
521 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, -1);
522 else
523 loop->n2 = fold_build2_loc (loc,
524 MINUS_EXPR, TREE_TYPE (loop->n2), loop->n2,
525 build_int_cst (TREE_TYPE (loop->n2), 1));
526 loop->cond_code = GT_EXPR;
527 break;
528 default:
529 gcc_unreachable ();
532 t = gimple_omp_for_incr (for_stmt, i);
533 gcc_assert (TREE_OPERAND (t, 0) == var);
534 switch (TREE_CODE (t))
536 case PLUS_EXPR:
537 loop->step = TREE_OPERAND (t, 1);
538 break;
539 case POINTER_PLUS_EXPR:
540 loop->step = fold_convert (ssizetype, TREE_OPERAND (t, 1));
541 break;
542 case MINUS_EXPR:
543 loop->step = TREE_OPERAND (t, 1);
544 loop->step = fold_build1_loc (loc,
545 NEGATE_EXPR, TREE_TYPE (loop->step),
546 loop->step);
547 break;
548 default:
549 gcc_unreachable ();
552 if (simd
553 || (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
554 && !fd->have_ordered))
556 if (fd->collapse == 1)
557 iter_type = TREE_TYPE (loop->v);
558 else if (i == 0
559 || TYPE_PRECISION (iter_type)
560 < TYPE_PRECISION (TREE_TYPE (loop->v)))
561 iter_type
562 = build_nonstandard_integer_type
563 (TYPE_PRECISION (TREE_TYPE (loop->v)), 1);
565 else if (iter_type != long_long_unsigned_type_node)
567 if (POINTER_TYPE_P (TREE_TYPE (loop->v)))
568 iter_type = long_long_unsigned_type_node;
569 else if (TYPE_UNSIGNED (TREE_TYPE (loop->v))
570 && TYPE_PRECISION (TREE_TYPE (loop->v))
571 >= TYPE_PRECISION (iter_type))
573 tree n;
575 if (loop->cond_code == LT_EXPR)
576 n = fold_build2_loc (loc,
577 PLUS_EXPR, TREE_TYPE (loop->v),
578 loop->n2, loop->step);
579 else
580 n = loop->n1;
581 if (TREE_CODE (n) != INTEGER_CST
582 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n))
583 iter_type = long_long_unsigned_type_node;
585 else if (TYPE_PRECISION (TREE_TYPE (loop->v))
586 > TYPE_PRECISION (iter_type))
588 tree n1, n2;
590 if (loop->cond_code == LT_EXPR)
592 n1 = loop->n1;
593 n2 = fold_build2_loc (loc,
594 PLUS_EXPR, TREE_TYPE (loop->v),
595 loop->n2, loop->step);
597 else
599 n1 = fold_build2_loc (loc,
600 MINUS_EXPR, TREE_TYPE (loop->v),
601 loop->n2, loop->step);
602 n2 = loop->n1;
604 if (TREE_CODE (n1) != INTEGER_CST
605 || TREE_CODE (n2) != INTEGER_CST
606 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1)
607 || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type)))
608 iter_type = long_long_unsigned_type_node;
612 if (collapse_count && *collapse_count == NULL)
614 t = fold_binary (loop->cond_code, boolean_type_node,
615 fold_convert (TREE_TYPE (loop->v), loop->n1),
616 fold_convert (TREE_TYPE (loop->v), loop->n2));
617 if (t && integer_zerop (t))
618 count = build_zero_cst (long_long_unsigned_type_node);
619 else if ((i == 0 || count != NULL_TREE)
620 && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE
621 && TREE_CONSTANT (loop->n1)
622 && TREE_CONSTANT (loop->n2)
623 && TREE_CODE (loop->step) == INTEGER_CST)
625 tree itype = TREE_TYPE (loop->v);
627 if (POINTER_TYPE_P (itype))
628 itype = signed_type_for (itype);
629 t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1));
630 t = fold_build2_loc (loc,
631 PLUS_EXPR, itype,
632 fold_convert_loc (loc, itype, loop->step), t);
633 t = fold_build2_loc (loc, PLUS_EXPR, itype, t,
634 fold_convert_loc (loc, itype, loop->n2));
635 t = fold_build2_loc (loc, MINUS_EXPR, itype, t,
636 fold_convert_loc (loc, itype, loop->n1));
637 if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR)
638 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype,
639 fold_build1_loc (loc, NEGATE_EXPR, itype, t),
640 fold_build1_loc (loc, NEGATE_EXPR, itype,
641 fold_convert_loc (loc, itype,
642 loop->step)));
643 else
644 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t,
645 fold_convert_loc (loc, itype, loop->step));
646 t = fold_convert_loc (loc, long_long_unsigned_type_node, t);
647 if (count != NULL_TREE)
648 count = fold_build2_loc (loc,
649 MULT_EXPR, long_long_unsigned_type_node,
650 count, t);
651 else
652 count = t;
653 if (TREE_CODE (count) != INTEGER_CST)
654 count = NULL_TREE;
656 else if (count && !integer_zerop (count))
657 count = NULL_TREE;
661 if (count
662 && !simd
663 && (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC
664 || fd->have_ordered))
666 if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node)))
667 iter_type = long_long_unsigned_type_node;
668 else
669 iter_type = long_integer_type_node;
671 else if (collapse_iter && *collapse_iter != NULL)
672 iter_type = TREE_TYPE (*collapse_iter);
673 fd->iter_type = iter_type;
674 if (collapse_iter && *collapse_iter == NULL)
675 *collapse_iter = create_tmp_var (iter_type, ".iter");
676 if (collapse_count && *collapse_count == NULL)
678 if (count)
679 *collapse_count = fold_convert_loc (loc, iter_type, count);
680 else
681 *collapse_count = create_tmp_var (iter_type, ".count");
684 if (fd->collapse > 1)
686 fd->loop.v = *collapse_iter;
687 fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0);
688 fd->loop.n2 = *collapse_count;
689 fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1);
690 fd->loop.cond_code = LT_EXPR;
693 //TODO
694 /* For OpenACC loops, force a chunk size of one, as this avoids the default
695 scheduling where several subsequent iterations are being executed by the
696 same thread. */
697 if (gimple_omp_for_kind (for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
699 gcc_assert (fd->chunk_size == NULL_TREE);
700 fd->chunk_size = build_int_cst (TREE_TYPE (fd->loop.v), 1);
705 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
706 is the immediate dominator of PAR_ENTRY_BB, return true if there
707 are no data dependencies that would prevent expanding the parallel
708 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
710 When expanding a combined parallel+workshare region, the call to
711 the child function may need additional arguments in the case of
712 GIMPLE_OMP_FOR regions. In some cases, these arguments are
713 computed out of variables passed in from the parent to the child
714 via 'struct .omp_data_s'. For instance:
716 #pragma omp parallel for schedule (guided, i * 4)
717 for (j ...)
719 Is lowered into:
721 # BLOCK 2 (PAR_ENTRY_BB)
722 .omp_data_o.i = i;
723 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
725 # BLOCK 3 (WS_ENTRY_BB)
726 .omp_data_i = &.omp_data_o;
727 D.1667 = .omp_data_i->i;
728 D.1598 = D.1667 * 4;
729 #pragma omp for schedule (guided, D.1598)
731 When we outline the parallel region, the call to the child function
732 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
733 that value is computed *after* the call site. So, in principle we
734 cannot do the transformation.
736 To see whether the code in WS_ENTRY_BB blocks the combined
737 parallel+workshare call, we collect all the variables used in the
738 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
739 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
740 call.
742 FIXME. If we had the SSA form built at this point, we could merely
743 hoist the code in block 3 into block 2 and be done with it. But at
744 this point we don't have dataflow information and though we could
745 hack something up here, it is really not worth the aggravation. */
747 static bool
748 workshare_safe_to_combine_p (basic_block ws_entry_bb)
750 struct omp_for_data fd;
751 gimple ws_stmt = last_stmt (ws_entry_bb);
753 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
754 return true;
756 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR);
758 extract_omp_for_data (ws_stmt, &fd, NULL);
760 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
761 return false;
762 if (fd.iter_type != long_integer_type_node)
763 return false;
765 /* FIXME. We give up too easily here. If any of these arguments
766 are not constants, they will likely involve variables that have
767 been mapped into fields of .omp_data_s for sharing with the child
768 function. With appropriate data flow, it would be possible to
769 see through this. */
770 if (!is_gimple_min_invariant (fd.loop.n1)
771 || !is_gimple_min_invariant (fd.loop.n2)
772 || !is_gimple_min_invariant (fd.loop.step)
773 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
774 return false;
776 return true;
780 /* Collect additional arguments needed to emit a combined
781 parallel+workshare call. WS_STMT is the workshare directive being
782 expanded. */
784 static vec<tree, va_gc> *
785 get_ws_args_for (gimple par_stmt, gimple ws_stmt)
787 tree t;
788 location_t loc = gimple_location (ws_stmt);
789 vec<tree, va_gc> *ws_args;
791 if (gimple_code (ws_stmt) == GIMPLE_OMP_FOR)
793 struct omp_for_data fd;
794 tree n1, n2;
796 extract_omp_for_data (ws_stmt, &fd, NULL);
797 n1 = fd.loop.n1;
798 n2 = fd.loop.n2;
800 if (gimple_omp_for_combined_into_p (ws_stmt))
802 tree innerc
803 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt),
804 OMP_CLAUSE__LOOPTEMP_);
805 gcc_assert (innerc);
806 n1 = OMP_CLAUSE_DECL (innerc);
807 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
808 OMP_CLAUSE__LOOPTEMP_);
809 gcc_assert (innerc);
810 n2 = OMP_CLAUSE_DECL (innerc);
813 vec_alloc (ws_args, 3 + (fd.chunk_size != 0));
815 t = fold_convert_loc (loc, long_integer_type_node, n1);
816 ws_args->quick_push (t);
818 t = fold_convert_loc (loc, long_integer_type_node, n2);
819 ws_args->quick_push (t);
821 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step);
822 ws_args->quick_push (t);
824 if (fd.chunk_size)
826 t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size);
827 ws_args->quick_push (t);
830 return ws_args;
832 else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS)
834 /* Number of sections is equal to the number of edges from the
835 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
836 the exit of the sections region. */
837 basic_block bb = single_succ (gimple_bb (ws_stmt));
838 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
839 vec_alloc (ws_args, 1);
840 ws_args->quick_push (t);
841 return ws_args;
844 gcc_unreachable ();
848 /* Discover whether REGION is a combined parallel+workshare region. */
850 static void
851 determine_parallel_type (struct omp_region *region)
853 basic_block par_entry_bb, par_exit_bb;
854 basic_block ws_entry_bb, ws_exit_bb;
856 if (region == NULL || region->inner == NULL
857 || region->exit == NULL || region->inner->exit == NULL
858 || region->inner->cont == NULL)
859 return;
861 /* We only support parallel+for and parallel+sections. */
862 if (region->type != GIMPLE_OMP_PARALLEL
863 || (region->inner->type != GIMPLE_OMP_FOR
864 && region->inner->type != GIMPLE_OMP_SECTIONS))
865 return;
867 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
868 WS_EXIT_BB -> PAR_EXIT_BB. */
869 par_entry_bb = region->entry;
870 par_exit_bb = region->exit;
871 ws_entry_bb = region->inner->entry;
872 ws_exit_bb = region->inner->exit;
874 if (single_succ (par_entry_bb) == ws_entry_bb
875 && single_succ (ws_exit_bb) == par_exit_bb
876 && workshare_safe_to_combine_p (ws_entry_bb)
877 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb))
878 || (last_and_only_stmt (ws_entry_bb)
879 && last_and_only_stmt (par_exit_bb))))
881 gimple par_stmt = last_stmt (par_entry_bb);
882 gimple ws_stmt = last_stmt (ws_entry_bb);
884 if (region->inner->type == GIMPLE_OMP_FOR)
886 /* If this is a combined parallel loop, we need to determine
887 whether or not to use the combined library calls. There
888 are two cases where we do not apply the transformation:
889 static loops and any kind of ordered loop. In the first
890 case, we already open code the loop so there is no need
891 to do anything else. In the latter case, the combined
892 parallel loop call would still need extra synchronization
893 to implement ordered semantics, so there would not be any
894 gain in using the combined call. */
895 tree clauses = gimple_omp_for_clauses (ws_stmt);
896 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE);
897 if (c == NULL
898 || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC
899 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED))
901 region->is_combined_parallel = false;
902 region->inner->is_combined_parallel = false;
903 return;
907 region->is_combined_parallel = true;
908 region->inner->is_combined_parallel = true;
909 region->ws_args = get_ws_args_for (par_stmt, ws_stmt);
914 /* Return true if EXPR is variable sized. */
916 static inline bool
917 is_variable_sized (const_tree expr)
919 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
922 /* Return true if DECL is a reference type. */
924 static inline bool
925 is_reference (tree decl)
927 return lang_hooks.decls.omp_privatize_by_reference (decl);
930 /* Return the type of a decl. If the decl is reference type,
931 return its base type. */
932 static inline tree
933 get_base_type (tree decl)
935 tree type = TREE_TYPE (decl);
936 if (is_reference (decl))
937 type = TREE_TYPE (type);
938 return type;
941 /* Lookup variables in the decl or field splay trees. The "maybe" form
942 allows for the variable form to not have been entered, otherwise we
943 assert that the variable must have been entered. */
945 static inline tree
946 lookup_decl (tree var, omp_context *ctx)
948 tree *n = ctx->cb.decl_map->get (var);
949 return *n;
952 static inline tree
953 maybe_lookup_decl (const_tree var, omp_context *ctx)
955 tree *n = ctx->cb.decl_map->get (const_cast<tree> (var));
956 return n ? *n : NULL_TREE;
959 static inline tree
960 lookup_field (tree var, omp_context *ctx)
962 splay_tree_node n;
963 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
964 return (tree) n->value;
967 static inline tree
968 lookup_sfield (tree var, omp_context *ctx)
970 splay_tree_node n;
971 n = splay_tree_lookup (ctx->sfield_map
972 ? ctx->sfield_map : ctx->field_map,
973 (splay_tree_key) var);
974 return (tree) n->value;
977 static inline tree
978 lookup_reduction (const char *id, omp_context *ctx)
980 gcc_assert (is_gimple_omp_oacc_specifically (ctx->stmt));
982 splay_tree_node n;
983 n = splay_tree_lookup (ctx->reduction_map,
984 (splay_tree_key) id);
985 return (tree) n->value;
988 static inline tree
989 maybe_lookup_field (tree var, omp_context *ctx)
991 splay_tree_node n;
992 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
993 return n ? (tree) n->value : NULL_TREE;
996 static inline tree
997 maybe_lookup_reduction (tree var, omp_context *ctx)
999 gcc_assert (is_gimple_omp_oacc_specifically (ctx->stmt));
1001 splay_tree_node n;
1002 n = splay_tree_lookup (ctx->reduction_map,
1003 (splay_tree_key) var);
1004 return n ?(tree) n->value : NULL_TREE;
1007 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1008 the parallel context if DECL is to be shared. */
1010 static bool
1011 use_pointer_for_field (tree decl, omp_context *shared_ctx)
1013 if (AGGREGATE_TYPE_P (TREE_TYPE (decl)))
1014 return true;
1016 /* We can only use copy-in/copy-out semantics for shared variables
1017 when we know the value is not accessible from an outer scope. */
1018 if (shared_ctx)
1020 gcc_assert (!is_gimple_omp_oacc_specifically (shared_ctx->stmt));
1022 /* ??? Trivially accessible from anywhere. But why would we even
1023 be passing an address in this case? Should we simply assert
1024 this to be false, or should we have a cleanup pass that removes
1025 these from the list of mappings? */
1026 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
1027 return true;
1029 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1030 without analyzing the expression whether or not its location
1031 is accessible to anyone else. In the case of nested parallel
1032 regions it certainly may be. */
1033 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
1034 return true;
1036 /* Do not use copy-in/copy-out for variables that have their
1037 address taken. */
1038 if (TREE_ADDRESSABLE (decl))
1039 return true;
1041 /* lower_send_shared_vars only uses copy-in, but not copy-out
1042 for these. */
1043 if (TREE_READONLY (decl)
1044 || ((TREE_CODE (decl) == RESULT_DECL
1045 || TREE_CODE (decl) == PARM_DECL)
1046 && DECL_BY_REFERENCE (decl)))
1047 return false;
1049 /* Disallow copy-in/out in nested parallel if
1050 decl is shared in outer parallel, otherwise
1051 each thread could store the shared variable
1052 in its own copy-in location, making the
1053 variable no longer really shared. */
1054 if (shared_ctx->is_nested)
1056 omp_context *up;
1058 for (up = shared_ctx->outer; up; up = up->outer)
1059 if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up))
1060 break;
1062 if (up)
1064 tree c;
1066 for (c = gimple_omp_taskreg_clauses (up->stmt);
1067 c; c = OMP_CLAUSE_CHAIN (c))
1068 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
1069 && OMP_CLAUSE_DECL (c) == decl)
1070 break;
1072 if (c)
1073 goto maybe_mark_addressable_and_ret;
1077 /* For tasks avoid using copy-in/out. As tasks can be
1078 deferred or executed in different thread, when GOMP_task
1079 returns, the task hasn't necessarily terminated. */
1080 if (is_task_ctx (shared_ctx))
1082 tree outer;
1083 maybe_mark_addressable_and_ret:
1084 outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx);
1085 if (is_gimple_reg (outer))
1087 /* Taking address of OUTER in lower_send_shared_vars
1088 might need regimplification of everything that uses the
1089 variable. */
1090 if (!task_shared_vars)
1091 task_shared_vars = BITMAP_ALLOC (NULL);
1092 bitmap_set_bit (task_shared_vars, DECL_UID (outer));
1093 TREE_ADDRESSABLE (outer) = 1;
1095 return true;
1099 return false;
1102 /* Construct a new automatic decl similar to VAR. */
1104 static tree
1105 omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
1107 tree copy = copy_var_decl (var, name, type);
1109 DECL_CONTEXT (copy) = current_function_decl;
1110 DECL_CHAIN (copy) = ctx->block_vars;
1111 ctx->block_vars = copy;
1113 return copy;
1116 static tree
1117 omp_copy_decl_1 (tree var, omp_context *ctx)
1119 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
1122 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1123 as appropriate. */
1124 static tree
1125 omp_build_component_ref (tree obj, tree field)
1127 tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL);
1128 if (TREE_THIS_VOLATILE (field))
1129 TREE_THIS_VOLATILE (ret) |= 1;
1130 if (TREE_READONLY (field))
1131 TREE_READONLY (ret) |= 1;
1132 return ret;
1135 /* Build tree nodes to access the field for VAR on the receiver side. */
1137 static tree
1138 build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
1140 tree x, field = lookup_field (var, ctx);
1142 /* If the receiver record type was remapped in the child function,
1143 remap the field into the new record type. */
1144 x = maybe_lookup_field (field, ctx);
1145 if (x != NULL)
1146 field = x;
1148 x = build_simple_mem_ref (ctx->receiver_decl);
1149 x = omp_build_component_ref (x, field);
1150 if (by_ref)
1151 x = build_simple_mem_ref (x);
1153 return x;
1156 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1157 of a parallel, this is a component reference; for workshare constructs
1158 this is some variable. */
1160 static tree
1161 build_outer_var_ref (tree var, omp_context *ctx)
1163 tree x;
1165 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
1166 x = var;
1167 else if (is_variable_sized (var))
1169 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
1170 x = build_outer_var_ref (x, ctx);
1171 x = build_simple_mem_ref (x);
1173 else if (is_taskreg_ctx (ctx))
1175 bool by_ref = use_pointer_for_field (var, NULL);
1176 x = build_receiver_ref (var, by_ref, ctx);
1178 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
1179 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
1181 /* #pragma omp simd isn't a worksharing construct, and can reference even
1182 private vars in its linear etc. clauses. */
1183 x = NULL_TREE;
1184 if (ctx->outer && is_taskreg_ctx (ctx))
1185 x = lookup_decl (var, ctx->outer);
1186 else if (ctx->outer)
1187 x = maybe_lookup_decl_in_outer_ctx (var, ctx);
1188 if (x == NULL_TREE)
1189 x = var;
1191 else if (ctx->outer)
1192 x = lookup_decl (var, ctx->outer);
1193 else if (is_reference (var))
1194 /* This can happen with orphaned constructs. If var is reference, it is
1195 possible it is shared and as such valid. */
1196 x = var;
1197 else
1198 gcc_unreachable ();
1200 if (is_reference (var))
1201 x = build_simple_mem_ref (x);
1203 return x;
1206 /* Build tree nodes to access the field for VAR on the sender side. */
1208 static tree
1209 build_sender_ref (tree var, omp_context *ctx)
1211 tree field = lookup_sfield (var, ctx);
1212 return omp_build_component_ref (ctx->sender_decl, field);
1215 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1217 static void
1218 install_var_field (tree var, bool by_ref, int mask, omp_context *ctx)
1220 tree field, type, sfield = NULL_TREE;
1222 gcc_assert ((mask & 1) == 0
1223 || !splay_tree_lookup (ctx->field_map, (splay_tree_key) var));
1224 gcc_assert ((mask & 2) == 0 || !ctx->sfield_map
1225 || !splay_tree_lookup (ctx->sfield_map, (splay_tree_key) var));
1226 gcc_assert ((mask & 3) == 3
1227 || !is_gimple_omp_oacc_specifically (ctx->stmt));
1229 type = TREE_TYPE (var);
1230 if (mask & 4)
1232 gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
1233 type = build_pointer_type (build_pointer_type (type));
1235 else if (by_ref)
1236 type = build_pointer_type (type);
1237 else if ((mask & 3) == 1 && is_reference (var))
1238 type = TREE_TYPE (type);
1240 field = build_decl (DECL_SOURCE_LOCATION (var),
1241 FIELD_DECL, DECL_NAME (var), type);
1243 /* Remember what variable this field was created for. This does have a
1244 side effect of making dwarf2out ignore this member, so for helpful
1245 debugging we clear it later in delete_omp_context. */
1246 DECL_ABSTRACT_ORIGIN (field) = var;
1247 if (type == TREE_TYPE (var))
1249 DECL_ALIGN (field) = DECL_ALIGN (var);
1250 DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
1251 TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var);
1253 else
1254 DECL_ALIGN (field) = TYPE_ALIGN (type);
1256 if ((mask & 3) == 3)
1258 insert_field_into_struct (ctx->record_type, field);
1259 if (ctx->srecord_type)
1261 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1262 FIELD_DECL, DECL_NAME (var), type);
1263 DECL_ABSTRACT_ORIGIN (sfield) = var;
1264 DECL_ALIGN (sfield) = DECL_ALIGN (field);
1265 DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field);
1266 TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field);
1267 insert_field_into_struct (ctx->srecord_type, sfield);
1270 else
1272 if (ctx->srecord_type == NULL_TREE)
1274 tree t;
1276 ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE);
1277 ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1278 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
1280 sfield = build_decl (DECL_SOURCE_LOCATION (var),
1281 FIELD_DECL, DECL_NAME (t), TREE_TYPE (t));
1282 DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t);
1283 insert_field_into_struct (ctx->srecord_type, sfield);
1284 splay_tree_insert (ctx->sfield_map,
1285 (splay_tree_key) DECL_ABSTRACT_ORIGIN (t),
1286 (splay_tree_value) sfield);
1289 sfield = field;
1290 insert_field_into_struct ((mask & 1) ? ctx->record_type
1291 : ctx->srecord_type, field);
1294 if (mask & 1)
1295 splay_tree_insert (ctx->field_map, (splay_tree_key) var,
1296 (splay_tree_value) field);
1297 if ((mask & 2) && ctx->sfield_map)
1298 splay_tree_insert (ctx->sfield_map, (splay_tree_key) var,
1299 (splay_tree_value) sfield);
1302 static tree
1303 install_var_local (tree var, omp_context *ctx)
1305 tree new_var = omp_copy_decl_1 (var, ctx);
1306 insert_decl_map (&ctx->cb, var, new_var);
1307 return new_var;
1310 /* Adjust the replacement for DECL in CTX for the new context. This means
1311 copying the DECL_VALUE_EXPR, and fixing up the type. */
1313 static void
1314 fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
1316 tree new_decl, size;
1318 new_decl = lookup_decl (decl, ctx);
1320 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
1322 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
1323 && DECL_HAS_VALUE_EXPR_P (decl))
1325 tree ve = DECL_VALUE_EXPR (decl);
1326 walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL);
1327 SET_DECL_VALUE_EXPR (new_decl, ve);
1328 DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
1331 if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
1333 size = remap_decl (DECL_SIZE (decl), &ctx->cb);
1334 if (size == error_mark_node)
1335 size = TYPE_SIZE (TREE_TYPE (new_decl));
1336 DECL_SIZE (new_decl) = size;
1338 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
1339 if (size == error_mark_node)
1340 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
1341 DECL_SIZE_UNIT (new_decl) = size;
1345 /* The callback for remap_decl. Search all containing contexts for a
1346 mapping of the variable; this avoids having to duplicate the splay
1347 tree ahead of time. We know a mapping doesn't already exist in the
1348 given context. Create new mappings to implement default semantics. */
1350 static tree
1351 omp_copy_decl (tree var, copy_body_data *cb)
1353 omp_context *ctx = (omp_context *) cb;
1354 tree new_var;
1356 if (TREE_CODE (var) == LABEL_DECL)
1358 new_var = create_artificial_label (DECL_SOURCE_LOCATION (var));
1359 DECL_CONTEXT (new_var) = current_function_decl;
1360 insert_decl_map (&ctx->cb, var, new_var);
1361 return new_var;
1364 while (!is_taskreg_ctx (ctx))
1366 ctx = ctx->outer;
1367 if (ctx == NULL)
1368 return var;
1369 new_var = maybe_lookup_decl (var, ctx);
1370 if (new_var)
1371 return new_var;
1374 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
1375 return var;
1377 return error_mark_node;
1381 /* Debugging dumps for parallel regions. */
1382 void dump_omp_region (FILE *, struct omp_region *, int);
1383 void debug_omp_region (struct omp_region *);
1384 void debug_all_omp_regions (void);
1386 /* Dump the parallel region tree rooted at REGION. */
1388 void
1389 dump_omp_region (FILE *file, struct omp_region *region, int indent)
1391 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
1392 gimple_code_name[region->type]);
1394 if (region->inner)
1395 dump_omp_region (file, region->inner, indent + 4);
1397 if (region->cont)
1399 fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "",
1400 region->cont->index);
1403 if (region->exit)
1404 fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "",
1405 region->exit->index);
1406 else
1407 fprintf (file, "%*s[no exit marker]\n", indent, "");
1409 if (region->next)
1410 dump_omp_region (file, region->next, indent);
1413 DEBUG_FUNCTION void
1414 debug_omp_region (struct omp_region *region)
1416 dump_omp_region (stderr, region, 0);
1419 DEBUG_FUNCTION void
1420 debug_all_omp_regions (void)
1422 dump_omp_region (stderr, root_omp_region, 0);
1426 /* Create a new parallel region starting at STMT inside region PARENT. */
1428 static struct omp_region *
1429 new_omp_region (basic_block bb, enum gimple_code type,
1430 struct omp_region *parent)
1432 struct omp_region *region = XCNEW (struct omp_region);
1434 region->outer = parent;
1435 region->entry = bb;
1436 region->type = type;
1438 if (parent)
1440 /* This is a nested region. Add it to the list of inner
1441 regions in PARENT. */
1442 region->next = parent->inner;
1443 parent->inner = region;
1445 else
1447 /* This is a toplevel region. Add it to the list of toplevel
1448 regions in ROOT_OMP_REGION. */
1449 region->next = root_omp_region;
1450 root_omp_region = region;
1453 return region;
1456 /* Release the memory associated with the region tree rooted at REGION. */
1458 static void
1459 free_omp_region_1 (struct omp_region *region)
1461 struct omp_region *i, *n;
1463 for (i = region->inner; i ; i = n)
1465 n = i->next;
1466 free_omp_region_1 (i);
1469 free (region);
1472 /* Release the memory for the entire omp region tree. */
1474 void
1475 free_omp_regions (void)
1477 struct omp_region *r, *n;
1478 for (r = root_omp_region; r ; r = n)
1480 n = r->next;
1481 free_omp_region_1 (r);
1483 root_omp_region = NULL;
1487 /* Create a new context, with OUTER_CTX being the surrounding context. */
1489 static omp_context *
1490 new_omp_context (gimple stmt, omp_context *outer_ctx)
1492 omp_context *ctx = XCNEW (omp_context);
1494 splay_tree_insert (all_contexts, (splay_tree_key) stmt,
1495 (splay_tree_value) ctx);
1496 ctx->stmt = stmt;
1498 if (outer_ctx)
1500 ctx->outer = outer_ctx;
1501 ctx->cb = outer_ctx->cb;
1502 ctx->cb.block = NULL;
1503 ctx->depth = outer_ctx->depth + 1;
1504 /* FIXME: handle reductions recursively. */
1505 ctx->reduction_map = outer_ctx->reduction_map;
1507 else
1509 ctx->cb.src_fn = current_function_decl;
1510 ctx->cb.dst_fn = current_function_decl;
1511 ctx->cb.src_node = cgraph_node::get (current_function_decl);
1512 gcc_checking_assert (ctx->cb.src_node);
1513 ctx->cb.dst_node = ctx->cb.src_node;
1514 ctx->cb.src_cfun = cfun;
1515 ctx->cb.copy_decl = omp_copy_decl;
1516 ctx->cb.eh_lp_nr = 0;
1517 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
1518 ctx->depth = 1;
1519 //TODO ctx->reduction_map = TODO;
1522 ctx->cb.decl_map = new hash_map<tree, tree>;
1524 return ctx;
1527 static gimple_seq maybe_catch_exception (gimple_seq);
1529 /* Finalize task copyfn. */
1531 static void
1532 finalize_task_copyfn (gimple task_stmt)
1534 struct function *child_cfun;
1535 tree child_fn;
1536 gimple_seq seq = NULL, new_seq;
1537 gimple bind;
1539 child_fn = gimple_omp_task_copy_fn (task_stmt);
1540 if (child_fn == NULL_TREE)
1541 return;
1543 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
1544 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
1546 push_cfun (child_cfun);
1547 bind = gimplify_body (child_fn, false);
1548 gimple_seq_add_stmt (&seq, bind);
1549 new_seq = maybe_catch_exception (seq);
1550 if (new_seq != seq)
1552 bind = gimple_build_bind (NULL, new_seq, NULL);
1553 seq = NULL;
1554 gimple_seq_add_stmt (&seq, bind);
1556 gimple_set_body (child_fn, seq);
1557 pop_cfun ();
1559 /* Inform the callgraph about the new function. */
1560 cgraph_node::add_new_function (child_fn, false);
1563 /* Destroy a omp_context data structures. Called through the splay tree
1564 value delete callback. */
1566 static void
1567 delete_omp_context (splay_tree_value value)
1569 omp_context *ctx = (omp_context *) value;
1571 delete ctx->cb.decl_map;
1573 if (ctx->field_map)
1574 splay_tree_delete (ctx->field_map);
1575 if (ctx->sfield_map)
1576 splay_tree_delete (ctx->sfield_map);
1578 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1579 it produces corrupt debug information. */
1580 if (ctx->record_type)
1582 tree t;
1583 for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t))
1584 DECL_ABSTRACT_ORIGIN (t) = NULL;
1586 if (ctx->srecord_type)
1588 tree t;
1589 for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t))
1590 DECL_ABSTRACT_ORIGIN (t) = NULL;
1593 if (is_task_ctx (ctx))
1594 finalize_task_copyfn (ctx->stmt);
1596 XDELETE (ctx);
1599 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1600 context. */
1602 static void
1603 fixup_child_record_type (omp_context *ctx)
1605 tree f, type = ctx->record_type;
1607 /* ??? It isn't sufficient to just call remap_type here, because
1608 variably_modified_type_p doesn't work the way we expect for
1609 record types. Testing each field for whether it needs remapping
1610 and creating a new record by hand works, however. */
1611 for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f))
1612 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
1613 break;
1614 if (f)
1616 tree name, new_fields = NULL;
1618 type = lang_hooks.types.make_type (RECORD_TYPE);
1619 name = DECL_NAME (TYPE_NAME (ctx->record_type));
1620 name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl),
1621 TYPE_DECL, name, type);
1622 TYPE_NAME (type) = name;
1624 for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f))
1626 tree new_f = copy_node (f);
1627 DECL_CONTEXT (new_f) = type;
1628 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
1629 DECL_CHAIN (new_f) = new_fields;
1630 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL);
1631 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r,
1632 &ctx->cb, NULL);
1633 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
1634 &ctx->cb, NULL);
1635 new_fields = new_f;
1637 /* Arrange to be able to look up the receiver field
1638 given the sender field. */
1639 splay_tree_insert (ctx->field_map, (splay_tree_key) f,
1640 (splay_tree_value) new_f);
1642 TYPE_FIELDS (type) = nreverse (new_fields);
1643 layout_type (type);
1646 TREE_TYPE (ctx->receiver_decl) = build_pointer_type (type);
1649 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1650 specified by CLAUSES. */
1652 static void
1653 scan_sharing_clauses (tree clauses, omp_context *ctx)
1655 tree c, decl;
1656 bool scan_array_reductions = false;
1658 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1660 bool by_ref;
1662 switch (OMP_CLAUSE_CODE (c))
1664 case OMP_CLAUSE_PRIVATE:
1665 decl = OMP_CLAUSE_DECL (c);
1666 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
1667 goto do_private;
1668 else if (!is_variable_sized (decl))
1669 install_var_local (decl, ctx);
1670 break;
1672 case OMP_CLAUSE_SHARED:
1673 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
1674 decl = OMP_CLAUSE_DECL (c);
1675 /* Ignore shared directives in teams construct. */
1676 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
1678 /* Global variables don't need to be copied,
1679 the receiver side will use them directly. */
1680 tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx);
1681 if (is_global_var (odecl))
1682 break;
1683 insert_decl_map (&ctx->cb, decl, odecl);
1684 break;
1686 gcc_assert (is_taskreg_ctx (ctx));
1687 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl))
1688 || !is_variable_sized (decl));
1689 /* Global variables don't need to be copied,
1690 the receiver side will use them directly. */
1691 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1692 break;
1693 by_ref = use_pointer_for_field (decl, ctx);
1694 if (! TREE_READONLY (decl)
1695 || TREE_ADDRESSABLE (decl)
1696 || by_ref
1697 || is_reference (decl))
1699 install_var_field (decl, by_ref, 3, ctx);
1700 install_var_local (decl, ctx);
1701 break;
1703 /* We don't need to copy const scalar vars back. */
1704 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
1705 goto do_private;
1707 case OMP_CLAUSE_LASTPRIVATE:
1708 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
1709 /* Let the corresponding firstprivate clause create
1710 the variable. */
1711 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1712 break;
1713 /* FALLTHRU */
1715 case OMP_CLAUSE_FIRSTPRIVATE:
1716 if (is_gimple_omp_oacc_specifically (ctx->stmt))
1718 sorry ("clause not supported yet");
1719 break;
1721 /* FALLTHRU */
1722 case OMP_CLAUSE_LINEAR:
1723 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
1724 /* FALLTHRU */
1725 case OMP_CLAUSE_REDUCTION:
1726 decl = OMP_CLAUSE_DECL (c);
1727 do_private:
1728 if (is_variable_sized (decl))
1730 if (is_task_ctx (ctx))
1731 install_var_field (decl, false, 1, ctx);
1732 break;
1734 else if (is_taskreg_ctx (ctx))
1736 bool global
1737 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx));
1738 by_ref = use_pointer_for_field (decl, NULL);
1740 if (is_task_ctx (ctx)
1741 && (global || by_ref || is_reference (decl)))
1743 install_var_field (decl, false, 1, ctx);
1744 if (!global)
1745 install_var_field (decl, by_ref, 2, ctx);
1747 else if (!global)
1748 install_var_field (decl, by_ref, 3, ctx);
1750 install_var_local (decl, ctx);
1751 //TODO
1752 if (is_gimple_omp_oacc_specifically (ctx->stmt))
1754 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
1756 /* Create a decl for the reduction array. */
1757 tree var = OMP_CLAUSE_DECL (c);
1758 tree type = get_base_type (var);
1759 tree ptype = build_pointer_type (type);
1760 tree array = create_tmp_var (ptype, omp_get_id (var));
1761 omp_context *c = (ctx->field_map ? ctx : ctx->outer);
1762 install_var_field (array, true, 3, c);
1763 install_var_local (array, c);
1765 /* Insert it into the current context. */
1766 splay_tree_insert (ctx->reduction_map,
1767 (splay_tree_key) omp_get_id(var),
1768 (splay_tree_value) array);
1769 splay_tree_insert (ctx->reduction_map,
1770 (splay_tree_key) array,
1771 (splay_tree_value) array);
1774 break;
1776 case OMP_CLAUSE__LOOPTEMP_:
1777 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
1778 gcc_assert (is_parallel_ctx (ctx));
1779 decl = OMP_CLAUSE_DECL (c);
1780 install_var_field (decl, false, 3, ctx);
1781 install_var_local (decl, ctx);
1782 break;
1784 case OMP_CLAUSE_COPYPRIVATE:
1785 case OMP_CLAUSE_COPYIN:
1786 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
1787 decl = OMP_CLAUSE_DECL (c);
1788 by_ref = use_pointer_for_field (decl, NULL);
1789 install_var_field (decl, by_ref, 3, ctx);
1790 break;
1792 case OMP_CLAUSE_DEFAULT:
1793 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
1794 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c);
1795 break;
1797 case OMP_CLAUSE_FINAL:
1798 case OMP_CLAUSE_NUM_THREADS:
1799 case OMP_CLAUSE_NUM_TEAMS:
1800 case OMP_CLAUSE_THREAD_LIMIT:
1801 case OMP_CLAUSE_DEVICE:
1802 case OMP_CLAUSE_SCHEDULE:
1803 case OMP_CLAUSE_DIST_SCHEDULE:
1804 case OMP_CLAUSE_DEPEND:
1805 case OMP_CLAUSE__CILK_FOR_COUNT_:
1806 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
1807 /* FALLTHRU */
1808 case OMP_CLAUSE_IF:
1809 case OMP_CLAUSE_NUM_GANGS:
1810 case OMP_CLAUSE_NUM_WORKERS:
1811 case OMP_CLAUSE_VECTOR_LENGTH:
1812 if (ctx->outer)
1813 scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1814 break;
1816 case OMP_CLAUSE_TO:
1817 case OMP_CLAUSE_FROM:
1818 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
1819 /* FALLTHRU */
1820 case OMP_CLAUSE_MAP:
1821 switch (OMP_CLAUSE_CODE (c))
1823 case OMP_CLAUSE_TO:
1824 case OMP_CLAUSE_FROM:
1825 /* The to and from clauses are only ever seen with OpenMP target
1826 update constructs. */
1827 gcc_assert (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
1828 && (gimple_omp_target_kind (ctx->stmt)
1829 == GF_OMP_TARGET_KIND_UPDATE));
1830 break;
1831 case OMP_CLAUSE_MAP:
1832 /* The map clause is never seen with OpenMP target update
1833 constructs. */
1834 gcc_assert (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
1835 || (gimple_omp_target_kind (ctx->stmt)
1836 != GF_OMP_TARGET_KIND_UPDATE));
1837 break;
1838 default:
1839 gcc_unreachable ();
1841 if (ctx->outer)
1842 scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer);
1843 decl = OMP_CLAUSE_DECL (c);
1844 /* Global variables with "omp declare target" attribute
1845 don't need to be copied, the receiver side will use them
1846 directly. */
1847 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1848 && DECL_P (decl)
1849 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
1850 && lookup_attribute ("omp declare target",
1851 DECL_ATTRIBUTES (decl)))
1853 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
1854 break;
1856 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1857 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER)
1859 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1860 target regions that are not offloaded; there is nothing to map for
1861 those. */
1862 if (!is_gimple_omp_offloaded (ctx->stmt)
1863 && !POINTER_TYPE_P (TREE_TYPE (decl)))
1864 break;
1866 #if 0
1867 /* In target regions that are not offloaded, libgomp won't pay
1868 attention to OMP_CLAUSE_MAP_FORCE_DEVICEPTR -- but I think we need
1869 to handle it here anyway, in order to create a visible copy of the
1870 variable. */
1871 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1872 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
1874 if (!is_gimple_omp_offloaded (ctx->stmt))
1875 break;
1877 #endif
1878 if (DECL_P (decl))
1880 if (DECL_SIZE (decl)
1881 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
1883 tree decl2 = DECL_VALUE_EXPR (decl);
1884 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
1885 decl2 = TREE_OPERAND (decl2, 0);
1886 gcc_assert (DECL_P (decl2));
1887 install_var_field (decl2, true, 3, ctx);
1888 install_var_local (decl2, ctx);
1889 install_var_local (decl, ctx);
1891 else
1893 gcc_assert (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
1894 || (OMP_CLAUSE_MAP_KIND (c)
1895 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
1896 || TREE_CODE (TREE_TYPE (decl)) != ARRAY_TYPE);
1897 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
1898 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
1899 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
1900 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1901 install_var_field (decl, true, 7, ctx);
1902 else
1904 if (!is_gimple_omp_oacc_specifically (ctx->stmt))
1905 install_var_field (decl, true, 3, ctx);
1906 else
1908 /* decl goes heres. */
1909 omp_context *c = (ctx->field_map ? ctx : ctx->outer);
1910 install_var_field (decl, true, 3, c);
1913 if (is_gimple_omp_offloaded (ctx->stmt))
1914 install_var_local (decl, ctx);
1917 else
1919 tree base = get_base_address (decl);
1920 tree nc = OMP_CLAUSE_CHAIN (c);
1921 gcc_assert (nc == NULL_TREE
1922 || OMP_CLAUSE_CODE (nc) != OMP_CLAUSE_MAP
1923 || (OMP_CLAUSE_MAP_KIND (nc)
1924 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR));
1925 if (DECL_P (base)
1926 && nc != NULL_TREE
1927 && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP
1928 && OMP_CLAUSE_DECL (nc) == base
1929 && OMP_CLAUSE_MAP_KIND (nc) == OMP_CLAUSE_MAP_POINTER
1930 && integer_zerop (OMP_CLAUSE_SIZE (nc)))
1932 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1;
1933 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1;
1935 else
1937 if (ctx->outer)
1939 scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer);
1940 decl = OMP_CLAUSE_DECL (c);
1942 gcc_assert (!splay_tree_lookup (ctx->field_map,
1943 (splay_tree_key) decl));
1944 tree field
1945 = build_decl (OMP_CLAUSE_LOCATION (c),
1946 FIELD_DECL, NULL_TREE, ptr_type_node);
1947 DECL_ALIGN (field) = TYPE_ALIGN (ptr_type_node);
1948 insert_field_into_struct (ctx->record_type, field);
1949 splay_tree_insert (ctx->field_map, (splay_tree_key) decl,
1950 (splay_tree_value) field);
1953 break;
1955 case OMP_CLAUSE_NOWAIT:
1956 case OMP_CLAUSE_ORDERED:
1957 case OMP_CLAUSE_UNTIED:
1958 case OMP_CLAUSE_MERGEABLE:
1959 case OMP_CLAUSE_PROC_BIND:
1960 case OMP_CLAUSE_SAFELEN:
1961 if (is_gimple_omp_oacc_specifically (ctx->stmt))
1963 sorry ("clause not supported yet");
1964 break;
1966 /* FALLTHRU */
1967 case OMP_CLAUSE_COLLAPSE:
1968 case OMP_CLAUSE_ASYNC:
1969 case OMP_CLAUSE_WAIT:
1970 break;
1972 case OMP_CLAUSE_ALIGNED:
1973 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
1974 decl = OMP_CLAUSE_DECL (c);
1975 if (is_global_var (decl)
1976 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
1977 install_var_local (decl, ctx);
1978 break;
1980 case OMP_CLAUSE_HOST:
1981 case OMP_CLAUSE_OACC_DEVICE:
1982 case OMP_CLAUSE_DEVICE_RESIDENT:
1983 case OMP_CLAUSE_USE_DEVICE:
1984 case OMP_CLAUSE_GANG:
1985 case OMP_CLAUSE__CACHE_:
1986 case OMP_CLAUSE_INDEPENDENT:
1987 case OMP_CLAUSE_WORKER:
1988 case OMP_CLAUSE_VECTOR:
1989 sorry ("Clause not supported yet");
1990 break;
1992 default:
1993 gcc_unreachable ();
1997 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1999 switch (OMP_CLAUSE_CODE (c))
2001 case OMP_CLAUSE_LASTPRIVATE:
2002 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
2003 /* Let the corresponding firstprivate clause create
2004 the variable. */
2005 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2006 scan_array_reductions = true;
2007 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
2008 break;
2009 /* FALLTHRU */
2011 case OMP_CLAUSE_FIRSTPRIVATE:
2012 if (is_gimple_omp_oacc_specifically (ctx->stmt))
2014 sorry ("clause not supported yet");
2015 break;
2017 /* FALLTHRU */
2018 case OMP_CLAUSE_LINEAR:
2019 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
2020 /* FALLTHRU */
2021 case OMP_CLAUSE_REDUCTION:
2022 case OMP_CLAUSE_PRIVATE:
2023 decl = OMP_CLAUSE_DECL (c);
2024 if (is_variable_sized (decl))
2025 install_var_local (decl, ctx);
2026 fixup_remapped_decl (decl, ctx,
2027 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
2028 && OMP_CLAUSE_PRIVATE_DEBUG (c));
2029 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
2030 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2031 scan_array_reductions = true;
2032 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2033 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2034 scan_array_reductions = true;
2035 break;
2037 case OMP_CLAUSE_SHARED:
2038 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
2039 /* Ignore shared directives in teams construct. */
2040 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2041 break;
2042 decl = OMP_CLAUSE_DECL (c);
2043 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2044 fixup_remapped_decl (decl, ctx, false);
2045 break;
2047 case OMP_CLAUSE_MAP:
2048 /* The map clause is never seen with OpenMP target update
2049 constructs. */
2050 gcc_assert (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
2051 || (gimple_omp_target_kind (ctx->stmt)
2052 != GF_OMP_TARGET_KIND_UPDATE));
2053 if (!is_gimple_omp_offloaded (ctx->stmt))
2054 break;
2055 decl = OMP_CLAUSE_DECL (c);
2056 if (DECL_P (decl)
2057 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
2058 && lookup_attribute ("omp declare target",
2059 DECL_ATTRIBUTES (decl)))
2061 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
2062 break;
2064 if (DECL_P (decl))
2066 gcc_assert ((OMP_CLAUSE_MAP_KIND (c)
2067 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
2068 || TREE_CODE (TREE_TYPE (decl)) != ARRAY_TYPE);
2069 if (OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
2070 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
2071 && !COMPLETE_TYPE_P (TREE_TYPE (decl)))
2073 tree new_decl = lookup_decl (decl, ctx);
2074 TREE_TYPE (new_decl)
2075 = remap_type (TREE_TYPE (decl), &ctx->cb);
2077 else if (DECL_SIZE (decl)
2078 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
2080 gcc_assert (OMP_CLAUSE_MAP_KIND (c)
2081 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR);
2083 tree decl2 = DECL_VALUE_EXPR (decl);
2084 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF);
2085 decl2 = TREE_OPERAND (decl2, 0);
2086 gcc_assert (DECL_P (decl2));
2087 fixup_remapped_decl (decl2, ctx, false);
2088 fixup_remapped_decl (decl, ctx, true);
2090 else
2091 fixup_remapped_decl (decl, ctx, false);
2093 break;
2095 case OMP_CLAUSE_COPYPRIVATE:
2096 case OMP_CLAUSE_COPYIN:
2097 case OMP_CLAUSE_DEFAULT:
2098 case OMP_CLAUSE_NUM_THREADS:
2099 case OMP_CLAUSE_NUM_TEAMS:
2100 case OMP_CLAUSE_THREAD_LIMIT:
2101 case OMP_CLAUSE_DEVICE:
2102 case OMP_CLAUSE_SCHEDULE:
2103 case OMP_CLAUSE_DIST_SCHEDULE:
2104 case OMP_CLAUSE_NOWAIT:
2105 case OMP_CLAUSE_ORDERED:
2106 case OMP_CLAUSE_UNTIED:
2107 case OMP_CLAUSE_FINAL:
2108 case OMP_CLAUSE_MERGEABLE:
2109 case OMP_CLAUSE_PROC_BIND:
2110 case OMP_CLAUSE_SAFELEN:
2111 case OMP_CLAUSE_ALIGNED:
2112 case OMP_CLAUSE_DEPEND:
2113 case OMP_CLAUSE__LOOPTEMP_:
2114 case OMP_CLAUSE_TO:
2115 case OMP_CLAUSE_FROM:
2116 case OMP_CLAUSE__CILK_FOR_COUNT_:
2117 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
2118 /* FALLTHRU */
2119 case OMP_CLAUSE_ASYNC:
2120 case OMP_CLAUSE_COLLAPSE:
2121 case OMP_CLAUSE_IF:
2122 case OMP_CLAUSE_NUM_GANGS:
2123 case OMP_CLAUSE_NUM_WORKERS:
2124 case OMP_CLAUSE_VECTOR_LENGTH:
2125 case OMP_CLAUSE_WAIT:
2126 break;
2128 case OMP_CLAUSE_HOST:
2129 case OMP_CLAUSE_OACC_DEVICE:
2130 case OMP_CLAUSE_DEVICE_RESIDENT:
2131 case OMP_CLAUSE_USE_DEVICE:
2132 case OMP_CLAUSE_GANG:
2133 case OMP_CLAUSE__CACHE_:
2134 case OMP_CLAUSE_INDEPENDENT:
2135 case OMP_CLAUSE_WORKER:
2136 case OMP_CLAUSE_VECTOR:
2137 sorry ("Clause not supported yet");
2138 break;
2140 default:
2141 gcc_unreachable ();
2145 if (scan_array_reductions)
2147 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
2148 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
2149 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
2150 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
2152 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx);
2153 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
2155 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
2156 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
2157 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
2158 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2159 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
2160 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
2164 /* Create a new name for omp child function. Returns an identifier. If
2165 IS_CILK_FOR is true then the suffix for the child function is
2166 "_cilk_for_fn." */
2168 static tree
2169 create_omp_child_function_name (bool task_copy, bool is_cilk_for)
2171 if (is_cilk_for)
2172 return clone_function_name (current_function_decl, "_cilk_for_fn");
2173 return clone_function_name (current_function_decl,
2174 task_copy ? "_omp_cpyfn" : "_omp_fn");
2177 /* Returns the type of the induction variable for the child function for
2178 _Cilk_for and the types for _high and _low variables based on TYPE. */
2180 static tree
2181 cilk_for_check_loop_diff_type (tree type)
2183 if (TYPE_PRECISION (type) <= TYPE_PRECISION (uint32_type_node))
2185 if (TYPE_UNSIGNED (type))
2186 return uint32_type_node;
2187 else
2188 return integer_type_node;
2190 else
2192 if (TYPE_UNSIGNED (type))
2193 return uint64_type_node;
2194 else
2195 return long_long_integer_type_node;
2199 /* Build a decl for the omp child function. It'll not contain a body
2200 yet, just the bare decl. */
2202 static void
2203 create_omp_child_function (omp_context *ctx, bool task_copy)
2205 tree decl, type, name, t;
2207 tree cilk_for_count
2208 = (flag_cilkplus && gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2209 ? find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
2210 OMP_CLAUSE__CILK_FOR_COUNT_) : NULL_TREE;
2211 tree cilk_var_type = NULL_TREE;
2213 name = create_omp_child_function_name (task_copy,
2214 cilk_for_count != NULL_TREE);
2215 if (task_copy)
2216 type = build_function_type_list (void_type_node, ptr_type_node,
2217 ptr_type_node, NULL_TREE);
2218 else if (cilk_for_count)
2220 type = TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count, 0));
2221 cilk_var_type = cilk_for_check_loop_diff_type (type);
2222 type = build_function_type_list (void_type_node, ptr_type_node,
2223 cilk_var_type, cilk_var_type, NULL_TREE);
2225 else
2226 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
2228 decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type);
2230 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt)
2231 || !task_copy);
2232 if (!task_copy)
2233 ctx->cb.dst_fn = decl;
2234 else
2235 gimple_omp_task_set_copy_fn (ctx->stmt, decl);
2237 TREE_STATIC (decl) = 1;
2238 TREE_USED (decl) = 1;
2239 DECL_ARTIFICIAL (decl) = 1;
2240 DECL_IGNORED_P (decl) = 0;
2241 TREE_PUBLIC (decl) = 0;
2242 DECL_UNINLINABLE (decl) = 1;
2243 DECL_EXTERNAL (decl) = 0;
2244 DECL_CONTEXT (decl) = NULL_TREE;
2245 DECL_INITIAL (decl) = make_node (BLOCK);
2246 bool target_p = false;
2247 if (lookup_attribute ("omp declare target",
2248 DECL_ATTRIBUTES (current_function_decl)))
2249 target_p = true;
2250 else
2252 omp_context *octx;
2253 for (octx = ctx; octx; octx = octx->outer)
2254 if (is_gimple_omp_offloaded (octx->stmt))
2256 target_p = true;
2257 break;
2260 if (target_p)
2261 DECL_ATTRIBUTES (decl)
2262 = tree_cons (get_identifier ("omp declare target"),
2263 NULL_TREE, DECL_ATTRIBUTES (decl));
2265 t = build_decl (DECL_SOURCE_LOCATION (decl),
2266 RESULT_DECL, NULL_TREE, void_type_node);
2267 DECL_ARTIFICIAL (t) = 1;
2268 DECL_IGNORED_P (t) = 1;
2269 DECL_CONTEXT (t) = decl;
2270 DECL_RESULT (decl) = t;
2272 /* _Cilk_for's child function requires two extra parameters called
2273 __low and __high that are set the by Cilk runtime when it calls this
2274 function. */
2275 if (cilk_for_count)
2277 t = build_decl (DECL_SOURCE_LOCATION (decl),
2278 PARM_DECL, get_identifier ("__high"), cilk_var_type);
2279 DECL_ARTIFICIAL (t) = 1;
2280 DECL_NAMELESS (t) = 1;
2281 DECL_ARG_TYPE (t) = ptr_type_node;
2282 DECL_CONTEXT (t) = current_function_decl;
2283 TREE_USED (t) = 1;
2284 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2285 DECL_ARGUMENTS (decl) = t;
2287 t = build_decl (DECL_SOURCE_LOCATION (decl),
2288 PARM_DECL, get_identifier ("__low"), cilk_var_type);
2289 DECL_ARTIFICIAL (t) = 1;
2290 DECL_NAMELESS (t) = 1;
2291 DECL_ARG_TYPE (t) = ptr_type_node;
2292 DECL_CONTEXT (t) = current_function_decl;
2293 TREE_USED (t) = 1;
2294 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2295 DECL_ARGUMENTS (decl) = t;
2298 tree data_name = get_identifier (".omp_data_i");
2299 t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name,
2300 ptr_type_node);
2301 DECL_ARTIFICIAL (t) = 1;
2302 DECL_NAMELESS (t) = 1;
2303 DECL_ARG_TYPE (t) = ptr_type_node;
2304 DECL_CONTEXT (t) = current_function_decl;
2305 TREE_USED (t) = 1;
2306 if (cilk_for_count)
2307 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2308 DECL_ARGUMENTS (decl) = t;
2309 if (!task_copy)
2310 ctx->receiver_decl = t;
2311 else
2313 t = build_decl (DECL_SOURCE_LOCATION (decl),
2314 PARM_DECL, get_identifier (".omp_data_o"),
2315 ptr_type_node);
2316 DECL_ARTIFICIAL (t) = 1;
2317 DECL_NAMELESS (t) = 1;
2318 DECL_ARG_TYPE (t) = ptr_type_node;
2319 DECL_CONTEXT (t) = current_function_decl;
2320 TREE_USED (t) = 1;
2321 TREE_ADDRESSABLE (t) = 1;
2322 DECL_CHAIN (t) = DECL_ARGUMENTS (decl);
2323 DECL_ARGUMENTS (decl) = t;
2326 /* Allocate memory for the function structure. The call to
2327 allocate_struct_function clobbers CFUN, so we need to restore
2328 it afterward. */
2329 push_struct_function (decl);
2330 cfun->function_end_locus = gimple_location (ctx->stmt);
2331 pop_cfun ();
2334 /* Callback for walk_gimple_seq. Check if combined parallel
2335 contains gimple_omp_for_combined_into_p OMP_FOR. */
2337 static tree
2338 find_combined_for (gimple_stmt_iterator *gsi_p,
2339 bool *handled_ops_p,
2340 struct walk_stmt_info *wi)
2342 gimple stmt = gsi_stmt (*gsi_p);
2344 *handled_ops_p = true;
2345 switch (gimple_code (stmt))
2347 WALK_SUBSTMTS;
2349 case GIMPLE_OMP_FOR:
2350 if (gimple_omp_for_combined_into_p (stmt)
2351 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
2353 wi->info = stmt;
2354 return integer_zero_node;
2356 break;
2357 default:
2358 break;
2360 return NULL;
2363 /* Scan an OpenACC offload directive. */
2365 static void
2366 scan_oacc_offload (gimple stmt, omp_context *outer_ctx)
2368 omp_context *ctx;
2369 tree name;
2370 void (*gimple_omp_set_child_fn) (gimple, tree);
2371 tree (*gimple_omp_clauses) (const_gimple);
2372 switch (gimple_code (stmt))
2374 case GIMPLE_OACC_KERNELS:
2375 gimple_omp_set_child_fn = gimple_oacc_kernels_set_child_fn;
2376 gimple_omp_clauses = gimple_oacc_kernels_clauses;
2377 break;
2378 case GIMPLE_OACC_PARALLEL:
2379 gimple_omp_set_child_fn = gimple_oacc_parallel_set_child_fn;
2380 gimple_omp_clauses = gimple_oacc_parallel_clauses;
2381 break;
2382 default:
2383 gcc_unreachable ();
2386 gcc_assert (taskreg_nesting_level == 0);
2387 gcc_assert (target_nesting_level == 0);
2389 ctx = new_omp_context (stmt, outer_ctx);
2390 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2391 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2392 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2393 name = create_tmp_var_name (".omp_data_t");
2394 name = build_decl (gimple_location (stmt),
2395 TYPE_DECL, name, ctx->record_type);
2396 DECL_ARTIFICIAL (name) = 1;
2397 DECL_NAMELESS (name) = 1;
2398 TYPE_NAME (ctx->record_type) = name;
2399 create_omp_child_function (ctx, false);
2400 ctx->reduction_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2402 gimple_omp_set_child_fn (stmt, ctx->cb.dst_fn);
2404 scan_sharing_clauses (gimple_omp_clauses (stmt), ctx);
2405 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2407 if (TYPE_FIELDS (ctx->record_type) == NULL)
2408 ctx->record_type = ctx->receiver_decl = NULL;
2409 else
2411 TYPE_FIELDS (ctx->record_type)
2412 = nreverse (TYPE_FIELDS (ctx->record_type));
2413 #ifdef ENABLE_CHECKING
2414 tree field;
2415 unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
2416 for (field = TYPE_FIELDS (ctx->record_type);
2417 field;
2418 field = DECL_CHAIN (field))
2419 gcc_assert (DECL_ALIGN (field) == align);
2420 #endif
2421 layout_type (ctx->record_type);
2422 fixup_child_record_type (ctx);
2426 /* Scan an OpenMP parallel directive. */
2428 static void
2429 scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2431 omp_context *ctx;
2432 tree name;
2433 gimple stmt = gsi_stmt (*gsi);
2435 /* Ignore parallel directives with empty bodies, unless there
2436 are copyin clauses. */
2437 if (optimize > 0
2438 && empty_body_p (gimple_omp_body (stmt))
2439 && find_omp_clause (gimple_omp_parallel_clauses (stmt),
2440 OMP_CLAUSE_COPYIN) == NULL)
2442 gsi_replace (gsi, gimple_build_nop (), false);
2443 return;
2446 if (gimple_omp_parallel_combined_p (stmt))
2448 gimple for_stmt;
2449 struct walk_stmt_info wi;
2451 memset (&wi, 0, sizeof (wi));
2452 wi.val_only = true;
2453 walk_gimple_seq (gimple_omp_body (stmt),
2454 find_combined_for, NULL, &wi);
2455 for_stmt = (gimple) wi.info;
2456 if (for_stmt)
2458 struct omp_for_data fd;
2459 extract_omp_for_data (for_stmt, &fd, NULL);
2460 /* We need two temporaries with fd.loop.v type (istart/iend)
2461 and then (fd.collapse - 1) temporaries with the same
2462 type for count2 ... countN-1 vars if not constant. */
2463 size_t count = 2, i;
2464 tree type = fd.iter_type;
2465 if (fd.collapse > 1
2466 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
2467 count += fd.collapse - 1;
2468 for (i = 0; i < count; i++)
2470 tree temp = create_tmp_var (type, NULL);
2471 tree c = build_omp_clause (UNKNOWN_LOCATION,
2472 OMP_CLAUSE__LOOPTEMP_);
2473 insert_decl_map (&outer_ctx->cb, temp, temp);
2474 OMP_CLAUSE_DECL (c) = temp;
2475 OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
2476 gimple_omp_parallel_set_clauses (stmt, c);
2481 ctx = new_omp_context (stmt, outer_ctx);
2482 taskreg_contexts.safe_push (ctx);
2483 if (taskreg_nesting_level > 1)
2484 ctx->is_nested = true;
2485 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2486 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2487 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2488 name = create_tmp_var_name (".omp_data_s");
2489 name = build_decl (gimple_location (stmt),
2490 TYPE_DECL, name, ctx->record_type);
2491 DECL_ARTIFICIAL (name) = 1;
2492 DECL_NAMELESS (name) = 1;
2493 TYPE_NAME (ctx->record_type) = name;
2494 create_omp_child_function (ctx, false);
2495 gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
2497 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
2498 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2500 if (TYPE_FIELDS (ctx->record_type) == NULL)
2501 ctx->record_type = ctx->receiver_decl = NULL;
2504 /* Scan an OpenMP task directive. */
2506 static void
2507 scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx)
2509 omp_context *ctx;
2510 tree name, t;
2511 gimple stmt = gsi_stmt (*gsi);
2513 /* Ignore task directives with empty bodies. */
2514 if (optimize > 0
2515 && empty_body_p (gimple_omp_body (stmt)))
2517 gsi_replace (gsi, gimple_build_nop (), false);
2518 return;
2521 ctx = new_omp_context (stmt, outer_ctx);
2522 taskreg_contexts.safe_push (ctx);
2523 if (taskreg_nesting_level > 1)
2524 ctx->is_nested = true;
2525 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2526 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2527 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2528 name = create_tmp_var_name (".omp_data_s");
2529 name = build_decl (gimple_location (stmt),
2530 TYPE_DECL, name, ctx->record_type);
2531 DECL_ARTIFICIAL (name) = 1;
2532 DECL_NAMELESS (name) = 1;
2533 TYPE_NAME (ctx->record_type) = name;
2534 create_omp_child_function (ctx, false);
2535 gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn);
2537 scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
2539 if (ctx->srecord_type)
2541 name = create_tmp_var_name (".omp_data_a");
2542 name = build_decl (gimple_location (stmt),
2543 TYPE_DECL, name, ctx->srecord_type);
2544 DECL_ARTIFICIAL (name) = 1;
2545 DECL_NAMELESS (name) = 1;
2546 TYPE_NAME (ctx->srecord_type) = name;
2547 create_omp_child_function (ctx, true);
2550 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2552 if (TYPE_FIELDS (ctx->record_type) == NULL)
2554 ctx->record_type = ctx->receiver_decl = NULL;
2555 t = build_int_cst (long_integer_type_node, 0);
2556 gimple_omp_task_set_arg_size (stmt, t);
2557 t = build_int_cst (long_integer_type_node, 1);
2558 gimple_omp_task_set_arg_align (stmt, t);
2563 /* If any decls have been made addressable during scan_omp,
2564 adjust their fields if needed, and layout record types
2565 of parallel/task constructs. */
2567 static void
2568 finish_taskreg_scan (omp_context *ctx)
2570 if (ctx->record_type == NULL_TREE)
2571 return;
2573 /* If any task_shared_vars were needed, verify all
2574 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2575 statements if use_pointer_for_field hasn't changed
2576 because of that. If it did, update field types now. */
2577 if (task_shared_vars)
2579 tree c;
2581 for (c = gimple_omp_taskreg_clauses (ctx->stmt);
2582 c; c = OMP_CLAUSE_CHAIN (c))
2583 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED)
2585 tree decl = OMP_CLAUSE_DECL (c);
2587 /* Global variables don't need to be copied,
2588 the receiver side will use them directly. */
2589 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
2590 continue;
2591 if (!bitmap_bit_p (task_shared_vars, DECL_UID (decl))
2592 || !use_pointer_for_field (decl, ctx))
2593 continue;
2594 tree field = lookup_field (decl, ctx);
2595 if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE
2596 && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl))
2597 continue;
2598 TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl));
2599 TREE_THIS_VOLATILE (field) = 0;
2600 DECL_USER_ALIGN (field) = 0;
2601 DECL_ALIGN (field) = TYPE_ALIGN (TREE_TYPE (field));
2602 if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field))
2603 TYPE_ALIGN (ctx->record_type) = DECL_ALIGN (field);
2604 if (ctx->srecord_type)
2606 tree sfield = lookup_sfield (decl, ctx);
2607 TREE_TYPE (sfield) = TREE_TYPE (field);
2608 TREE_THIS_VOLATILE (sfield) = 0;
2609 DECL_USER_ALIGN (sfield) = 0;
2610 DECL_ALIGN (sfield) = DECL_ALIGN (field);
2611 if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield))
2612 TYPE_ALIGN (ctx->srecord_type) = DECL_ALIGN (sfield);
2617 if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
2619 layout_type (ctx->record_type);
2620 fixup_child_record_type (ctx);
2622 else
2624 location_t loc = gimple_location (ctx->stmt);
2625 tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
2626 /* Move VLA fields to the end. */
2627 p = &TYPE_FIELDS (ctx->record_type);
2628 while (*p)
2629 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p))
2630 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p))))
2632 *q = *p;
2633 *p = TREE_CHAIN (*p);
2634 TREE_CHAIN (*q) = NULL_TREE;
2635 q = &TREE_CHAIN (*q);
2637 else
2638 p = &DECL_CHAIN (*p);
2639 *p = vla_fields;
2640 layout_type (ctx->record_type);
2641 fixup_child_record_type (ctx);
2642 if (ctx->srecord_type)
2643 layout_type (ctx->srecord_type);
2644 tree t = fold_convert_loc (loc, long_integer_type_node,
2645 TYPE_SIZE_UNIT (ctx->record_type));
2646 gimple_omp_task_set_arg_size (ctx->stmt, t);
2647 t = build_int_cst (long_integer_type_node,
2648 TYPE_ALIGN_UNIT (ctx->record_type));
2649 gimple_omp_task_set_arg_align (ctx->stmt, t);
2654 /* Scan a GIMPLE_OMP_FOR. */
2656 static void
2657 scan_omp_for (gimple stmt, omp_context *outer_ctx)
2659 omp_context *ctx;
2660 size_t i;
2662 ctx = new_omp_context (stmt, outer_ctx);
2664 scan_sharing_clauses (gimple_omp_for_clauses (stmt), ctx);
2666 scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
2667 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
2669 scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx);
2670 scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx);
2671 scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx);
2672 scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx);
2674 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2677 /* Scan an OpenMP sections directive. */
2679 static void
2680 scan_omp_sections (gimple stmt, omp_context *outer_ctx)
2682 omp_context *ctx;
2684 ctx = new_omp_context (stmt, outer_ctx);
2685 scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx);
2686 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2689 /* Scan an OpenMP single directive. */
2691 static void
2692 scan_omp_single (gimple stmt, omp_context *outer_ctx)
2694 omp_context *ctx;
2695 tree name;
2697 ctx = new_omp_context (stmt, outer_ctx);
2698 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2699 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2700 name = create_tmp_var_name (".omp_copy_s");
2701 name = build_decl (gimple_location (stmt),
2702 TYPE_DECL, name, ctx->record_type);
2703 TYPE_NAME (ctx->record_type) = name;
2705 scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx);
2706 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2708 if (TYPE_FIELDS (ctx->record_type) == NULL)
2709 ctx->record_type = NULL;
2710 else
2711 layout_type (ctx->record_type);
2714 /* Scan a GIMPLE_OMP_TARGET. */
2716 static void
2717 scan_omp_target (gimple stmt, omp_context *outer_ctx)
2719 omp_context *ctx;
2720 tree name;
2721 int kind = gimple_omp_target_kind (stmt);
2723 if (kind == GF_OMP_TARGET_KIND_OACC_DATA
2724 || kind == GF_OMP_TARGET_KIND_OACC_UPDATE)
2726 gcc_assert (taskreg_nesting_level == 0);
2727 gcc_assert (target_nesting_level == 0);
2730 ctx = new_omp_context (stmt, outer_ctx);
2731 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
2732 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
2733 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
2734 name = create_tmp_var_name (".omp_data_t");
2735 name = build_decl (gimple_location (stmt),
2736 TYPE_DECL, name, ctx->record_type);
2737 DECL_ARTIFICIAL (name) = 1;
2738 DECL_NAMELESS (name) = 1;
2739 TYPE_NAME (ctx->record_type) = name;
2740 if (kind == GF_OMP_TARGET_KIND_REGION)
2742 create_omp_child_function (ctx, false);
2743 gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
2746 scan_sharing_clauses (gimple_omp_target_clauses (stmt), ctx);
2747 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2749 if (TYPE_FIELDS (ctx->record_type) == NULL)
2750 ctx->record_type = ctx->receiver_decl = NULL;
2751 else
2753 TYPE_FIELDS (ctx->record_type)
2754 = nreverse (TYPE_FIELDS (ctx->record_type));
2755 #ifdef ENABLE_CHECKING
2756 tree field;
2757 unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type));
2758 for (field = TYPE_FIELDS (ctx->record_type);
2759 field;
2760 field = DECL_CHAIN (field))
2761 gcc_assert (DECL_ALIGN (field) == align);
2762 #endif
2763 layout_type (ctx->record_type);
2764 if (kind == GF_OMP_TARGET_KIND_REGION)
2765 fixup_child_record_type (ctx);
2769 /* Scan an OpenMP teams directive. */
2771 static void
2772 scan_omp_teams (gimple stmt, omp_context *outer_ctx)
2774 omp_context *ctx = new_omp_context (stmt, outer_ctx);
2775 scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
2776 scan_omp (gimple_omp_body_ptr (stmt), ctx);
2779 /* Check OpenMP nesting restrictions. */
2780 static bool
2781 check_omp_nesting_restrictions (gimple stmt, omp_context *ctx)
2783 /* TODO: While the OpenACC specification does allow for certain kinds of
2784 nesting, we don't support many of these yet. */
2785 if (is_gimple_omp (stmt)
2786 && is_gimple_omp_oacc_specifically (stmt))
2788 /* Regular handling of OpenACC loop constructs. */
2789 if (gimple_code (stmt) == GIMPLE_OMP_FOR
2790 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
2791 goto cont;
2792 /* No nesting of OpenACC STMT inside any OpenACC or OpenMP CTX different
2793 from an OpenACC data construct. */
2794 for (omp_context *ctx_ = ctx; ctx_ != NULL; ctx_ = ctx_->outer)
2795 if (is_gimple_omp (ctx_->stmt)
2796 && !(gimple_code (ctx_->stmt) == GIMPLE_OMP_TARGET
2797 && (gimple_omp_target_kind (ctx_->stmt)
2798 == GF_OMP_TARGET_KIND_OACC_DATA)))
2800 error_at (gimple_location (stmt),
2801 "may not be nested");
2802 return false;
2805 else
2807 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP
2808 builtin) inside any OpenACC CTX. */
2809 for (omp_context *ctx_ = ctx; ctx_ != NULL; ctx_ = ctx_->outer)
2810 if (is_gimple_omp (ctx_->stmt)
2811 && is_gimple_omp_oacc_specifically (ctx_->stmt))
2813 error_at (gimple_location (stmt),
2814 "may not be nested");
2815 return false;
2818 cont:
2820 if (ctx != NULL)
2822 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
2823 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
2825 error_at (gimple_location (stmt),
2826 "OpenMP constructs may not be nested inside simd region");
2827 return false;
2829 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
2831 if ((gimple_code (stmt) != GIMPLE_OMP_FOR
2832 || (gimple_omp_for_kind (stmt)
2833 != GF_OMP_FOR_KIND_DISTRIBUTE))
2834 && gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
2836 error_at (gimple_location (stmt),
2837 "only distribute or parallel constructs are allowed to "
2838 "be closely nested inside teams construct");
2839 return false;
2843 switch (gimple_code (stmt))
2845 case GIMPLE_OMP_FOR:
2846 if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_SIMD)
2847 return true;
2848 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
2850 if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS)
2852 error_at (gimple_location (stmt),
2853 "distribute construct must be closely nested inside "
2854 "teams construct");
2855 return false;
2857 return true;
2859 /* FALLTHRU */
2860 case GIMPLE_CALL:
2861 if (is_gimple_call (stmt)
2862 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2863 == BUILT_IN_GOMP_CANCEL
2864 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2865 == BUILT_IN_GOMP_CANCELLATION_POINT))
2867 const char *bad = NULL;
2868 const char *kind = NULL;
2869 if (ctx == NULL)
2871 error_at (gimple_location (stmt), "orphaned %qs construct",
2872 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2873 == BUILT_IN_GOMP_CANCEL
2874 ? "#pragma omp cancel"
2875 : "#pragma omp cancellation point");
2876 return false;
2878 switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0))
2879 ? tree_to_shwi (gimple_call_arg (stmt, 0))
2880 : 0)
2882 case 1:
2883 if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
2884 bad = "#pragma omp parallel";
2885 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2886 == BUILT_IN_GOMP_CANCEL
2887 && !integer_zerop (gimple_call_arg (stmt, 1)))
2888 ctx->cancellable = true;
2889 kind = "parallel";
2890 break;
2891 case 2:
2892 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
2893 || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
2894 bad = "#pragma omp for";
2895 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2896 == BUILT_IN_GOMP_CANCEL
2897 && !integer_zerop (gimple_call_arg (stmt, 1)))
2899 ctx->cancellable = true;
2900 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2901 OMP_CLAUSE_NOWAIT))
2902 warning_at (gimple_location (stmt), 0,
2903 "%<#pragma omp cancel for%> inside "
2904 "%<nowait%> for construct");
2905 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
2906 OMP_CLAUSE_ORDERED))
2907 warning_at (gimple_location (stmt), 0,
2908 "%<#pragma omp cancel for%> inside "
2909 "%<ordered%> for construct");
2911 kind = "for";
2912 break;
2913 case 4:
2914 if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
2915 && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
2916 bad = "#pragma omp sections";
2917 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2918 == BUILT_IN_GOMP_CANCEL
2919 && !integer_zerop (gimple_call_arg (stmt, 1)))
2921 if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
2923 ctx->cancellable = true;
2924 if (find_omp_clause (gimple_omp_sections_clauses
2925 (ctx->stmt),
2926 OMP_CLAUSE_NOWAIT))
2927 warning_at (gimple_location (stmt), 0,
2928 "%<#pragma omp cancel sections%> inside "
2929 "%<nowait%> sections construct");
2931 else
2933 gcc_assert (ctx->outer
2934 && gimple_code (ctx->outer->stmt)
2935 == GIMPLE_OMP_SECTIONS);
2936 ctx->outer->cancellable = true;
2937 if (find_omp_clause (gimple_omp_sections_clauses
2938 (ctx->outer->stmt),
2939 OMP_CLAUSE_NOWAIT))
2940 warning_at (gimple_location (stmt), 0,
2941 "%<#pragma omp cancel sections%> inside "
2942 "%<nowait%> sections construct");
2945 kind = "sections";
2946 break;
2947 case 8:
2948 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK)
2949 bad = "#pragma omp task";
2950 else
2951 ctx->cancellable = true;
2952 kind = "taskgroup";
2953 break;
2954 default:
2955 error_at (gimple_location (stmt), "invalid arguments");
2956 return false;
2958 if (bad)
2960 error_at (gimple_location (stmt),
2961 "%<%s %s%> construct not closely nested inside of %qs",
2962 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2963 == BUILT_IN_GOMP_CANCEL
2964 ? "#pragma omp cancel"
2965 : "#pragma omp cancellation point", kind, bad);
2966 return false;
2969 /* FALLTHRU */
2970 case GIMPLE_OMP_SECTIONS:
2971 case GIMPLE_OMP_SINGLE:
2972 for (; ctx != NULL; ctx = ctx->outer)
2973 switch (gimple_code (ctx->stmt))
2975 case GIMPLE_OMP_FOR:
2976 case GIMPLE_OMP_SECTIONS:
2977 case GIMPLE_OMP_SINGLE:
2978 case GIMPLE_OMP_ORDERED:
2979 case GIMPLE_OMP_MASTER:
2980 case GIMPLE_OMP_TASK:
2981 case GIMPLE_OMP_CRITICAL:
2982 if (is_gimple_call (stmt))
2984 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
2985 != BUILT_IN_GOMP_BARRIER)
2986 return true;
2987 error_at (gimple_location (stmt),
2988 "barrier region may not be closely nested inside "
2989 "of work-sharing, critical, ordered, master or "
2990 "explicit task region");
2991 return false;
2993 error_at (gimple_location (stmt),
2994 "work-sharing region may not be closely nested inside "
2995 "of work-sharing, critical, ordered, master or explicit "
2996 "task region");
2997 return false;
2998 case GIMPLE_OACC_KERNELS:
2999 case GIMPLE_OACC_PARALLEL:
3000 case GIMPLE_OMP_PARALLEL:
3001 return true;
3002 default:
3003 break;
3005 break;
3006 case GIMPLE_OMP_MASTER:
3007 for (; ctx != NULL; ctx = ctx->outer)
3008 switch (gimple_code (ctx->stmt))
3010 case GIMPLE_OMP_FOR:
3011 case GIMPLE_OMP_SECTIONS:
3012 case GIMPLE_OMP_SINGLE:
3013 case GIMPLE_OMP_TASK:
3014 error_at (gimple_location (stmt),
3015 "master region may not be closely nested inside "
3016 "of work-sharing or explicit task region");
3017 return false;
3018 case GIMPLE_OMP_PARALLEL:
3019 return true;
3020 default:
3021 break;
3023 break;
3024 case GIMPLE_OMP_ORDERED:
3025 for (; ctx != NULL; ctx = ctx->outer)
3026 switch (gimple_code (ctx->stmt))
3028 case GIMPLE_OMP_CRITICAL:
3029 case GIMPLE_OMP_TASK:
3030 error_at (gimple_location (stmt),
3031 "ordered region may not be closely nested inside "
3032 "of critical or explicit task region");
3033 return false;
3034 case GIMPLE_OMP_FOR:
3035 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3036 OMP_CLAUSE_ORDERED) == NULL)
3038 error_at (gimple_location (stmt),
3039 "ordered region must be closely nested inside "
3040 "a loop region with an ordered clause");
3041 return false;
3043 return true;
3044 case GIMPLE_OMP_PARALLEL:
3045 error_at (gimple_location (stmt),
3046 "ordered region must be closely nested inside "
3047 "a loop region with an ordered clause");
3048 return false;
3049 default:
3050 break;
3052 break;
3053 case GIMPLE_OMP_CRITICAL:
3054 for (; ctx != NULL; ctx = ctx->outer)
3055 if (gimple_code (ctx->stmt) == GIMPLE_OMP_CRITICAL
3056 && (gimple_omp_critical_name (stmt)
3057 == gimple_omp_critical_name (ctx->stmt)))
3059 error_at (gimple_location (stmt),
3060 "critical region may not be nested inside a critical "
3061 "region with the same name");
3062 return false;
3064 break;
3065 case GIMPLE_OMP_TEAMS:
3066 if (ctx == NULL
3067 || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
3068 || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION)
3070 error_at (gimple_location (stmt),
3071 "teams construct not closely nested inside of target "
3072 "region");
3073 return false;
3075 break;
3076 case GIMPLE_OMP_TARGET:
3077 for (; ctx != NULL; ctx = ctx->outer)
3078 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET
3079 && gimple_omp_target_kind (ctx->stmt) == GF_OMP_TARGET_KIND_REGION)
3081 const char *name;
3082 switch (gimple_omp_target_kind (stmt))
3084 case GF_OMP_TARGET_KIND_REGION: name = "target"; break;
3085 case GF_OMP_TARGET_KIND_DATA: name = "target data"; break;
3086 case GF_OMP_TARGET_KIND_UPDATE: name = "target update"; break;
3087 default: gcc_unreachable ();
3089 warning_at (gimple_location (stmt), 0,
3090 "%s construct inside of target region", name);
3092 break;
3093 default:
3094 break;
3096 return true;
3100 /* Helper function scan_omp.
3102 Callback for walk_tree or operators in walk_gimple_stmt used to
3103 scan for OpenMP directives in TP. */
3105 static tree
3106 scan_omp_1_op (tree *tp, int *walk_subtrees, void *data)
3108 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
3109 omp_context *ctx = (omp_context *) wi->info;
3110 tree t = *tp;
3112 switch (TREE_CODE (t))
3114 case VAR_DECL:
3115 case PARM_DECL:
3116 case LABEL_DECL:
3117 case RESULT_DECL:
3118 if (ctx)
3119 *tp = remap_decl (t, &ctx->cb);
3120 break;
3122 default:
3123 if (ctx && TYPE_P (t))
3124 *tp = remap_type (t, &ctx->cb);
3125 else if (!DECL_P (t))
3127 *walk_subtrees = 1;
3128 if (ctx)
3130 tree tem = remap_type (TREE_TYPE (t), &ctx->cb);
3131 if (tem != TREE_TYPE (t))
3133 if (TREE_CODE (t) == INTEGER_CST)
3134 *tp = wide_int_to_tree (tem, t);
3135 else
3136 TREE_TYPE (t) = tem;
3140 break;
3143 return NULL_TREE;
3146 /* Return true if FNDECL is a setjmp or a longjmp. */
3148 static bool
3149 setjmp_or_longjmp_p (const_tree fndecl)
3151 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
3152 && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP
3153 || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP))
3154 return true;
3156 tree declname = DECL_NAME (fndecl);
3157 if (!declname)
3158 return false;
3159 const char *name = IDENTIFIER_POINTER (declname);
3160 return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
3164 /* Helper function for scan_omp.
3166 Callback for walk_gimple_stmt used to scan for OpenMP directives in
3167 the current statement in GSI. */
3169 static tree
3170 scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
3171 struct walk_stmt_info *wi)
3173 gimple stmt = gsi_stmt (*gsi);
3174 omp_context *ctx = (omp_context *) wi->info;
3176 if (gimple_has_location (stmt))
3177 input_location = gimple_location (stmt);
3179 /* Check the OpenMP nesting restrictions. */
3180 bool remove = false;
3181 if (is_gimple_omp (stmt))
3182 remove = !check_omp_nesting_restrictions (stmt, ctx);
3183 else if (is_gimple_call (stmt))
3185 tree fndecl = gimple_call_fndecl (stmt);
3186 if (fndecl)
3188 if (setjmp_or_longjmp_p (fndecl)
3189 && ctx
3190 && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3191 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
3193 remove = true;
3194 error_at (gimple_location (stmt),
3195 "setjmp/longjmp inside simd construct");
3197 else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
3198 switch (DECL_FUNCTION_CODE (fndecl))
3200 case BUILT_IN_GOMP_BARRIER:
3201 case BUILT_IN_GOMP_CANCEL:
3202 case BUILT_IN_GOMP_CANCELLATION_POINT:
3203 case BUILT_IN_GOMP_TASKYIELD:
3204 case BUILT_IN_GOMP_TASKWAIT:
3205 case BUILT_IN_GOMP_TASKGROUP_START:
3206 case BUILT_IN_GOMP_TASKGROUP_END:
3207 remove = !check_omp_nesting_restrictions (stmt, ctx);
3208 break;
3209 default:
3210 break;
3214 if (remove)
3216 stmt = gimple_build_nop ();
3217 gsi_replace (gsi, stmt, false);
3220 *handled_ops_p = true;
3222 switch (gimple_code (stmt))
3224 case GIMPLE_OACC_KERNELS:
3225 case GIMPLE_OACC_PARALLEL:
3226 scan_oacc_offload (stmt, ctx);
3227 break;
3229 case GIMPLE_OMP_PARALLEL:
3230 taskreg_nesting_level++;
3231 scan_omp_parallel (gsi, ctx);
3232 taskreg_nesting_level--;
3233 break;
3235 case GIMPLE_OMP_TASK:
3236 taskreg_nesting_level++;
3237 scan_omp_task (gsi, ctx);
3238 taskreg_nesting_level--;
3239 break;
3241 case GIMPLE_OMP_FOR:
3242 scan_omp_for (stmt, ctx);
3243 break;
3245 case GIMPLE_OMP_SECTIONS:
3246 scan_omp_sections (stmt, ctx);
3247 break;
3249 case GIMPLE_OMP_SINGLE:
3250 scan_omp_single (stmt, ctx);
3251 break;
3253 case GIMPLE_OMP_SECTION:
3254 case GIMPLE_OMP_MASTER:
3255 case GIMPLE_OMP_TASKGROUP:
3256 case GIMPLE_OMP_ORDERED:
3257 case GIMPLE_OMP_CRITICAL:
3258 ctx = new_omp_context (stmt, ctx);
3259 scan_omp (gimple_omp_body_ptr (stmt), ctx);
3260 break;
3262 case GIMPLE_OMP_TARGET:
3263 scan_omp_target (stmt, ctx);
3264 break;
3266 case GIMPLE_OMP_TEAMS:
3267 scan_omp_teams (stmt, ctx);
3268 break;
3270 case GIMPLE_BIND:
3272 tree var;
3274 *handled_ops_p = false;
3275 if (ctx)
3276 for (var = gimple_bind_vars (stmt); var ; var = DECL_CHAIN (var))
3277 insert_decl_map (&ctx->cb, var, var);
3279 break;
3280 default:
3281 *handled_ops_p = false;
3282 break;
3285 return NULL_TREE;
3289 /* Scan all the statements starting at the current statement. CTX
3290 contains context information about the OpenMP directives and
3291 clauses found during the scan. */
3293 static void
3294 scan_omp (gimple_seq *body_p, omp_context *ctx)
3296 location_t saved_location;
3297 struct walk_stmt_info wi;
3299 memset (&wi, 0, sizeof (wi));
3300 wi.info = ctx;
3301 wi.want_locations = true;
3303 saved_location = input_location;
3304 walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi);
3305 input_location = saved_location;
3308 /* Re-gimplification and code generation routines. */
3310 /* Build a call to GOMP_barrier. */
3312 static gimple
3313 build_omp_barrier (tree lhs)
3315 tree fndecl = builtin_decl_explicit (lhs ? BUILT_IN_GOMP_BARRIER_CANCEL
3316 : BUILT_IN_GOMP_BARRIER);
3317 gimple g = gimple_build_call (fndecl, 0);
3318 if (lhs)
3319 gimple_call_set_lhs (g, lhs);
3320 return g;
3323 /* If a context was created for STMT when it was scanned, return it. */
3325 static omp_context *
3326 maybe_lookup_ctx (gimple stmt)
3328 splay_tree_node n;
3329 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
3330 return n ? (omp_context *) n->value : NULL;
3334 /* Find the mapping for DECL in CTX or the immediately enclosing
3335 context that has a mapping for DECL.
3337 If CTX is a nested parallel directive, we may have to use the decl
3338 mappings created in CTX's parent context. Suppose that we have the
3339 following parallel nesting (variable UIDs showed for clarity):
3341 iD.1562 = 0;
3342 #omp parallel shared(iD.1562) -> outer parallel
3343 iD.1562 = iD.1562 + 1;
3345 #omp parallel shared (iD.1562) -> inner parallel
3346 iD.1562 = iD.1562 - 1;
3348 Each parallel structure will create a distinct .omp_data_s structure
3349 for copying iD.1562 in/out of the directive:
3351 outer parallel .omp_data_s.1.i -> iD.1562
3352 inner parallel .omp_data_s.2.i -> iD.1562
3354 A shared variable mapping will produce a copy-out operation before
3355 the parallel directive and a copy-in operation after it. So, in
3356 this case we would have:
3358 iD.1562 = 0;
3359 .omp_data_o.1.i = iD.1562;
3360 #omp parallel shared(iD.1562) -> outer parallel
3361 .omp_data_i.1 = &.omp_data_o.1
3362 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3364 .omp_data_o.2.i = iD.1562; -> **
3365 #omp parallel shared(iD.1562) -> inner parallel
3366 .omp_data_i.2 = &.omp_data_o.2
3367 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3370 ** This is a problem. The symbol iD.1562 cannot be referenced
3371 inside the body of the outer parallel region. But since we are
3372 emitting this copy operation while expanding the inner parallel
3373 directive, we need to access the CTX structure of the outer
3374 parallel directive to get the correct mapping:
3376 .omp_data_o.2.i = .omp_data_i.1->i
3378 Since there may be other workshare or parallel directives enclosing
3379 the parallel directive, it may be necessary to walk up the context
3380 parent chain. This is not a problem in general because nested
3381 parallelism happens only rarely. */
3383 static tree
3384 lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
3386 tree t;
3387 omp_context *up;
3389 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
3390 t = maybe_lookup_decl (decl, up);
3392 gcc_assert (!ctx->is_nested || t || is_global_var (decl));
3394 return t ? t : decl;
3398 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3399 in outer contexts. */
3401 static tree
3402 maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
3404 tree t = NULL;
3405 omp_context *up;
3407 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
3408 t = maybe_lookup_decl (decl, up);
3410 return t ? t : decl;
3414 /* Construct the initialization value for reduction CLAUSE. */
3416 tree
3417 omp_reduction_init (tree clause, tree type)
3419 location_t loc = OMP_CLAUSE_LOCATION (clause);
3420 switch (OMP_CLAUSE_REDUCTION_CODE (clause))
3422 case PLUS_EXPR:
3423 case MINUS_EXPR:
3424 case BIT_IOR_EXPR:
3425 case BIT_XOR_EXPR:
3426 case TRUTH_OR_EXPR:
3427 case TRUTH_ORIF_EXPR:
3428 case TRUTH_XOR_EXPR:
3429 case NE_EXPR:
3430 return build_zero_cst (type);
3432 case MULT_EXPR:
3433 case TRUTH_AND_EXPR:
3434 case TRUTH_ANDIF_EXPR:
3435 case EQ_EXPR:
3436 return fold_convert_loc (loc, type, integer_one_node);
3438 case BIT_AND_EXPR:
3439 return fold_convert_loc (loc, type, integer_minus_one_node);
3441 case MAX_EXPR:
3442 if (SCALAR_FLOAT_TYPE_P (type))
3444 REAL_VALUE_TYPE max, min;
3445 if (HONOR_INFINITIES (TYPE_MODE (type)))
3447 real_inf (&max);
3448 real_arithmetic (&min, NEGATE_EXPR, &max, NULL);
3450 else
3451 real_maxval (&min, 1, TYPE_MODE (type));
3452 return build_real (type, min);
3454 else
3456 gcc_assert (INTEGRAL_TYPE_P (type));
3457 return TYPE_MIN_VALUE (type);
3460 case MIN_EXPR:
3461 if (SCALAR_FLOAT_TYPE_P (type))
3463 REAL_VALUE_TYPE max;
3464 if (HONOR_INFINITIES (TYPE_MODE (type)))
3465 real_inf (&max);
3466 else
3467 real_maxval (&max, 0, TYPE_MODE (type));
3468 return build_real (type, max);
3470 else
3472 gcc_assert (INTEGRAL_TYPE_P (type));
3473 return TYPE_MAX_VALUE (type);
3476 default:
3477 gcc_unreachable ();
3481 /* Return alignment to be assumed for var in CLAUSE, which should be
3482 OMP_CLAUSE_ALIGNED. */
3484 static tree
3485 omp_clause_aligned_alignment (tree clause)
3487 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause))
3488 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause);
3490 /* Otherwise return implementation defined alignment. */
3491 unsigned int al = 1;
3492 machine_mode mode, vmode;
3493 int vs = targetm.vectorize.autovectorize_vector_sizes ();
3494 if (vs)
3495 vs = 1 << floor_log2 (vs);
3496 static enum mode_class classes[]
3497 = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
3498 for (int i = 0; i < 4; i += 2)
3499 for (mode = GET_CLASS_NARROWEST_MODE (classes[i]);
3500 mode != VOIDmode;
3501 mode = GET_MODE_WIDER_MODE (mode))
3503 vmode = targetm.vectorize.preferred_simd_mode (mode);
3504 if (GET_MODE_CLASS (vmode) != classes[i + 1])
3505 continue;
3506 while (vs
3507 && GET_MODE_SIZE (vmode) < vs
3508 && GET_MODE_2XWIDER_MODE (vmode) != VOIDmode)
3509 vmode = GET_MODE_2XWIDER_MODE (vmode);
3511 tree type = lang_hooks.types.type_for_mode (mode, 1);
3512 if (type == NULL_TREE || TYPE_MODE (type) != mode)
3513 continue;
3514 type = build_vector_type (type, GET_MODE_SIZE (vmode)
3515 / GET_MODE_SIZE (mode));
3516 if (TYPE_MODE (type) != vmode)
3517 continue;
3518 if (TYPE_ALIGN_UNIT (type) > al)
3519 al = TYPE_ALIGN_UNIT (type);
3521 return build_int_cst (integer_type_node, al);
3524 /* Return maximum possible vectorization factor for the target. */
3526 static int
3527 omp_max_vf (void)
3529 if (!optimize
3530 || optimize_debug
3531 || !flag_tree_loop_optimize
3532 || (!flag_tree_loop_vectorize
3533 && (global_options_set.x_flag_tree_loop_vectorize
3534 || global_options_set.x_flag_tree_vectorize)))
3535 return 1;
3537 int vs = targetm.vectorize.autovectorize_vector_sizes ();
3538 if (vs)
3540 vs = 1 << floor_log2 (vs);
3541 return vs;
3543 machine_mode vqimode = targetm.vectorize.preferred_simd_mode (QImode);
3544 if (GET_MODE_CLASS (vqimode) == MODE_VECTOR_INT)
3545 return GET_MODE_NUNITS (vqimode);
3546 return 1;
3549 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3550 privatization. */
3552 static bool
3553 lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, int &max_vf,
3554 tree &idx, tree &lane, tree &ivar, tree &lvar)
3556 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
3558 if (max_vf == 0)
3560 max_vf = omp_max_vf ();
3561 if (max_vf > 1)
3563 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
3564 OMP_CLAUSE_SAFELEN);
3565 if (c && TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) != INTEGER_CST)
3566 max_vf = 1;
3567 else if (c && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
3568 max_vf) == -1)
3569 max_vf = tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c));
3571 if (max_vf > 1)
3573 idx = create_tmp_var (unsigned_type_node, NULL);
3574 lane = create_tmp_var (unsigned_type_node, NULL);
3577 if (max_vf == 1)
3578 return false;
3580 tree atype = build_array_type_nelts (TREE_TYPE (new_var), max_vf);
3581 tree avar = create_tmp_var_raw (atype, NULL);
3582 if (TREE_ADDRESSABLE (new_var))
3583 TREE_ADDRESSABLE (avar) = 1;
3584 DECL_ATTRIBUTES (avar)
3585 = tree_cons (get_identifier ("omp simd array"), NULL,
3586 DECL_ATTRIBUTES (avar));
3587 gimple_add_tmp_var (avar);
3588 ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, idx,
3589 NULL_TREE, NULL_TREE);
3590 lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, lane,
3591 NULL_TREE, NULL_TREE);
3592 if (DECL_P (new_var))
3594 SET_DECL_VALUE_EXPR (new_var, lvar);
3595 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3597 return true;
3600 /* Helper function of lower_rec_input_clauses. For a reference
3601 in simd reduction, add an underlying variable it will reference. */
3603 static void
3604 handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist)
3606 tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard)));
3607 if (TREE_CONSTANT (z))
3609 const char *name = NULL;
3610 if (DECL_NAME (new_vard))
3611 name = IDENTIFIER_POINTER (DECL_NAME (new_vard));
3613 z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)), name);
3614 gimple_add_tmp_var (z);
3615 TREE_ADDRESSABLE (z) = 1;
3616 z = build_fold_addr_expr_loc (loc, z);
3617 gimplify_assign (new_vard, z, ilist);
3621 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3622 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3623 private variables. Initialization statements go in ILIST, while calls
3624 to destructors go in DLIST. */
3626 static void
3627 lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
3628 omp_context *ctx, struct omp_for_data *fd)
3630 tree c, dtor, copyin_seq, x, ptr;
3631 bool copyin_by_ref = false;
3632 bool lastprivate_firstprivate = false;
3633 bool reduction_omp_orig_ref = false;
3634 int pass;
3635 bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
3636 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD);
3637 int max_vf = 0;
3638 tree lane = NULL_TREE, idx = NULL_TREE;
3639 tree ivar = NULL_TREE, lvar = NULL_TREE;
3640 gimple_seq llist[2] = { NULL, NULL };
3642 copyin_seq = NULL;
3644 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3645 with data sharing clauses referencing variable sized vars. That
3646 is unnecessarily hard to support and very unlikely to result in
3647 vectorized code anyway. */
3648 if (is_simd)
3649 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3650 switch (OMP_CLAUSE_CODE (c))
3652 case OMP_CLAUSE_LINEAR:
3653 if (OMP_CLAUSE_LINEAR_ARRAY (c))
3654 max_vf = 1;
3655 /* FALLTHRU */
3656 case OMP_CLAUSE_REDUCTION:
3657 case OMP_CLAUSE_PRIVATE:
3658 case OMP_CLAUSE_FIRSTPRIVATE:
3659 case OMP_CLAUSE_LASTPRIVATE:
3660 if (is_variable_sized (OMP_CLAUSE_DECL (c)))
3661 max_vf = 1;
3662 break;
3663 default:
3664 continue;
3667 /* Do all the fixed sized types in the first pass, and the variable sized
3668 types in the second pass. This makes sure that the scalar arguments to
3669 the variable sized types are processed before we use them in the
3670 variable sized operations. */
3671 for (pass = 0; pass < 2; ++pass)
3673 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
3675 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
3676 tree var, new_var;
3677 bool by_ref;
3678 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
3680 switch (c_kind)
3682 case OMP_CLAUSE_PRIVATE:
3683 if (OMP_CLAUSE_PRIVATE_DEBUG (c))
3684 continue;
3685 break;
3686 case OMP_CLAUSE_SHARED:
3687 /* Ignore shared directives in teams construct. */
3688 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3689 continue;
3690 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
3692 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c)));
3693 continue;
3695 case OMP_CLAUSE_FIRSTPRIVATE:
3696 case OMP_CLAUSE_COPYIN:
3697 case OMP_CLAUSE_LINEAR:
3698 break;
3699 case OMP_CLAUSE_REDUCTION:
3700 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
3701 reduction_omp_orig_ref = true;
3702 break;
3703 case OMP_CLAUSE__LOOPTEMP_:
3704 /* Handle _looptemp_ clauses only on parallel. */
3705 if (fd)
3706 continue;
3707 break;
3708 case OMP_CLAUSE_LASTPRIVATE:
3709 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3711 lastprivate_firstprivate = true;
3712 if (pass != 0)
3713 continue;
3715 /* Even without corresponding firstprivate, if
3716 decl is Fortran allocatable, it needs outer var
3717 reference. */
3718 else if (pass == 0
3719 && lang_hooks.decls.omp_private_outer_ref
3720 (OMP_CLAUSE_DECL (c)))
3721 lastprivate_firstprivate = true;
3722 break;
3723 case OMP_CLAUSE_ALIGNED:
3724 if (pass == 0)
3725 continue;
3726 var = OMP_CLAUSE_DECL (c);
3727 if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
3728 && !is_global_var (var))
3730 new_var = maybe_lookup_decl (var, ctx);
3731 if (new_var == NULL_TREE)
3732 new_var = maybe_lookup_decl_in_outer_ctx (var, ctx);
3733 x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3734 x = build_call_expr_loc (clause_loc, x, 2, new_var,
3735 omp_clause_aligned_alignment (c));
3736 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3737 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
3738 gimplify_and_add (x, ilist);
3740 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE
3741 && is_global_var (var))
3743 tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2;
3744 new_var = lookup_decl (var, ctx);
3745 t = maybe_lookup_decl_in_outer_ctx (var, ctx);
3746 t = build_fold_addr_expr_loc (clause_loc, t);
3747 t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
3748 t = build_call_expr_loc (clause_loc, t2, 2, t,
3749 omp_clause_aligned_alignment (c));
3750 t = fold_convert_loc (clause_loc, ptype, t);
3751 x = create_tmp_var (ptype, NULL);
3752 t = build2 (MODIFY_EXPR, ptype, x, t);
3753 gimplify_and_add (t, ilist);
3754 t = build_simple_mem_ref_loc (clause_loc, x);
3755 SET_DECL_VALUE_EXPR (new_var, t);
3756 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3758 continue;
3759 default:
3760 continue;
3763 new_var = var = OMP_CLAUSE_DECL (c);
3764 if (c_kind != OMP_CLAUSE_COPYIN)
3765 new_var = lookup_decl (var, ctx);
3767 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
3769 if (pass != 0)
3770 continue;
3772 else if (is_variable_sized (var))
3774 /* For variable sized types, we need to allocate the
3775 actual storage here. Call alloca and store the
3776 result in the pointer decl that we created elsewhere. */
3777 if (pass == 0)
3778 continue;
3780 if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
3782 gimple stmt;
3783 tree tmp, atmp;
3785 ptr = DECL_VALUE_EXPR (new_var);
3786 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
3787 ptr = TREE_OPERAND (ptr, 0);
3788 gcc_assert (DECL_P (ptr));
3789 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
3791 /* void *tmp = __builtin_alloca */
3792 atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3793 stmt = gimple_build_call (atmp, 1, x);
3794 tmp = create_tmp_var_raw (ptr_type_node, NULL);
3795 gimple_add_tmp_var (tmp);
3796 gimple_call_set_lhs (stmt, tmp);
3798 gimple_seq_add_stmt (ilist, stmt);
3800 x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
3801 gimplify_assign (ptr, x, ilist);
3804 else if (is_reference (var))
3806 /* For references that are being privatized for Fortran,
3807 allocate new backing storage for the new pointer
3808 variable. This allows us to avoid changing all the
3809 code that expects a pointer to something that expects
3810 a direct variable. */
3811 if (pass == 0)
3812 continue;
3814 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
3815 if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
3817 x = build_receiver_ref (var, false, ctx);
3818 x = build_fold_addr_expr_loc (clause_loc, x);
3820 else if (TREE_CONSTANT (x))
3822 /* For reduction in SIMD loop, defer adding the
3823 initialization of the reference, because if we decide
3824 to use SIMD array for it, the initilization could cause
3825 expansion ICE. */
3826 if (c_kind == OMP_CLAUSE_REDUCTION && is_simd)
3827 x = NULL_TREE;
3828 else
3830 const char *name = NULL;
3831 if (DECL_NAME (var))
3832 name = IDENTIFIER_POINTER (DECL_NAME (new_var));
3834 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
3835 name);
3836 gimple_add_tmp_var (x);
3837 TREE_ADDRESSABLE (x) = 1;
3838 x = build_fold_addr_expr_loc (clause_loc, x);
3841 else
3843 tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA);
3844 x = build_call_expr_loc (clause_loc, atmp, 1, x);
3847 if (x)
3849 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
3850 gimplify_assign (new_var, x, ilist);
3853 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
3855 else if (c_kind == OMP_CLAUSE_REDUCTION
3856 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
3858 if (pass == 0)
3859 continue;
3861 else if (pass != 0)
3862 continue;
3864 switch (OMP_CLAUSE_CODE (c))
3866 case OMP_CLAUSE_SHARED:
3867 /* Ignore shared directives in teams construct. */
3868 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
3869 continue;
3870 /* Shared global vars are just accessed directly. */
3871 if (is_global_var (new_var))
3872 break;
3873 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3874 needs to be delayed until after fixup_child_record_type so
3875 that we get the correct type during the dereference. */
3876 by_ref = use_pointer_for_field (var, ctx);
3877 x = build_receiver_ref (var, by_ref, ctx);
3878 SET_DECL_VALUE_EXPR (new_var, x);
3879 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3881 /* ??? If VAR is not passed by reference, and the variable
3882 hasn't been initialized yet, then we'll get a warning for
3883 the store into the omp_data_s structure. Ideally, we'd be
3884 able to notice this and not store anything at all, but
3885 we're generating code too early. Suppress the warning. */
3886 if (!by_ref)
3887 TREE_NO_WARNING (var) = 1;
3888 break;
3890 case OMP_CLAUSE_LASTPRIVATE:
3891 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
3892 break;
3893 /* FALLTHRU */
3895 case OMP_CLAUSE_PRIVATE:
3896 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE)
3897 x = build_outer_var_ref (var, ctx);
3898 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
3900 if (is_task_ctx (ctx))
3901 x = build_receiver_ref (var, false, ctx);
3902 else
3903 x = build_outer_var_ref (var, ctx);
3905 else
3906 x = NULL;
3907 do_private:
3908 tree nx;
3909 nx = lang_hooks.decls.omp_clause_default_ctor (c, new_var, x);
3910 if (is_simd)
3912 tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
3913 if ((TREE_ADDRESSABLE (new_var) || nx || y
3914 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
3915 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
3916 idx, lane, ivar, lvar))
3918 if (nx)
3919 x = lang_hooks.decls.omp_clause_default_ctor
3920 (c, unshare_expr (ivar), x);
3921 if (nx && x)
3922 gimplify_and_add (x, &llist[0]);
3923 if (y)
3925 y = lang_hooks.decls.omp_clause_dtor (c, ivar);
3926 if (y)
3928 gimple_seq tseq = NULL;
3930 dtor = y;
3931 gimplify_stmt (&dtor, &tseq);
3932 gimple_seq_add_seq (&llist[1], tseq);
3935 break;
3938 if (nx)
3939 gimplify_and_add (nx, ilist);
3940 /* FALLTHRU */
3942 do_dtor:
3943 x = lang_hooks.decls.omp_clause_dtor (c, new_var);
3944 if (x)
3946 gimple_seq tseq = NULL;
3948 dtor = x;
3949 gimplify_stmt (&dtor, &tseq);
3950 gimple_seq_add_seq (dlist, tseq);
3952 break;
3954 case OMP_CLAUSE_LINEAR:
3955 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c))
3956 goto do_firstprivate;
3957 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c))
3958 x = NULL;
3959 else
3960 x = build_outer_var_ref (var, ctx);
3961 goto do_private;
3963 case OMP_CLAUSE_FIRSTPRIVATE:
3964 if (is_task_ctx (ctx))
3966 if (is_reference (var) || is_variable_sized (var))
3967 goto do_dtor;
3968 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
3969 ctx))
3970 || use_pointer_for_field (var, NULL))
3972 x = build_receiver_ref (var, false, ctx);
3973 SET_DECL_VALUE_EXPR (new_var, x);
3974 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
3975 goto do_dtor;
3978 do_firstprivate:
3979 x = build_outer_var_ref (var, ctx);
3980 if (is_simd)
3982 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
3983 && gimple_omp_for_combined_into_p (ctx->stmt))
3985 tree t = OMP_CLAUSE_LINEAR_STEP (c);
3986 tree stept = TREE_TYPE (t);
3987 tree ct = find_omp_clause (clauses,
3988 OMP_CLAUSE__LOOPTEMP_);
3989 gcc_assert (ct);
3990 tree l = OMP_CLAUSE_DECL (ct);
3991 tree n1 = fd->loop.n1;
3992 tree step = fd->loop.step;
3993 tree itype = TREE_TYPE (l);
3994 if (POINTER_TYPE_P (itype))
3995 itype = signed_type_for (itype);
3996 l = fold_build2 (MINUS_EXPR, itype, l, n1);
3997 if (TYPE_UNSIGNED (itype)
3998 && fd->loop.cond_code == GT_EXPR)
3999 l = fold_build2 (TRUNC_DIV_EXPR, itype,
4000 fold_build1 (NEGATE_EXPR, itype, l),
4001 fold_build1 (NEGATE_EXPR,
4002 itype, step));
4003 else
4004 l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step);
4005 t = fold_build2 (MULT_EXPR, stept,
4006 fold_convert (stept, l), t);
4008 if (OMP_CLAUSE_LINEAR_ARRAY (c))
4010 x = lang_hooks.decls.omp_clause_linear_ctor
4011 (c, new_var, x, t);
4012 gimplify_and_add (x, ilist);
4013 goto do_dtor;
4016 if (POINTER_TYPE_P (TREE_TYPE (x)))
4017 x = fold_build2 (POINTER_PLUS_EXPR,
4018 TREE_TYPE (x), x, t);
4019 else
4020 x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, t);
4023 if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
4024 || TREE_ADDRESSABLE (new_var))
4025 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
4026 idx, lane, ivar, lvar))
4028 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
4030 tree iv = create_tmp_var (TREE_TYPE (new_var), NULL);
4031 x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x);
4032 gimplify_and_add (x, ilist);
4033 gimple_stmt_iterator gsi
4034 = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
4035 gimple g
4036 = gimple_build_assign (unshare_expr (lvar), iv);
4037 gsi_insert_before_without_update (&gsi, g,
4038 GSI_SAME_STMT);
4039 tree t = OMP_CLAUSE_LINEAR_STEP (c);
4040 enum tree_code code = PLUS_EXPR;
4041 if (POINTER_TYPE_P (TREE_TYPE (new_var)))
4042 code = POINTER_PLUS_EXPR;
4043 g = gimple_build_assign_with_ops (code, iv, iv, t);
4044 gsi_insert_before_without_update (&gsi, g,
4045 GSI_SAME_STMT);
4046 break;
4048 x = lang_hooks.decls.omp_clause_copy_ctor
4049 (c, unshare_expr (ivar), x);
4050 gimplify_and_add (x, &llist[0]);
4051 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
4052 if (x)
4054 gimple_seq tseq = NULL;
4056 dtor = x;
4057 gimplify_stmt (&dtor, &tseq);
4058 gimple_seq_add_seq (&llist[1], tseq);
4060 break;
4063 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
4064 gimplify_and_add (x, ilist);
4065 goto do_dtor;
4067 case OMP_CLAUSE__LOOPTEMP_:
4068 gcc_assert (is_parallel_ctx (ctx));
4069 x = build_outer_var_ref (var, ctx);
4070 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
4071 gimplify_and_add (x, ilist);
4072 break;
4074 case OMP_CLAUSE_COPYIN:
4075 by_ref = use_pointer_for_field (var, NULL);
4076 x = build_receiver_ref (var, by_ref, ctx);
4077 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
4078 append_to_statement_list (x, &copyin_seq);
4079 copyin_by_ref |= by_ref;
4080 break;
4082 case OMP_CLAUSE_REDUCTION:
4083 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4085 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
4086 gimple tseq;
4087 x = build_outer_var_ref (var, ctx);
4089 if (is_reference (var)
4090 && !useless_type_conversion_p (TREE_TYPE (placeholder),
4091 TREE_TYPE (x)))
4092 x = build_fold_addr_expr_loc (clause_loc, x);
4093 SET_DECL_VALUE_EXPR (placeholder, x);
4094 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
4095 tree new_vard = new_var;
4096 if (is_reference (var))
4098 gcc_assert (TREE_CODE (new_var) == MEM_REF);
4099 new_vard = TREE_OPERAND (new_var, 0);
4100 gcc_assert (DECL_P (new_vard));
4102 if (is_simd
4103 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
4104 idx, lane, ivar, lvar))
4106 if (new_vard == new_var)
4108 gcc_assert (DECL_VALUE_EXPR (new_var) == lvar);
4109 SET_DECL_VALUE_EXPR (new_var, ivar);
4111 else
4113 SET_DECL_VALUE_EXPR (new_vard,
4114 build_fold_addr_expr (ivar));
4115 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
4117 x = lang_hooks.decls.omp_clause_default_ctor
4118 (c, unshare_expr (ivar),
4119 build_outer_var_ref (var, ctx));
4120 if (x)
4121 gimplify_and_add (x, &llist[0]);
4122 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
4124 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
4125 lower_omp (&tseq, ctx);
4126 gimple_seq_add_seq (&llist[0], tseq);
4128 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
4129 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
4130 lower_omp (&tseq, ctx);
4131 gimple_seq_add_seq (&llist[1], tseq);
4132 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
4133 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
4134 if (new_vard == new_var)
4135 SET_DECL_VALUE_EXPR (new_var, lvar);
4136 else
4137 SET_DECL_VALUE_EXPR (new_vard,
4138 build_fold_addr_expr (lvar));
4139 x = lang_hooks.decls.omp_clause_dtor (c, ivar);
4140 if (x)
4142 tseq = NULL;
4143 dtor = x;
4144 gimplify_stmt (&dtor, &tseq);
4145 gimple_seq_add_seq (&llist[1], tseq);
4147 break;
4149 /* If this is a reference to constant size reduction var
4150 with placeholder, we haven't emitted the initializer
4151 for it because it is undesirable if SIMD arrays are used.
4152 But if they aren't used, we need to emit the deferred
4153 initialization now. */
4154 else if (is_reference (var) && is_simd)
4155 handle_simd_reference (clause_loc, new_vard, ilist);
4156 x = lang_hooks.decls.omp_clause_default_ctor
4157 (c, unshare_expr (new_var),
4158 build_outer_var_ref (var, ctx));
4159 if (x)
4160 gimplify_and_add (x, ilist);
4161 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
4163 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
4164 lower_omp (&tseq, ctx);
4165 gimple_seq_add_seq (ilist, tseq);
4167 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
4168 if (is_simd)
4170 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
4171 lower_omp (&tseq, ctx);
4172 gimple_seq_add_seq (dlist, tseq);
4173 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
4175 DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
4176 goto do_dtor;
4178 else
4180 x = omp_reduction_init (c, TREE_TYPE (new_var));
4181 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
4182 enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
4184 /* reduction(-:var) sums up the partial results, so it
4185 acts identically to reduction(+:var). */
4186 if (code == MINUS_EXPR)
4187 code = PLUS_EXPR;
4189 tree new_vard = new_var;
4190 if (is_simd && is_reference (var))
4192 gcc_assert (TREE_CODE (new_var) == MEM_REF);
4193 new_vard = TREE_OPERAND (new_var, 0);
4194 gcc_assert (DECL_P (new_vard));
4196 if (is_simd
4197 && lower_rec_simd_input_clauses (new_var, ctx, max_vf,
4198 idx, lane, ivar, lvar))
4200 tree ref = build_outer_var_ref (var, ctx);
4202 gimplify_assign (unshare_expr (ivar), x, &llist[0]);
4204 x = build2 (code, TREE_TYPE (ref), ref, ivar);
4205 ref = build_outer_var_ref (var, ctx);
4206 gimplify_assign (ref, x, &llist[1]);
4208 if (new_vard != new_var)
4210 SET_DECL_VALUE_EXPR (new_vard,
4211 build_fold_addr_expr (lvar));
4212 DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
4215 else
4217 if (is_reference (var) && is_simd)
4218 handle_simd_reference (clause_loc, new_vard, ilist);
4219 gimplify_assign (new_var, x, ilist);
4220 if (is_simd)
4222 tree ref = build_outer_var_ref (var, ctx);
4224 x = build2 (code, TREE_TYPE (ref), ref, new_var);
4225 ref = build_outer_var_ref (var, ctx);
4226 gimplify_assign (ref, x, dlist);
4230 break;
4232 default:
4233 gcc_unreachable ();
4238 if (lane)
4240 tree uid = create_tmp_var (ptr_type_node, "simduid");
4241 /* Don't want uninit warnings on simduid, it is always uninitialized,
4242 but we use it not for the value, but for the DECL_UID only. */
4243 TREE_NO_WARNING (uid) = 1;
4244 gimple g
4245 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid);
4246 gimple_call_set_lhs (g, lane);
4247 gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
4248 gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
4249 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_);
4250 OMP_CLAUSE__SIMDUID__DECL (c) = uid;
4251 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
4252 gimple_omp_for_set_clauses (ctx->stmt, c);
4253 g = gimple_build_assign_with_ops (INTEGER_CST, lane,
4254 build_int_cst (unsigned_type_node, 0),
4255 NULL_TREE);
4256 gimple_seq_add_stmt (ilist, g);
4257 for (int i = 0; i < 2; i++)
4258 if (llist[i])
4260 tree vf = create_tmp_var (unsigned_type_node, NULL);
4261 g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid);
4262 gimple_call_set_lhs (g, vf);
4263 gimple_seq *seq = i == 0 ? ilist : dlist;
4264 gimple_seq_add_stmt (seq, g);
4265 tree t = build_int_cst (unsigned_type_node, 0);
4266 g = gimple_build_assign_with_ops (INTEGER_CST, idx, t, NULL_TREE);
4267 gimple_seq_add_stmt (seq, g);
4268 tree body = create_artificial_label (UNKNOWN_LOCATION);
4269 tree header = create_artificial_label (UNKNOWN_LOCATION);
4270 tree end = create_artificial_label (UNKNOWN_LOCATION);
4271 gimple_seq_add_stmt (seq, gimple_build_goto (header));
4272 gimple_seq_add_stmt (seq, gimple_build_label (body));
4273 gimple_seq_add_seq (seq, llist[i]);
4274 t = build_int_cst (unsigned_type_node, 1);
4275 g = gimple_build_assign_with_ops (PLUS_EXPR, idx, idx, t);
4276 gimple_seq_add_stmt (seq, g);
4277 gimple_seq_add_stmt (seq, gimple_build_label (header));
4278 g = gimple_build_cond (LT_EXPR, idx, vf, body, end);
4279 gimple_seq_add_stmt (seq, g);
4280 gimple_seq_add_stmt (seq, gimple_build_label (end));
4284 /* The copyin sequence is not to be executed by the main thread, since
4285 that would result in self-copies. Perhaps not visible to scalars,
4286 but it certainly is to C++ operator=. */
4287 if (copyin_seq)
4289 x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM),
4291 x = build2 (NE_EXPR, boolean_type_node, x,
4292 build_int_cst (TREE_TYPE (x), 0));
4293 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
4294 gimplify_and_add (x, ilist);
4297 /* If any copyin variable is passed by reference, we must ensure the
4298 master thread doesn't modify it before it is copied over in all
4299 threads. Similarly for variables in both firstprivate and
4300 lastprivate clauses we need to ensure the lastprivate copying
4301 happens after firstprivate copying in all threads. And similarly
4302 for UDRs if initializer expression refers to omp_orig. */
4303 if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref)
4305 /* Don't add any barrier for #pragma omp simd or
4306 #pragma omp distribute. */
4307 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
4308 || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR)
4309 gimple_seq_add_stmt (ilist, build_omp_barrier (NULL_TREE));
4312 /* If max_vf is non-zero, then we can use only a vectorization factor
4313 up to the max_vf we chose. So stick it into the safelen clause. */
4314 if (max_vf)
4316 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt),
4317 OMP_CLAUSE_SAFELEN);
4318 if (c == NULL_TREE
4319 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) == INTEGER_CST
4320 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c),
4321 max_vf) == 1))
4323 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN);
4324 OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node,
4325 max_vf);
4326 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt);
4327 gimple_omp_for_set_clauses (ctx->stmt, c);
4333 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4334 both parallel and workshare constructs. PREDICATE may be NULL if it's
4335 always true. */
4337 static void
4338 lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list,
4339 omp_context *ctx)
4341 tree x, c, label = NULL, orig_clauses = clauses;
4342 bool par_clauses = false;
4343 tree simduid = NULL, lastlane = NULL;
4345 /* Early exit if there are no lastprivate or linear clauses. */
4346 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
4347 if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
4348 || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
4349 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses)))
4350 break;
4351 if (clauses == NULL)
4353 /* If this was a workshare clause, see if it had been combined
4354 with its parallel. In that case, look for the clauses on the
4355 parallel statement itself. */
4356 if (is_parallel_ctx (ctx))
4357 return;
4359 ctx = ctx->outer;
4360 if (ctx == NULL || !is_parallel_ctx (ctx))
4361 return;
4363 clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
4364 OMP_CLAUSE_LASTPRIVATE);
4365 if (clauses == NULL)
4366 return;
4367 par_clauses = true;
4370 if (predicate)
4372 gimple stmt;
4373 tree label_true, arm1, arm2;
4375 label = create_artificial_label (UNKNOWN_LOCATION);
4376 label_true = create_artificial_label (UNKNOWN_LOCATION);
4377 arm1 = TREE_OPERAND (predicate, 0);
4378 arm2 = TREE_OPERAND (predicate, 1);
4379 gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue);
4380 gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue);
4381 stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2,
4382 label_true, label);
4383 gimple_seq_add_stmt (stmt_list, stmt);
4384 gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
4387 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4388 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
4390 simduid = find_omp_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
4391 if (simduid)
4392 simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
4395 for (c = clauses; c ;)
4397 tree var, new_var;
4398 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4400 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
4401 || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
4402 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c)))
4404 var = OMP_CLAUSE_DECL (c);
4405 new_var = lookup_decl (var, ctx);
4407 if (simduid && DECL_HAS_VALUE_EXPR_P (new_var))
4409 tree val = DECL_VALUE_EXPR (new_var);
4410 if (TREE_CODE (val) == ARRAY_REF
4411 && VAR_P (TREE_OPERAND (val, 0))
4412 && lookup_attribute ("omp simd array",
4413 DECL_ATTRIBUTES (TREE_OPERAND (val,
4414 0))))
4416 if (lastlane == NULL)
4418 lastlane = create_tmp_var (unsigned_type_node, NULL);
4419 gimple g
4420 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
4421 2, simduid,
4422 TREE_OPERAND (val, 1));
4423 gimple_call_set_lhs (g, lastlane);
4424 gimple_seq_add_stmt (stmt_list, g);
4426 new_var = build4 (ARRAY_REF, TREE_TYPE (val),
4427 TREE_OPERAND (val, 0), lastlane,
4428 NULL_TREE, NULL_TREE);
4432 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
4433 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
4435 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
4436 gimple_seq_add_seq (stmt_list,
4437 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
4438 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
4440 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
4441 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
4443 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
4444 gimple_seq_add_seq (stmt_list,
4445 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
4446 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
4449 x = build_outer_var_ref (var, ctx);
4450 if (is_reference (var))
4451 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4452 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
4453 gimplify_and_add (x, stmt_list);
4455 c = OMP_CLAUSE_CHAIN (c);
4456 if (c == NULL && !par_clauses)
4458 /* If this was a workshare clause, see if it had been combined
4459 with its parallel. In that case, continue looking for the
4460 clauses also on the parallel statement itself. */
4461 if (is_parallel_ctx (ctx))
4462 break;
4464 ctx = ctx->outer;
4465 if (ctx == NULL || !is_parallel_ctx (ctx))
4466 break;
4468 c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt),
4469 OMP_CLAUSE_LASTPRIVATE);
4470 par_clauses = true;
4474 if (label)
4475 gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
4478 static void
4479 lower_reduction_var_helper (gimple_seq *stmt_seqp, omp_context *ctx, tree tid,
4480 tree var, tree new_var)
4482 /* The atomic add at the end of the sum creates unnecessary
4483 write contention on accelerators. To work around this,
4484 create an array to store the partial reductions. Later, in
4485 lower_omp_for (for openacc), the values of array will be
4486 combined. */
4488 tree t = NULL_TREE, array, x;
4489 tree type = get_base_type (var);
4490 gimple stmt;
4492 /* Now insert the partial reductions into the array. */
4494 /* Find the reduction array. */
4496 tree ptype = build_pointer_type (type);
4498 t = lookup_reduction (omp_get_id (var), ctx);
4499 t = build_receiver_ref (t, false, ctx->outer);
4501 array = create_tmp_var (ptype, NULL);
4502 gimplify_assign (array, t, stmt_seqp);
4504 tree ptr = create_tmp_var (TREE_TYPE (array), NULL);
4506 /* Find the reduction array. */
4508 /* testing a unary conversion. */
4509 tree offset = create_tmp_var (sizetype, NULL);
4510 gimplify_assign (offset, TYPE_SIZE_UNIT (type),
4511 stmt_seqp);
4512 t = create_tmp_var (sizetype, NULL);
4513 gimplify_assign (t, unshare_expr (fold_build1 (NOP_EXPR, sizetype, tid)),
4514 stmt_seqp);
4515 stmt = gimple_build_assign_with_ops (MULT_EXPR, offset, offset, t);
4516 gimple_seq_add_stmt (stmt_seqp, stmt);
4518 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4519 of adding sizeof(var) to the array? */
4520 ptr = create_tmp_var (ptype, NULL);
4521 stmt = gimple_build_assign_with_ops (POINTER_PLUS_EXPR, unshare_expr(ptr),
4522 array, offset);
4523 gimple_seq_add_stmt (stmt_seqp, stmt);
4525 /* Move the local sum to gfc$sum[i]. */
4526 x = unshare_expr (build_simple_mem_ref (ptr));
4527 stmt = gimplify_assign (x, new_var, stmt_seqp);
4530 /* Generate code to implement the REDUCTION clauses. */
4532 static void
4533 lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx)
4535 gimple_seq sub_seq = NULL;
4536 gimple stmt;
4537 tree x, c, tid;
4538 int count = 0;
4540 /* SIMD reductions are handled in lower_rec_input_clauses. */
4541 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
4542 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
4543 return;
4545 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4546 update in that case, otherwise use a lock. */
4547 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
4548 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
4550 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4552 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4553 count = -1;
4554 break;
4556 count++;
4559 if (count == 0)
4560 return;
4562 /* Initialize thread info for OpenACC. */
4563 if (is_gimple_omp_oacc_specifically (ctx->stmt))
4565 /* Get the current thread id. */
4566 tree call = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
4567 tid = create_tmp_var (TREE_TYPE (TREE_TYPE (call)), NULL);
4568 gimple stmt = gimple_build_call (call, 0);
4569 gimple_call_set_lhs (stmt, tid);
4570 gimple_seq_add_stmt (stmt_seqp, stmt);
4573 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4575 tree var, ref, new_var;
4576 enum tree_code code;
4577 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4579 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
4580 continue;
4582 var = OMP_CLAUSE_DECL (c);
4583 new_var = lookup_decl (var, ctx);
4584 if (is_reference (var))
4585 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4586 ref = build_outer_var_ref (var, ctx);
4587 code = OMP_CLAUSE_REDUCTION_CODE (c);
4589 /* reduction(-:var) sums up the partial results, so it acts
4590 identically to reduction(+:var). */
4591 if (code == MINUS_EXPR)
4592 code = PLUS_EXPR;
4594 if (count == 1)
4596 if (!is_gimple_omp_oacc_specifically (ctx->stmt))
4598 tree addr = build_fold_addr_expr_loc (clause_loc, ref);
4600 addr = save_expr (addr);
4601 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
4602 x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var);
4603 x = build2 (OMP_ATOMIC, void_type_node, addr, x);
4604 gimplify_and_add (x, stmt_seqp);
4605 return;
4607 else
4609 lower_reduction_var_helper (stmt_seqp, ctx, tid, var, new_var);
4610 return;
4614 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
4616 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
4618 if (is_reference (var)
4619 && !useless_type_conversion_p (TREE_TYPE (placeholder),
4620 TREE_TYPE (ref)))
4621 ref = build_fold_addr_expr_loc (clause_loc, ref);
4622 SET_DECL_VALUE_EXPR (placeholder, ref);
4623 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
4624 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
4625 gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
4626 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
4627 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
4629 else
4631 if (is_gimple_omp_oacc_specifically (ctx->stmt))
4633 lower_reduction_var_helper (stmt_seqp, ctx, tid, var, new_var);
4635 else
4637 x = build2 (code, TREE_TYPE (ref), ref, new_var);
4638 ref = build_outer_var_ref (var, ctx);
4639 gimplify_assign (ref, x, &sub_seq);
4644 if (is_gimple_omp_oacc_specifically (ctx->stmt))
4645 return;
4647 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START),
4649 gimple_seq_add_stmt (stmt_seqp, stmt);
4651 gimple_seq_add_seq (stmt_seqp, sub_seq);
4653 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
4655 gimple_seq_add_stmt (stmt_seqp, stmt);
4659 /* Generate code to implement the COPYPRIVATE clauses. */
4661 static void
4662 lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist,
4663 omp_context *ctx)
4665 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
4667 tree c;
4669 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4671 tree var, new_var, ref, x;
4672 bool by_ref;
4673 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4675 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
4676 continue;
4678 var = OMP_CLAUSE_DECL (c);
4679 by_ref = use_pointer_for_field (var, NULL);
4681 ref = build_sender_ref (var, ctx);
4682 x = new_var = lookup_decl_in_outer_ctx (var, ctx);
4683 if (by_ref)
4685 x = build_fold_addr_expr_loc (clause_loc, new_var);
4686 x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x);
4688 gimplify_assign (ref, x, slist);
4690 ref = build_receiver_ref (var, false, ctx);
4691 if (by_ref)
4693 ref = fold_convert_loc (clause_loc,
4694 build_pointer_type (TREE_TYPE (new_var)),
4695 ref);
4696 ref = build_fold_indirect_ref_loc (clause_loc, ref);
4698 if (is_reference (var))
4700 ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref);
4701 ref = build_simple_mem_ref_loc (clause_loc, ref);
4702 new_var = build_simple_mem_ref_loc (clause_loc, new_var);
4704 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref);
4705 gimplify_and_add (x, rlist);
4710 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4711 and REDUCTION from the sender (aka parent) side. */
4713 static void
4714 lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist,
4715 omp_context *ctx)
4717 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
4719 tree c;
4721 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
4723 tree val, ref, x, var;
4724 bool by_ref, do_in = false, do_out = false;
4725 location_t clause_loc = OMP_CLAUSE_LOCATION (c);
4727 switch (OMP_CLAUSE_CODE (c))
4729 case OMP_CLAUSE_PRIVATE:
4730 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c))
4731 break;
4732 continue;
4733 case OMP_CLAUSE_FIRSTPRIVATE:
4734 case OMP_CLAUSE_COPYIN:
4735 case OMP_CLAUSE_LASTPRIVATE:
4736 case OMP_CLAUSE_REDUCTION:
4737 case OMP_CLAUSE__LOOPTEMP_:
4738 break;
4739 default:
4740 continue;
4743 val = OMP_CLAUSE_DECL (c);
4744 var = lookup_decl_in_outer_ctx (val, ctx);
4746 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
4747 && is_global_var (var))
4748 continue;
4749 if (is_variable_sized (val))
4750 continue;
4751 by_ref = use_pointer_for_field (val, NULL);
4753 switch (OMP_CLAUSE_CODE (c))
4755 case OMP_CLAUSE_PRIVATE:
4756 case OMP_CLAUSE_FIRSTPRIVATE:
4757 case OMP_CLAUSE_COPYIN:
4758 case OMP_CLAUSE__LOOPTEMP_:
4759 do_in = true;
4760 break;
4762 case OMP_CLAUSE_LASTPRIVATE:
4763 if (by_ref || is_reference (val))
4765 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
4766 continue;
4767 do_in = true;
4769 else
4771 do_out = true;
4772 if (lang_hooks.decls.omp_private_outer_ref (val))
4773 do_in = true;
4775 break;
4777 case OMP_CLAUSE_REDUCTION:
4778 do_in = true;
4779 do_out = !(by_ref || is_reference (val));
4780 break;
4782 default:
4783 gcc_unreachable ();
4786 if (do_in)
4788 ref = build_sender_ref (val, ctx);
4789 x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var;
4790 gimplify_assign (ref, x, ilist);
4791 if (is_task_ctx (ctx))
4792 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL;
4795 if (do_out)
4797 ref = build_sender_ref (val, ctx);
4798 gimplify_assign (var, ref, olist);
4803 /* Generate code to implement SHARED from the sender (aka parent)
4804 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4805 list things that got automatically shared. */
4807 static void
4808 lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx)
4810 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
4812 tree var, ovar, nvar, f, x, record_type;
4814 if (ctx->record_type == NULL)
4815 return;
4817 record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type;
4818 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
4820 ovar = DECL_ABSTRACT_ORIGIN (f);
4821 nvar = maybe_lookup_decl (ovar, ctx);
4822 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar))
4823 continue;
4825 /* If CTX is a nested parallel directive. Find the immediately
4826 enclosing parallel or workshare construct that contains a
4827 mapping for OVAR. */
4828 var = lookup_decl_in_outer_ctx (ovar, ctx);
4830 if (use_pointer_for_field (ovar, ctx))
4832 x = build_sender_ref (ovar, ctx);
4833 var = build_fold_addr_expr (var);
4834 gimplify_assign (x, var, ilist);
4836 else
4838 x = build_sender_ref (ovar, ctx);
4839 gimplify_assign (x, var, ilist);
4841 if (!TREE_READONLY (var)
4842 /* We don't need to receive a new reference to a result
4843 or parm decl. In fact we may not store to it as we will
4844 invalidate any pending RSO and generate wrong gimple
4845 during inlining. */
4846 && !((TREE_CODE (var) == RESULT_DECL
4847 || TREE_CODE (var) == PARM_DECL)
4848 && DECL_BY_REFERENCE (var)))
4850 x = build_sender_ref (ovar, ctx);
4851 gimplify_assign (var, x, olist);
4858 /* A convenience function to build an empty GIMPLE_COND with just the
4859 condition. */
4861 static gimple
4862 gimple_build_cond_empty (tree cond)
4864 enum tree_code pred_code;
4865 tree lhs, rhs;
4867 gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs);
4868 return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE);
4872 /* Build the function calls to GOMP_parallel_start etc to actually
4873 generate the parallel operation. REGION is the parallel region
4874 being expanded. BB is the block where to insert the code. WS_ARGS
4875 will be set if this is a call to a combined parallel+workshare
4876 construct, it contains the list of additional arguments needed by
4877 the workshare construct. */
4879 static void
4880 expand_parallel_call (struct omp_region *region, basic_block bb,
4881 gimple entry_stmt, vec<tree, va_gc> *ws_args)
4883 tree t, t1, t2, val, cond, c, clauses, flags;
4884 gimple_stmt_iterator gsi;
4885 gimple stmt;
4886 enum built_in_function start_ix;
4887 int start_ix2;
4888 location_t clause_loc;
4889 vec<tree, va_gc> *args;
4891 clauses = gimple_omp_parallel_clauses (entry_stmt);
4893 /* Determine what flavor of GOMP_parallel we will be
4894 emitting. */
4895 start_ix = BUILT_IN_GOMP_PARALLEL;
4896 if (is_combined_parallel (region))
4898 switch (region->inner->type)
4900 case GIMPLE_OMP_FOR:
4901 gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
4902 start_ix2 = ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4903 + (region->inner->sched_kind
4904 == OMP_CLAUSE_SCHEDULE_RUNTIME
4905 ? 3 : region->inner->sched_kind));
4906 start_ix = (enum built_in_function)start_ix2;
4907 break;
4908 case GIMPLE_OMP_SECTIONS:
4909 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS;
4910 break;
4911 default:
4912 gcc_unreachable ();
4916 /* By default, the value of NUM_THREADS is zero (selected at run time)
4917 and there is no conditional. */
4918 cond = NULL_TREE;
4919 val = build_int_cst (unsigned_type_node, 0);
4920 flags = build_int_cst (unsigned_type_node, 0);
4922 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
4923 if (c)
4924 cond = OMP_CLAUSE_IF_EXPR (c);
4926 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS);
4927 if (c)
4929 val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
4930 clause_loc = OMP_CLAUSE_LOCATION (c);
4932 else
4933 clause_loc = gimple_location (entry_stmt);
4935 c = find_omp_clause (clauses, OMP_CLAUSE_PROC_BIND);
4936 if (c)
4937 flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c));
4939 /* Ensure 'val' is of the correct type. */
4940 val = fold_convert_loc (clause_loc, unsigned_type_node, val);
4942 /* If we found the clause 'if (cond)', build either
4943 (cond != 0) or (cond ? val : 1u). */
4944 if (cond)
4946 cond = gimple_boolify (cond);
4948 if (integer_zerop (val))
4949 val = fold_build2_loc (clause_loc,
4950 EQ_EXPR, unsigned_type_node, cond,
4951 build_int_cst (TREE_TYPE (cond), 0));
4952 else
4954 basic_block cond_bb, then_bb, else_bb;
4955 edge e, e_then, e_else;
4956 tree tmp_then, tmp_else, tmp_join, tmp_var;
4958 tmp_var = create_tmp_var (TREE_TYPE (val), NULL);
4959 if (gimple_in_ssa_p (cfun))
4961 tmp_then = make_ssa_name (tmp_var, NULL);
4962 tmp_else = make_ssa_name (tmp_var, NULL);
4963 tmp_join = make_ssa_name (tmp_var, NULL);
4965 else
4967 tmp_then = tmp_var;
4968 tmp_else = tmp_var;
4969 tmp_join = tmp_var;
4972 e = split_block (bb, NULL);
4973 cond_bb = e->src;
4974 bb = e->dest;
4975 remove_edge (e);
4977 then_bb = create_empty_bb (cond_bb);
4978 else_bb = create_empty_bb (then_bb);
4979 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
4980 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
4982 stmt = gimple_build_cond_empty (cond);
4983 gsi = gsi_start_bb (cond_bb);
4984 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4986 gsi = gsi_start_bb (then_bb);
4987 stmt = gimple_build_assign (tmp_then, val);
4988 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4990 gsi = gsi_start_bb (else_bb);
4991 stmt = gimple_build_assign
4992 (tmp_else, build_int_cst (unsigned_type_node, 1));
4993 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
4995 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
4996 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
4997 add_bb_to_loop (then_bb, cond_bb->loop_father);
4998 add_bb_to_loop (else_bb, cond_bb->loop_father);
4999 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
5000 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
5002 if (gimple_in_ssa_p (cfun))
5004 gimple phi = create_phi_node (tmp_join, bb);
5005 add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION);
5006 add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION);
5009 val = tmp_join;
5012 gsi = gsi_start_bb (bb);
5013 val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE,
5014 false, GSI_CONTINUE_LINKING);
5017 gsi = gsi_last_bb (bb);
5018 t = gimple_omp_parallel_data_arg (entry_stmt);
5019 if (t == NULL)
5020 t1 = null_pointer_node;
5021 else
5022 t1 = build_fold_addr_expr (t);
5023 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
5025 vec_alloc (args, 4 + vec_safe_length (ws_args));
5026 args->quick_push (t2);
5027 args->quick_push (t1);
5028 args->quick_push (val);
5029 if (ws_args)
5030 args->splice (*ws_args);
5031 args->quick_push (flags);
5033 t = build_call_expr_loc_vec (UNKNOWN_LOCATION,
5034 builtin_decl_explicit (start_ix), args);
5036 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5037 false, GSI_CONTINUE_LINKING);
5040 /* Insert a function call whose name is FUNC_NAME with the information from
5041 ENTRY_STMT into the basic_block BB. */
5043 static void
5044 expand_cilk_for_call (basic_block bb, gimple entry_stmt,
5045 vec <tree, va_gc> *ws_args)
5047 tree t, t1, t2;
5048 gimple_stmt_iterator gsi;
5049 vec <tree, va_gc> *args;
5051 gcc_assert (vec_safe_length (ws_args) == 2);
5052 tree func_name = (*ws_args)[0];
5053 tree grain = (*ws_args)[1];
5055 tree clauses = gimple_omp_parallel_clauses (entry_stmt);
5056 tree count = find_omp_clause (clauses, OMP_CLAUSE__CILK_FOR_COUNT_);
5057 gcc_assert (count != NULL_TREE);
5058 count = OMP_CLAUSE_OPERAND (count, 0);
5060 gsi = gsi_last_bb (bb);
5061 t = gimple_omp_parallel_data_arg (entry_stmt);
5062 if (t == NULL)
5063 t1 = null_pointer_node;
5064 else
5065 t1 = build_fold_addr_expr (t);
5066 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt));
5068 vec_alloc (args, 4);
5069 args->quick_push (t2);
5070 args->quick_push (t1);
5071 args->quick_push (count);
5072 args->quick_push (grain);
5073 t = build_call_expr_loc_vec (UNKNOWN_LOCATION, func_name, args);
5075 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, false,
5076 GSI_CONTINUE_LINKING);
5079 /* Build the function call to GOMP_task to actually
5080 generate the task operation. BB is the block where to insert the code. */
5082 static void
5083 expand_task_call (basic_block bb, gimple entry_stmt)
5085 tree t, t1, t2, t3, flags, cond, c, c2, clauses, depend;
5086 gimple_stmt_iterator gsi;
5087 location_t loc = gimple_location (entry_stmt);
5089 clauses = gimple_omp_task_clauses (entry_stmt);
5091 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
5092 if (c)
5093 cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (c));
5094 else
5095 cond = boolean_true_node;
5097 c = find_omp_clause (clauses, OMP_CLAUSE_UNTIED);
5098 c2 = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE);
5099 depend = find_omp_clause (clauses, OMP_CLAUSE_DEPEND);
5100 flags = build_int_cst (unsigned_type_node,
5101 (c ? 1 : 0) + (c2 ? 4 : 0) + (depend ? 8 : 0));
5103 c = find_omp_clause (clauses, OMP_CLAUSE_FINAL);
5104 if (c)
5106 c = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c));
5107 c = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, c,
5108 build_int_cst (unsigned_type_node, 2),
5109 build_int_cst (unsigned_type_node, 0));
5110 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, c);
5112 if (depend)
5113 depend = OMP_CLAUSE_DECL (depend);
5114 else
5115 depend = build_int_cst (ptr_type_node, 0);
5117 gsi = gsi_last_bb (bb);
5118 t = gimple_omp_task_data_arg (entry_stmt);
5119 if (t == NULL)
5120 t2 = null_pointer_node;
5121 else
5122 t2 = build_fold_addr_expr_loc (loc, t);
5123 t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt));
5124 t = gimple_omp_task_copy_fn (entry_stmt);
5125 if (t == NULL)
5126 t3 = null_pointer_node;
5127 else
5128 t3 = build_fold_addr_expr_loc (loc, t);
5130 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
5131 8, t1, t2, t3,
5132 gimple_omp_task_arg_size (entry_stmt),
5133 gimple_omp_task_arg_align (entry_stmt), cond, flags,
5134 depend);
5136 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
5137 false, GSI_CONTINUE_LINKING);
5141 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5142 catch handler and return it. This prevents programs from violating the
5143 structured block semantics with throws. */
5145 static gimple_seq
5146 maybe_catch_exception (gimple_seq body)
5148 gimple g;
5149 tree decl;
5151 if (!flag_exceptions)
5152 return body;
5154 if (lang_hooks.eh_protect_cleanup_actions != NULL)
5155 decl = lang_hooks.eh_protect_cleanup_actions ();
5156 else
5157 decl = builtin_decl_explicit (BUILT_IN_TRAP);
5159 g = gimple_build_eh_must_not_throw (decl);
5160 g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g),
5161 GIMPLE_TRY_CATCH);
5163 return gimple_seq_alloc_with_stmt (g);
5166 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5168 static tree
5169 vec2chain (vec<tree, va_gc> *v)
5171 tree chain = NULL_TREE, t;
5172 unsigned ix;
5174 FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t)
5176 DECL_CHAIN (t) = chain;
5177 chain = t;
5180 return chain;
5184 /* Remove barriers in REGION->EXIT's block. Note that this is only
5185 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5186 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5187 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5188 removed. */
5190 static void
5191 remove_exit_barrier (struct omp_region *region)
5193 gimple_stmt_iterator gsi;
5194 basic_block exit_bb;
5195 edge_iterator ei;
5196 edge e;
5197 gimple stmt;
5198 int any_addressable_vars = -1;
5200 exit_bb = region->exit;
5202 /* If the parallel region doesn't return, we don't have REGION->EXIT
5203 block at all. */
5204 if (! exit_bb)
5205 return;
5207 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5208 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5209 statements that can appear in between are extremely limited -- no
5210 memory operations at all. Here, we allow nothing at all, so the
5211 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5212 gsi = gsi_last_bb (exit_bb);
5213 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
5214 gsi_prev (&gsi);
5215 if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL)
5216 return;
5218 FOR_EACH_EDGE (e, ei, exit_bb->preds)
5220 gsi = gsi_last_bb (e->src);
5221 if (gsi_end_p (gsi))
5222 continue;
5223 stmt = gsi_stmt (gsi);
5224 if (gimple_code (stmt) == GIMPLE_OMP_RETURN
5225 && !gimple_omp_return_nowait_p (stmt))
5227 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5228 in many cases. If there could be tasks queued, the barrier
5229 might be needed to let the tasks run before some local
5230 variable of the parallel that the task uses as shared
5231 runs out of scope. The task can be spawned either
5232 from within current function (this would be easy to check)
5233 or from some function it calls and gets passed an address
5234 of such a variable. */
5235 if (any_addressable_vars < 0)
5237 gimple parallel_stmt = last_stmt (region->entry);
5238 tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt);
5239 tree local_decls, block, decl;
5240 unsigned ix;
5242 any_addressable_vars = 0;
5243 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl)
5244 if (TREE_ADDRESSABLE (decl))
5246 any_addressable_vars = 1;
5247 break;
5249 for (block = gimple_block (stmt);
5250 !any_addressable_vars
5251 && block
5252 && TREE_CODE (block) == BLOCK;
5253 block = BLOCK_SUPERCONTEXT (block))
5255 for (local_decls = BLOCK_VARS (block);
5256 local_decls;
5257 local_decls = DECL_CHAIN (local_decls))
5258 if (TREE_ADDRESSABLE (local_decls))
5260 any_addressable_vars = 1;
5261 break;
5263 if (block == gimple_block (parallel_stmt))
5264 break;
5267 if (!any_addressable_vars)
5268 gimple_omp_return_set_nowait (stmt);
5273 static void
5274 remove_exit_barriers (struct omp_region *region)
5276 if (region->type == GIMPLE_OMP_PARALLEL)
5277 remove_exit_barrier (region);
5279 if (region->inner)
5281 region = region->inner;
5282 remove_exit_barriers (region);
5283 while (region->next)
5285 region = region->next;
5286 remove_exit_barriers (region);
5291 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5292 calls. These can't be declared as const functions, but
5293 within one parallel body they are constant, so they can be
5294 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5295 which are declared const. Similarly for task body, except
5296 that in untied task omp_get_thread_num () can change at any task
5297 scheduling point. */
5299 static void
5300 optimize_omp_library_calls (gimple entry_stmt)
5302 basic_block bb;
5303 gimple_stmt_iterator gsi;
5304 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5305 tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree);
5306 tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
5307 tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree);
5308 bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK
5309 && find_omp_clause (gimple_omp_task_clauses (entry_stmt),
5310 OMP_CLAUSE_UNTIED) != NULL);
5312 FOR_EACH_BB_FN (bb, cfun)
5313 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
5315 gimple call = gsi_stmt (gsi);
5316 tree decl;
5318 if (is_gimple_call (call)
5319 && (decl = gimple_call_fndecl (call))
5320 && DECL_EXTERNAL (decl)
5321 && TREE_PUBLIC (decl)
5322 && DECL_INITIAL (decl) == NULL)
5324 tree built_in;
5326 if (DECL_NAME (decl) == thr_num_id)
5328 /* In #pragma omp task untied omp_get_thread_num () can change
5329 during the execution of the task region. */
5330 if (untied_task)
5331 continue;
5332 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
5334 else if (DECL_NAME (decl) == num_thr_id)
5335 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
5336 else
5337 continue;
5339 if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in)
5340 || gimple_call_num_args (call) != 0)
5341 continue;
5343 if (flag_exceptions && !TREE_NOTHROW (decl))
5344 continue;
5346 if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE
5347 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)),
5348 TREE_TYPE (TREE_TYPE (built_in))))
5349 continue;
5351 gimple_call_set_fndecl (call, built_in);
5356 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5357 regimplified. */
5359 static tree
5360 expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *)
5362 tree t = *tp;
5364 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5365 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t))
5366 return t;
5368 if (TREE_CODE (t) == ADDR_EXPR)
5369 recompute_tree_invariant_for_addr_expr (t);
5371 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
5372 return NULL_TREE;
5375 /* Prepend TO = FROM assignment before *GSI_P. */
5377 static void
5378 expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from)
5380 bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to);
5381 from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE,
5382 true, GSI_SAME_STMT);
5383 gimple stmt = gimple_build_assign (to, from);
5384 gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT);
5385 if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL)
5386 || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL))
5388 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
5389 gimple_regimplify_operands (stmt, &gsi);
5393 /* Expand the OpenACC offload directive starting at REGION. */
5395 static void
5396 expand_oacc_offload (struct omp_region *region)
5398 basic_block entry_bb, exit_bb, new_bb;
5399 struct function *child_cfun;
5400 tree child_fn, block, t;
5401 gimple_stmt_iterator gsi;
5402 gimple entry_stmt, stmt;
5403 edge e;
5404 tree (*gimple_omp_child_fn) (const_gimple);
5405 tree (*gimple_omp_data_arg) (const_gimple);
5406 switch (region->type)
5408 case GIMPLE_OACC_KERNELS:
5409 gimple_omp_child_fn = gimple_oacc_kernels_child_fn;
5410 gimple_omp_data_arg = gimple_oacc_kernels_data_arg;
5411 break;
5412 case GIMPLE_OACC_PARALLEL:
5413 gimple_omp_child_fn = gimple_oacc_parallel_child_fn;
5414 gimple_omp_data_arg = gimple_oacc_parallel_data_arg;
5415 break;
5416 default:
5417 gcc_unreachable ();
5420 entry_stmt = last_stmt (region->entry);
5421 child_fn = gimple_omp_child_fn (entry_stmt);
5422 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
5424 /* Supported by expand_omp_taskreg, but not here. */
5425 gcc_assert (!child_cfun->cfg);
5426 gcc_assert (!gimple_in_ssa_p (cfun));
5428 entry_bb = region->entry;
5429 exit_bb = region->exit;
5431 /* Preserve indentation of expand_omp_target and expand_omp_taskreg. */
5432 if (1)
5434 unsigned srcidx, dstidx, num;
5436 /* If the parallel region needs data sent from the parent
5437 function, then the very first statement (except possible
5438 tree profile counter updates) of the parallel body
5439 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5440 &.OMP_DATA_O is passed as an argument to the child function,
5441 we need to replace it with the argument as seen by the child
5442 function.
5444 In most cases, this will end up being the identity assignment
5445 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5446 a function call that has been inlined, the original PARM_DECL
5447 .OMP_DATA_I may have been converted into a different local
5448 variable. In which case, we need to keep the assignment. */
5449 if (gimple_omp_data_arg (entry_stmt))
5451 basic_block entry_succ_bb = single_succ (entry_bb);
5452 gimple_stmt_iterator gsi;
5453 tree arg;
5454 gimple parcopy_stmt = NULL;
5455 tree sender = TREE_VEC_ELT (gimple_omp_data_arg (entry_stmt), 0);
5457 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
5459 gcc_assert (!gsi_end_p (gsi));
5460 stmt = gsi_stmt (gsi);
5461 if (gimple_code (stmt) != GIMPLE_ASSIGN)
5462 continue;
5464 if (gimple_num_ops (stmt) == 2)
5466 tree arg = gimple_assign_rhs1 (stmt);
5468 /* We're ignore the subcode because we're
5469 effectively doing a STRIP_NOPS. */
5471 if (TREE_CODE (arg) == ADDR_EXPR
5472 && TREE_OPERAND (arg, 0) == sender)
5474 parcopy_stmt = stmt;
5475 break;
5480 gcc_assert (parcopy_stmt != NULL);
5481 arg = DECL_ARGUMENTS (child_fn);
5483 gcc_assert (gimple_assign_lhs (parcopy_stmt) == arg);
5484 gsi_remove (&gsi, true);
5487 /* Declare local variables needed in CHILD_CFUN. */
5488 block = DECL_INITIAL (child_fn);
5489 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
5490 /* The gimplifier could record temporaries in the block
5491 rather than in containing function's local_decls chain,
5492 which would mean cgraph missed finalizing them. Do it now. */
5493 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
5494 if (TREE_CODE (t) == VAR_DECL
5495 && TREE_STATIC (t)
5496 && !DECL_EXTERNAL (t))
5497 varpool_node::finalize_decl (t);
5498 DECL_SAVED_TREE (child_fn) = NULL;
5499 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5500 gimple_set_body (child_fn, NULL);
5501 TREE_USED (block) = 1;
5503 /* Reset DECL_CONTEXT on function arguments. */
5504 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
5505 DECL_CONTEXT (t) = child_fn;
5507 /* Split ENTRY_BB at GIMPLE_OACC_PARALLEL,
5508 so that it can be moved to the child function. */
5509 gsi = gsi_last_bb (entry_bb);
5510 stmt = gsi_stmt (gsi);
5511 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OACC_KERNELS
5512 || gimple_code (stmt) == GIMPLE_OACC_PARALLEL));
5513 gsi_remove (&gsi, true);
5514 e = split_block (entry_bb, stmt);
5515 entry_bb = e->dest;
5516 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
5518 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5519 if (exit_bb)
5521 gsi = gsi_last_bb (exit_bb);
5522 gcc_assert (!gsi_end_p (gsi)
5523 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
5524 stmt = gimple_build_return (NULL);
5525 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
5526 gsi_remove (&gsi, true);
5529 /* Move the region into CHILD_CFUN. */
5531 block = gimple_block (entry_stmt);
5533 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
5534 if (exit_bb)
5535 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
5536 /* When the expansion process cannot guarantee an up-to-date
5537 loop tree arrange for the child function to fixup loops. */
5538 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
5539 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
5541 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5542 num = vec_safe_length (child_cfun->local_decls);
5543 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
5545 t = (*child_cfun->local_decls)[srcidx];
5546 if (DECL_CONTEXT (t) == cfun->decl)
5547 continue;
5548 if (srcidx != dstidx)
5549 (*child_cfun->local_decls)[dstidx] = t;
5550 dstidx++;
5552 if (dstidx != num)
5553 vec_safe_truncate (child_cfun->local_decls, dstidx);
5555 /* Inform the callgraph about the new function. */
5556 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
5557 cgraph_node::add_new_function (child_fn, true);
5559 /* Add the new function to the offload table. */
5560 vec_safe_push (offload_funcs, child_fn);
5562 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5563 fixed in a following pass. */
5564 push_cfun (child_cfun);
5565 cgraph_edge::rebuild_edges ();
5567 /* Some EH regions might become dead, see PR34608. If
5568 pass_cleanup_cfg isn't the first pass to happen with the
5569 new child, these dead EH edges might cause problems.
5570 Clean them up now. */
5571 if (flag_exceptions)
5573 basic_block bb;
5574 bool changed = false;
5576 FOR_EACH_BB_FN (bb, cfun)
5577 changed |= gimple_purge_dead_eh_edges (bb);
5578 if (changed)
5579 cleanup_tree_cfg ();
5581 pop_cfun ();
5584 /* Emit a library call to launch CHILD_FN. */
5585 tree t1, t2, t3, t4,
5586 t_num_gangs, t_num_workers, t_vector_length, t_async,
5587 device, cond, c, clauses;
5588 enum built_in_function start_ix;
5589 location_t clause_loc;
5590 tree (*gimple_omp_clauses) (const_gimple);
5591 switch (region->type)
5593 case GIMPLE_OACC_KERNELS:
5594 gimple_omp_clauses = gimple_oacc_kernels_clauses;
5595 start_ix = BUILT_IN_GOACC_KERNELS;
5596 break;
5597 case GIMPLE_OACC_PARALLEL:
5598 gimple_omp_clauses = gimple_oacc_parallel_clauses;
5599 start_ix = BUILT_IN_GOACC_PARALLEL;
5600 break;
5601 default:
5602 gcc_unreachable ();
5605 clauses = gimple_omp_clauses (entry_stmt);
5607 /* Default values for NUM_GANGS, NUM_WORKERS, and VECTOR_LENGTH. */
5608 t_num_gangs = t_num_workers = t_vector_length
5609 = fold_convert_loc (gimple_location (entry_stmt),
5610 integer_type_node, integer_one_node);
5611 /* TODO: XXX FIX -2. */
5612 t_async = fold_convert_loc (gimple_location (entry_stmt),
5613 integer_type_node, build_int_cst (integer_type_node, -2));
5614 switch (region->type)
5616 case GIMPLE_OACC_PARALLEL:
5617 /* ..., but if present, use the values specified by the respective
5618 clauses, making sure these are of the correct type. */
5619 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS);
5620 if (c)
5621 t_num_gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
5622 integer_type_node,
5623 OMP_CLAUSE_NUM_GANGS_EXPR (c));
5624 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_WORKERS);
5625 if (c)
5626 t_num_workers = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
5627 integer_type_node,
5628 OMP_CLAUSE_NUM_WORKERS_EXPR (c));
5629 c = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH);
5630 if (c)
5631 t_vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
5632 integer_type_node,
5633 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c));
5634 /* FALL THROUGH. */
5635 case GIMPLE_OACC_KERNELS:
5636 c = find_omp_clause (clauses, OMP_CLAUSE_ASYNC);
5637 if (c)
5638 t_async = fold_convert_loc (OMP_CLAUSE_LOCATION (c),
5639 integer_type_node,
5640 OMP_CLAUSE_ASYNC_EXPR (c));
5641 break;
5643 default:
5644 break;
5647 /* By default, the value of DEVICE is -1 (let runtime library choose)
5648 and there is no conditional. */
5649 cond = NULL_TREE;
5650 device = build_int_cst (integer_type_node, -1);
5652 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
5653 if (c)
5654 cond = OMP_CLAUSE_IF_EXPR (c);
5656 c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE);
5657 gcc_assert (c == NULL);
5658 if (c)
5660 device = OMP_CLAUSE_DEVICE_ID (c);
5661 clause_loc = OMP_CLAUSE_LOCATION (c);
5663 else
5664 clause_loc = gimple_location (entry_stmt);
5666 /* Ensure 'device' is of the correct type. */
5667 device = fold_convert_loc (clause_loc, integer_type_node, device);
5669 /* If we found the clause 'if (cond)', build
5670 (cond ? device : -2). */
5671 if (cond)
5673 cond = gimple_boolify (cond);
5675 basic_block cond_bb, then_bb, else_bb;
5676 edge e;
5677 tree tmp_var;
5679 tmp_var = create_tmp_var (TREE_TYPE (device), NULL);
5680 /* Preserve indentation of expand_omp_target. */
5681 if (0)
5683 gsi = gsi_last_bb (new_bb);
5684 gsi_prev (&gsi);
5685 e = split_block (new_bb, gsi_stmt (gsi));
5687 else
5688 e = split_block (new_bb, NULL);
5689 cond_bb = e->src;
5690 new_bb = e->dest;
5691 remove_edge (e);
5693 then_bb = create_empty_bb (cond_bb);
5694 else_bb = create_empty_bb (then_bb);
5695 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
5696 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
5698 stmt = gimple_build_cond_empty (cond);
5699 gsi = gsi_last_bb (cond_bb);
5700 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5702 gsi = gsi_start_bb (then_bb);
5703 stmt = gimple_build_assign (tmp_var, device);
5704 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5706 gsi = gsi_start_bb (else_bb);
5707 stmt = gimple_build_assign (tmp_var,
5708 build_int_cst (integer_type_node, -2));
5709 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
5711 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
5712 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
5713 add_bb_to_loop (then_bb, cond_bb->loop_father);
5714 add_bb_to_loop (else_bb, cond_bb->loop_father);
5715 make_edge (then_bb, new_bb, EDGE_FALLTHRU);
5716 make_edge (else_bb, new_bb, EDGE_FALLTHRU);
5718 device = tmp_var;
5721 gsi = gsi_last_bb (new_bb);
5722 t = gimple_omp_data_arg (entry_stmt);
5723 if (t == NULL)
5725 t1 = size_zero_node;
5726 t2 = build_zero_cst (ptr_type_node);
5727 t3 = t2;
5728 t4 = t2;
5730 else
5732 t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1))));
5733 t1 = size_binop (PLUS_EXPR, t1, size_int (1));
5734 t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0));
5735 t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1));
5736 t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2));
5739 gimple g;
5740 tree openmp_target = get_offload_symbol_decl ();
5741 tree fnaddr = build_fold_addr_expr (child_fn);
5743 vec<tree> *args;
5744 int idx;
5745 unsigned int argcnt = 12;
5747 c = find_omp_clause (clauses, OMP_CLAUSE_WAIT);
5748 if (c)
5750 for (t = c; t; t = OMP_CLAUSE_CHAIN (t))
5752 if (OMP_CLAUSE_CODE (t) == OMP_CLAUSE_WAIT)
5753 argcnt++;
5757 vec_alloc (args, argcnt);
5758 args->quick_push (device);
5759 args->quick_push (fnaddr);
5760 args->quick_push (build_fold_addr_expr (openmp_target));
5761 args->quick_push (t1);
5762 args->quick_push (t2);
5763 args->quick_push (t3);
5764 args->quick_push (t4);
5765 args->quick_push (t_num_gangs);
5766 args->quick_push (t_num_workers);
5767 args->quick_push (t_vector_length);
5768 args->quick_push (t_async);
5769 idx = args->length ();
5770 args->quick_push (fold_convert_loc (gimple_location (entry_stmt),
5771 integer_type_node, integer_minus_one_node));
5772 if (c)
5774 int n = 0;
5776 for (t = c; t; t = OMP_CLAUSE_CHAIN (t))
5778 if (OMP_CLAUSE_CODE (t) == OMP_CLAUSE_WAIT)
5780 args->quick_push (fold_convert (integer_type_node,
5781 OMP_CLAUSE_WAIT_EXPR (t)));
5782 n++;
5786 args->ordered_remove (idx);
5787 args->quick_insert (idx, fold_convert_loc (gimple_location (entry_stmt),
5788 integer_type_node,
5789 build_int_cst (integer_type_node, n)));
5792 g = gimple_build_call_vec (builtin_decl_explicit (start_ix), *args);
5793 args->release ();
5794 gimple_set_location (g, gimple_location (entry_stmt));
5795 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5798 /* Expand the OpenMP parallel or task directive starting at REGION. */
5800 static void
5801 expand_omp_taskreg (struct omp_region *region)
5803 basic_block entry_bb, exit_bb, new_bb;
5804 struct function *child_cfun;
5805 tree child_fn, block, t;
5806 gimple_stmt_iterator gsi;
5807 gimple entry_stmt, stmt;
5808 edge e;
5809 vec<tree, va_gc> *ws_args;
5811 entry_stmt = last_stmt (region->entry);
5812 child_fn = gimple_omp_taskreg_child_fn (entry_stmt);
5813 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
5815 entry_bb = region->entry;
5816 exit_bb = region->exit;
5818 bool is_cilk_for
5819 = (flag_cilkplus
5820 && gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL
5821 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt),
5822 OMP_CLAUSE__CILK_FOR_COUNT_) != NULL_TREE);
5824 if (is_cilk_for)
5825 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5826 and the inner statement contains the name of the built-in function
5827 and grain. */
5828 ws_args = region->inner->ws_args;
5829 else if (is_combined_parallel (region))
5830 ws_args = region->ws_args;
5831 else
5832 ws_args = NULL;
5834 if (child_cfun->cfg)
5836 /* Due to inlining, it may happen that we have already outlined
5837 the region, in which case all we need to do is make the
5838 sub-graph unreachable and emit the parallel call. */
5839 edge entry_succ_e, exit_succ_e;
5841 entry_succ_e = single_succ_edge (entry_bb);
5843 gsi = gsi_last_bb (entry_bb);
5844 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL
5845 || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK);
5846 gsi_remove (&gsi, true);
5848 new_bb = entry_bb;
5849 if (exit_bb)
5851 exit_succ_e = single_succ_edge (exit_bb);
5852 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
5854 remove_edge_and_dominated_blocks (entry_succ_e);
5856 else
5858 unsigned srcidx, dstidx, num;
5860 /* If the parallel region needs data sent from the parent
5861 function, then the very first statement (except possible
5862 tree profile counter updates) of the parallel body
5863 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5864 &.OMP_DATA_O is passed as an argument to the child function,
5865 we need to replace it with the argument as seen by the child
5866 function.
5868 In most cases, this will end up being the identity assignment
5869 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5870 a function call that has been inlined, the original PARM_DECL
5871 .OMP_DATA_I may have been converted into a different local
5872 variable. In which case, we need to keep the assignment. */
5873 if (gimple_omp_taskreg_data_arg (entry_stmt))
5875 basic_block entry_succ_bb = single_succ (entry_bb);
5876 tree arg, narg;
5877 gimple parcopy_stmt = NULL;
5879 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
5881 gimple stmt;
5883 gcc_assert (!gsi_end_p (gsi));
5884 stmt = gsi_stmt (gsi);
5885 if (gimple_code (stmt) != GIMPLE_ASSIGN)
5886 continue;
5888 if (gimple_num_ops (stmt) == 2)
5890 tree arg = gimple_assign_rhs1 (stmt);
5892 /* We're ignore the subcode because we're
5893 effectively doing a STRIP_NOPS. */
5895 if (TREE_CODE (arg) == ADDR_EXPR
5896 && TREE_OPERAND (arg, 0)
5897 == gimple_omp_taskreg_data_arg (entry_stmt))
5899 parcopy_stmt = stmt;
5900 break;
5905 gcc_assert (parcopy_stmt != NULL);
5906 arg = DECL_ARGUMENTS (child_fn);
5908 if (!gimple_in_ssa_p (cfun))
5910 if (gimple_assign_lhs (parcopy_stmt) == arg)
5911 gsi_remove (&gsi, true);
5912 else
5914 /* ?? Is setting the subcode really necessary ?? */
5915 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg));
5916 gimple_assign_set_rhs1 (parcopy_stmt, arg);
5919 else
5921 /* If we are in ssa form, we must load the value from the default
5922 definition of the argument. That should not be defined now,
5923 since the argument is not used uninitialized. */
5924 gcc_assert (ssa_default_def (cfun, arg) == NULL);
5925 narg = make_ssa_name (arg, gimple_build_nop ());
5926 set_ssa_default_def (cfun, arg, narg);
5927 /* ?? Is setting the subcode really necessary ?? */
5928 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (narg));
5929 gimple_assign_set_rhs1 (parcopy_stmt, narg);
5930 update_stmt (parcopy_stmt);
5934 /* Declare local variables needed in CHILD_CFUN. */
5935 block = DECL_INITIAL (child_fn);
5936 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
5937 /* The gimplifier could record temporaries in parallel/task block
5938 rather than in containing function's local_decls chain,
5939 which would mean cgraph missed finalizing them. Do it now. */
5940 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
5941 if (TREE_CODE (t) == VAR_DECL
5942 && TREE_STATIC (t)
5943 && !DECL_EXTERNAL (t))
5944 varpool_node::finalize_decl (t);
5945 DECL_SAVED_TREE (child_fn) = NULL;
5946 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5947 gimple_set_body (child_fn, NULL);
5948 TREE_USED (block) = 1;
5950 /* Reset DECL_CONTEXT on function arguments. */
5951 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
5952 DECL_CONTEXT (t) = child_fn;
5954 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5955 so that it can be moved to the child function. */
5956 gsi = gsi_last_bb (entry_bb);
5957 stmt = gsi_stmt (gsi);
5958 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
5959 || gimple_code (stmt) == GIMPLE_OMP_TASK));
5960 gsi_remove (&gsi, true);
5961 e = split_block (entry_bb, stmt);
5962 entry_bb = e->dest;
5963 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
5965 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5966 if (exit_bb)
5968 gsi = gsi_last_bb (exit_bb);
5969 gcc_assert (!gsi_end_p (gsi)
5970 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
5971 stmt = gimple_build_return (NULL);
5972 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
5973 gsi_remove (&gsi, true);
5976 /* Move the parallel region into CHILD_CFUN. */
5978 if (gimple_in_ssa_p (cfun))
5980 init_tree_ssa (child_cfun);
5981 init_ssa_operands (child_cfun);
5982 child_cfun->gimple_df->in_ssa_p = true;
5983 block = NULL_TREE;
5985 else
5986 block = gimple_block (entry_stmt);
5988 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
5989 if (exit_bb)
5990 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
5991 /* When the OMP expansion process cannot guarantee an up-to-date
5992 loop tree arrange for the child function to fixup loops. */
5993 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
5994 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
5996 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5997 num = vec_safe_length (child_cfun->local_decls);
5998 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
6000 t = (*child_cfun->local_decls)[srcidx];
6001 if (DECL_CONTEXT (t) == cfun->decl)
6002 continue;
6003 if (srcidx != dstidx)
6004 (*child_cfun->local_decls)[dstidx] = t;
6005 dstidx++;
6007 if (dstidx != num)
6008 vec_safe_truncate (child_cfun->local_decls, dstidx);
6010 /* Inform the callgraph about the new function. */
6011 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
6012 cgraph_node::add_new_function (child_fn, true);
6014 /* Fix the callgraph edges for child_cfun. Those for cfun will be
6015 fixed in a following pass. */
6016 push_cfun (child_cfun);
6017 if (optimize)
6018 optimize_omp_library_calls (entry_stmt);
6019 cgraph_edge::rebuild_edges ();
6021 /* Some EH regions might become dead, see PR34608. If
6022 pass_cleanup_cfg isn't the first pass to happen with the
6023 new child, these dead EH edges might cause problems.
6024 Clean them up now. */
6025 if (flag_exceptions)
6027 basic_block bb;
6028 bool changed = false;
6030 FOR_EACH_BB_FN (bb, cfun)
6031 changed |= gimple_purge_dead_eh_edges (bb);
6032 if (changed)
6033 cleanup_tree_cfg ();
6035 if (gimple_in_ssa_p (cfun))
6036 update_ssa (TODO_update_ssa);
6037 pop_cfun ();
6040 /* Emit a library call to launch the children threads. */
6041 if (is_cilk_for)
6042 expand_cilk_for_call (new_bb, entry_stmt, ws_args);
6043 else if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL)
6044 expand_parallel_call (region, new_bb, entry_stmt, ws_args);
6045 else
6046 expand_task_call (new_bb, entry_stmt);
6047 if (gimple_in_ssa_p (cfun))
6048 update_ssa (TODO_update_ssa_only_virtuals);
6052 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
6053 of the combined collapse > 1 loop constructs, generate code like:
6054 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
6055 if (cond3 is <)
6056 adj = STEP3 - 1;
6057 else
6058 adj = STEP3 + 1;
6059 count3 = (adj + N32 - N31) / STEP3;
6060 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
6061 if (cond2 is <)
6062 adj = STEP2 - 1;
6063 else
6064 adj = STEP2 + 1;
6065 count2 = (adj + N22 - N21) / STEP2;
6066 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
6067 if (cond1 is <)
6068 adj = STEP1 - 1;
6069 else
6070 adj = STEP1 + 1;
6071 count1 = (adj + N12 - N11) / STEP1;
6072 count = count1 * count2 * count3;
6073 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
6074 count = 0;
6075 and set ZERO_ITER_BB to that bb. If this isn't the outermost
6076 of the combined loop constructs, just initialize COUNTS array
6077 from the _looptemp_ clauses. */
6079 /* NOTE: It *could* be better to moosh all of the BBs together,
6080 creating one larger BB with all the computation and the unexpected
6081 jump at the end. I.e.
6083 bool zero3, zero2, zero1, zero;
6085 zero3 = N32 c3 N31;
6086 count3 = (N32 - N31) /[cl] STEP3;
6087 zero2 = N22 c2 N21;
6088 count2 = (N22 - N21) /[cl] STEP2;
6089 zero1 = N12 c1 N11;
6090 count1 = (N12 - N11) /[cl] STEP1;
6091 zero = zero3 || zero2 || zero1;
6092 count = count1 * count2 * count3;
6093 if (__builtin_expect(zero, false)) goto zero_iter_bb;
6095 After all, we expect the zero=false, and thus we expect to have to
6096 evaluate all of the comparison expressions, so short-circuiting
6097 oughtn't be a win. Since the condition isn't protecting a
6098 denominator, we're not concerned about divide-by-zero, so we can
6099 fully evaluate count even if a numerator turned out to be wrong.
6101 It seems like putting this all together would create much better
6102 scheduling opportunities, and less pressure on the chip's branch
6103 predictor. */
6105 static void
6106 expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
6107 basic_block &entry_bb, tree *counts,
6108 basic_block &zero_iter_bb, int &first_zero_iter,
6109 basic_block &l2_dom_bb)
6111 tree t, type = TREE_TYPE (fd->loop.v);
6112 gimple stmt;
6113 edge e, ne;
6114 int i;
6116 /* Collapsed loops need work for expansion into SSA form. */
6117 gcc_assert (!gimple_in_ssa_p (cfun));
6119 if (gimple_omp_for_combined_into_p (fd->for_stmt)
6120 && TREE_CODE (fd->loop.n2) != INTEGER_CST)
6122 /* First two _looptemp_ clauses are for istart/iend, counts[0]
6123 isn't supposed to be handled, as the inner loop doesn't
6124 use it. */
6125 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6126 OMP_CLAUSE__LOOPTEMP_);
6127 gcc_assert (innerc);
6128 for (i = 0; i < fd->collapse; i++)
6130 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6131 OMP_CLAUSE__LOOPTEMP_);
6132 gcc_assert (innerc);
6133 if (i)
6134 counts[i] = OMP_CLAUSE_DECL (innerc);
6135 else
6136 counts[0] = NULL_TREE;
6138 return;
6141 for (i = 0; i < fd->collapse; i++)
6143 tree itype = TREE_TYPE (fd->loops[i].v);
6145 if (SSA_VAR_P (fd->loop.n2)
6146 && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node,
6147 fold_convert (itype, fd->loops[i].n1),
6148 fold_convert (itype, fd->loops[i].n2)))
6149 == NULL_TREE || !integer_onep (t)))
6151 tree n1, n2;
6152 n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1));
6153 n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE,
6154 true, GSI_SAME_STMT);
6155 n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2));
6156 n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE,
6157 true, GSI_SAME_STMT);
6158 stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2,
6159 NULL_TREE, NULL_TREE);
6160 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
6161 if (walk_tree (gimple_cond_lhs_ptr (stmt),
6162 expand_omp_regimplify_p, NULL, NULL)
6163 || walk_tree (gimple_cond_rhs_ptr (stmt),
6164 expand_omp_regimplify_p, NULL, NULL))
6166 *gsi = gsi_for_stmt (stmt);
6167 gimple_regimplify_operands (stmt, gsi);
6169 e = split_block (entry_bb, stmt);
6170 if (zero_iter_bb == NULL)
6172 first_zero_iter = i;
6173 zero_iter_bb = create_empty_bb (entry_bb);
6174 add_bb_to_loop (zero_iter_bb, entry_bb->loop_father);
6175 *gsi = gsi_after_labels (zero_iter_bb);
6176 stmt = gimple_build_assign (fd->loop.n2,
6177 build_zero_cst (type));
6178 gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
6179 set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb,
6180 entry_bb);
6182 ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
6183 ne->probability = REG_BR_PROB_BASE / 2000 - 1;
6184 e->flags = EDGE_TRUE_VALUE;
6185 e->probability = REG_BR_PROB_BASE - ne->probability;
6186 if (l2_dom_bb == NULL)
6187 l2_dom_bb = entry_bb;
6188 entry_bb = e->dest;
6189 *gsi = gsi_last_bb (entry_bb);
6192 if (POINTER_TYPE_P (itype))
6193 itype = signed_type_for (itype);
6194 t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR
6195 ? -1 : 1));
6196 t = fold_build2 (PLUS_EXPR, itype,
6197 fold_convert (itype, fd->loops[i].step), t);
6198 t = fold_build2 (PLUS_EXPR, itype, t,
6199 fold_convert (itype, fd->loops[i].n2));
6200 t = fold_build2 (MINUS_EXPR, itype, t,
6201 fold_convert (itype, fd->loops[i].n1));
6202 /* ?? We could probably use CEIL_DIV_EXPR instead of
6203 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
6204 generate the same code in the end because generically we
6205 don't know that the values involved must be negative for
6206 GT?? */
6207 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR)
6208 t = fold_build2 (TRUNC_DIV_EXPR, itype,
6209 fold_build1 (NEGATE_EXPR, itype, t),
6210 fold_build1 (NEGATE_EXPR, itype,
6211 fold_convert (itype,
6212 fd->loops[i].step)));
6213 else
6214 t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
6215 fold_convert (itype, fd->loops[i].step));
6216 t = fold_convert (type, t);
6217 if (TREE_CODE (t) == INTEGER_CST)
6218 counts[i] = t;
6219 else
6221 counts[i] = create_tmp_reg (type, ".count");
6222 expand_omp_build_assign (gsi, counts[i], t);
6224 if (SSA_VAR_P (fd->loop.n2))
6226 if (i == 0)
6227 t = counts[0];
6228 else
6229 t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]);
6230 expand_omp_build_assign (gsi, fd->loop.n2, t);
6236 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
6237 T = V;
6238 V3 = N31 + (T % count3) * STEP3;
6239 T = T / count3;
6240 V2 = N21 + (T % count2) * STEP2;
6241 T = T / count2;
6242 V1 = N11 + T * STEP1;
6243 if this loop doesn't have an inner loop construct combined with it.
6244 If it does have an inner loop construct combined with it and the
6245 iteration count isn't known constant, store values from counts array
6246 into its _looptemp_ temporaries instead. */
6248 static void
6249 expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi,
6250 tree *counts, gimple inner_stmt, tree startvar)
6252 int i;
6253 if (gimple_omp_for_combined_p (fd->for_stmt))
6255 /* If fd->loop.n2 is constant, then no propagation of the counts
6256 is needed, they are constant. */
6257 if (TREE_CODE (fd->loop.n2) == INTEGER_CST)
6258 return;
6260 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
6261 ? gimple_omp_parallel_clauses (inner_stmt)
6262 : gimple_omp_for_clauses (inner_stmt);
6263 /* First two _looptemp_ clauses are for istart/iend, counts[0]
6264 isn't supposed to be handled, as the inner loop doesn't
6265 use it. */
6266 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
6267 gcc_assert (innerc);
6268 for (i = 0; i < fd->collapse; i++)
6270 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6271 OMP_CLAUSE__LOOPTEMP_);
6272 gcc_assert (innerc);
6273 if (i)
6275 tree tem = OMP_CLAUSE_DECL (innerc);
6276 tree t = fold_convert (TREE_TYPE (tem), counts[i]);
6277 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
6278 false, GSI_CONTINUE_LINKING);
6279 gimple stmt = gimple_build_assign (tem, t);
6280 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
6283 return;
6286 tree type = TREE_TYPE (fd->loop.v);
6287 tree tem = create_tmp_reg (type, ".tem");
6288 gimple stmt = gimple_build_assign (tem, startvar);
6289 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
6291 for (i = fd->collapse - 1; i >= 0; i--)
6293 tree vtype = TREE_TYPE (fd->loops[i].v), itype, t;
6294 itype = vtype;
6295 if (POINTER_TYPE_P (vtype))
6296 itype = signed_type_for (vtype);
6297 if (i != 0)
6298 t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]);
6299 else
6300 t = tem;
6301 t = fold_convert (itype, t);
6302 t = fold_build2 (MULT_EXPR, itype, t,
6303 fold_convert (itype, fd->loops[i].step));
6304 if (POINTER_TYPE_P (vtype))
6305 t = fold_build_pointer_plus (fd->loops[i].n1, t);
6306 else
6307 t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t);
6308 t = force_gimple_operand_gsi (gsi, t,
6309 DECL_P (fd->loops[i].v)
6310 && TREE_ADDRESSABLE (fd->loops[i].v),
6311 NULL_TREE, false,
6312 GSI_CONTINUE_LINKING);
6313 stmt = gimple_build_assign (fd->loops[i].v, t);
6314 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
6315 if (i != 0)
6317 t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]);
6318 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE,
6319 false, GSI_CONTINUE_LINKING);
6320 stmt = gimple_build_assign (tem, t);
6321 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING);
6327 /* Helper function for expand_omp_for_*. Generate code like:
6328 L10:
6329 V3 += STEP3;
6330 if (V3 cond3 N32) goto BODY_BB; else goto L11;
6331 L11:
6332 V3 = N31;
6333 V2 += STEP2;
6334 if (V2 cond2 N22) goto BODY_BB; else goto L12;
6335 L12:
6336 V2 = N21;
6337 V1 += STEP1;
6338 goto BODY_BB; */
6340 static basic_block
6341 extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb,
6342 basic_block body_bb)
6344 basic_block last_bb, bb, collapse_bb = NULL;
6345 int i;
6346 gimple_stmt_iterator gsi;
6347 edge e;
6348 tree t;
6349 gimple stmt;
6351 last_bb = cont_bb;
6352 for (i = fd->collapse - 1; i >= 0; i--)
6354 tree vtype = TREE_TYPE (fd->loops[i].v);
6356 bb = create_empty_bb (last_bb);
6357 add_bb_to_loop (bb, last_bb->loop_father);
6358 gsi = gsi_start_bb (bb);
6360 if (i < fd->collapse - 1)
6362 e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
6363 e->probability = REG_BR_PROB_BASE / 8;
6365 t = fd->loops[i + 1].n1;
6366 t = force_gimple_operand_gsi (&gsi, t,
6367 DECL_P (fd->loops[i + 1].v)
6368 && TREE_ADDRESSABLE (fd->loops[i
6369 + 1].v),
6370 NULL_TREE, false,
6371 GSI_CONTINUE_LINKING);
6372 stmt = gimple_build_assign (fd->loops[i + 1].v, t);
6373 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6375 else
6376 collapse_bb = bb;
6378 set_immediate_dominator (CDI_DOMINATORS, bb, last_bb);
6380 if (POINTER_TYPE_P (vtype))
6381 t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step);
6382 else
6383 t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step);
6384 t = force_gimple_operand_gsi (&gsi, t,
6385 DECL_P (fd->loops[i].v)
6386 && TREE_ADDRESSABLE (fd->loops[i].v),
6387 NULL_TREE, false, GSI_CONTINUE_LINKING);
6388 stmt = gimple_build_assign (fd->loops[i].v, t);
6389 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6391 if (i > 0)
6393 t = fd->loops[i].n2;
6394 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6395 false, GSI_CONTINUE_LINKING);
6396 tree v = fd->loops[i].v;
6397 if (DECL_P (v) && TREE_ADDRESSABLE (v))
6398 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE,
6399 false, GSI_CONTINUE_LINKING);
6400 t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t);
6401 stmt = gimple_build_cond_empty (t);
6402 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6403 e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
6404 e->probability = REG_BR_PROB_BASE * 7 / 8;
6406 else
6407 make_edge (bb, body_bb, EDGE_FALLTHRU);
6408 last_bb = bb;
6411 return collapse_bb;
6415 /* A subroutine of expand_omp_for. Generate code for a parallel
6416 loop with any schedule. Given parameters:
6418 for (V = N1; V cond N2; V += STEP) BODY;
6420 where COND is "<" or ">", we generate pseudocode
6422 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
6423 if (more) goto L0; else goto L3;
6425 V = istart0;
6426 iend = iend0;
6428 BODY;
6429 V += STEP;
6430 if (V cond iend) goto L1; else goto L2;
6432 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6435 If this is a combined omp parallel loop, instead of the call to
6436 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
6437 If this is gimple_omp_for_combined_p loop, then instead of assigning
6438 V and iend in L0 we assign the first two _looptemp_ clause decls of the
6439 inner GIMPLE_OMP_FOR and V += STEP; and
6440 if (V cond iend) goto L1; else goto L2; are removed.
6442 For collapsed loops, given parameters:
6443 collapse(3)
6444 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6445 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6446 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6447 BODY;
6449 we generate pseudocode
6451 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6452 if (cond3 is <)
6453 adj = STEP3 - 1;
6454 else
6455 adj = STEP3 + 1;
6456 count3 = (adj + N32 - N31) / STEP3;
6457 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6458 if (cond2 is <)
6459 adj = STEP2 - 1;
6460 else
6461 adj = STEP2 + 1;
6462 count2 = (adj + N22 - N21) / STEP2;
6463 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6464 if (cond1 is <)
6465 adj = STEP1 - 1;
6466 else
6467 adj = STEP1 + 1;
6468 count1 = (adj + N12 - N11) / STEP1;
6469 count = count1 * count2 * count3;
6470 goto Z1;
6472 count = 0;
6474 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6475 if (more) goto L0; else goto L3;
6477 V = istart0;
6478 T = V;
6479 V3 = N31 + (T % count3) * STEP3;
6480 T = T / count3;
6481 V2 = N21 + (T % count2) * STEP2;
6482 T = T / count2;
6483 V1 = N11 + T * STEP1;
6484 iend = iend0;
6486 BODY;
6487 V += 1;
6488 if (V < iend) goto L10; else goto L2;
6489 L10:
6490 V3 += STEP3;
6491 if (V3 cond3 N32) goto L1; else goto L11;
6492 L11:
6493 V3 = N31;
6494 V2 += STEP2;
6495 if (V2 cond2 N22) goto L1; else goto L12;
6496 L12:
6497 V2 = N21;
6498 V1 += STEP1;
6499 goto L1;
6501 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6506 static void
6507 expand_omp_for_generic (struct omp_region *region,
6508 struct omp_for_data *fd,
6509 enum built_in_function start_fn,
6510 enum built_in_function next_fn,
6511 gimple inner_stmt)
6513 gcc_assert (gimple_omp_for_kind (fd->for_stmt) != GF_OMP_FOR_KIND_OACC_LOOP);
6515 tree type, istart0, iend0, iend;
6516 tree t, vmain, vback, bias = NULL_TREE;
6517 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb;
6518 basic_block l2_bb = NULL, l3_bb = NULL;
6519 gimple_stmt_iterator gsi;
6520 gimple stmt;
6521 bool in_combined_parallel = is_combined_parallel (region);
6522 bool broken_loop = region->cont == NULL;
6523 edge e, ne;
6524 tree *counts = NULL;
6525 int i;
6527 gcc_assert (!broken_loop || !in_combined_parallel);
6528 gcc_assert (fd->iter_type == long_integer_type_node
6529 || !in_combined_parallel);
6531 type = TREE_TYPE (fd->loop.v);
6532 istart0 = create_tmp_var (fd->iter_type, ".istart0");
6533 iend0 = create_tmp_var (fd->iter_type, ".iend0");
6534 TREE_ADDRESSABLE (istart0) = 1;
6535 TREE_ADDRESSABLE (iend0) = 1;
6537 /* See if we need to bias by LLONG_MIN. */
6538 if (fd->iter_type == long_long_unsigned_type_node
6539 && TREE_CODE (type) == INTEGER_TYPE
6540 && !TYPE_UNSIGNED (type))
6542 tree n1, n2;
6544 if (fd->loop.cond_code == LT_EXPR)
6546 n1 = fd->loop.n1;
6547 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step);
6549 else
6551 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step);
6552 n2 = fd->loop.n1;
6554 if (TREE_CODE (n1) != INTEGER_CST
6555 || TREE_CODE (n2) != INTEGER_CST
6556 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0)))
6557 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type));
6560 entry_bb = region->entry;
6561 cont_bb = region->cont;
6562 collapse_bb = NULL;
6563 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6564 gcc_assert (broken_loop
6565 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
6566 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
6567 l1_bb = single_succ (l0_bb);
6568 if (!broken_loop)
6570 l2_bb = create_empty_bb (cont_bb);
6571 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb);
6572 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6574 else
6575 l2_bb = NULL;
6576 l3_bb = BRANCH_EDGE (entry_bb)->dest;
6577 exit_bb = region->exit;
6579 gsi = gsi_last_bb (entry_bb);
6581 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6582 if (fd->collapse > 1)
6584 gcc_assert (gimple_omp_for_kind (gsi_stmt (gsi))
6585 != GF_OMP_FOR_KIND_OACC_LOOP);
6587 int first_zero_iter = -1;
6588 basic_block zero_iter_bb = NULL, l2_dom_bb = NULL;
6590 counts = XALLOCAVEC (tree, fd->collapse);
6591 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6592 zero_iter_bb, first_zero_iter,
6593 l2_dom_bb);
6595 if (zero_iter_bb)
6597 /* Some counts[i] vars might be uninitialized if
6598 some loop has zero iterations. But the body shouldn't
6599 be executed in that case, so just avoid uninit warnings. */
6600 for (i = first_zero_iter; i < fd->collapse; i++)
6601 if (SSA_VAR_P (counts[i]))
6602 TREE_NO_WARNING (counts[i]) = 1;
6603 gsi_prev (&gsi);
6604 e = split_block (entry_bb, gsi_stmt (gsi));
6605 entry_bb = e->dest;
6606 make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU);
6607 gsi = gsi_last_bb (entry_bb);
6608 set_immediate_dominator (CDI_DOMINATORS, entry_bb,
6609 get_immediate_dominator (CDI_DOMINATORS,
6610 zero_iter_bb));
6613 if (in_combined_parallel)
6615 gcc_assert (gimple_omp_for_kind (gsi_stmt (gsi))
6616 != GF_OMP_FOR_KIND_OACC_LOOP);
6618 /* In a combined parallel loop, emit a call to
6619 GOMP_loop_foo_next. */
6620 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
6621 build_fold_addr_expr (istart0),
6622 build_fold_addr_expr (iend0));
6624 else
6626 tree t0, t1, t2, t3, t4;
6627 /* If this is not a combined parallel loop, emit a call to
6628 GOMP_loop_foo_start in ENTRY_BB. */
6629 t4 = build_fold_addr_expr (iend0);
6630 t3 = build_fold_addr_expr (istart0);
6631 t2 = fold_convert (fd->iter_type, fd->loop.step);
6632 t1 = fd->loop.n2;
6633 t0 = fd->loop.n1;
6634 if (gimple_omp_for_combined_into_p (fd->for_stmt))
6636 gcc_assert (gimple_omp_for_kind (gsi_stmt (gsi))
6637 != GF_OMP_FOR_KIND_OACC_LOOP);
6639 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
6640 OMP_CLAUSE__LOOPTEMP_);
6641 gcc_assert (innerc);
6642 t0 = OMP_CLAUSE_DECL (innerc);
6643 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6644 OMP_CLAUSE__LOOPTEMP_);
6645 gcc_assert (innerc);
6646 t1 = OMP_CLAUSE_DECL (innerc);
6648 if (POINTER_TYPE_P (TREE_TYPE (t0))
6649 && TYPE_PRECISION (TREE_TYPE (t0))
6650 != TYPE_PRECISION (fd->iter_type))
6652 /* Avoid casting pointers to integer of a different size. */
6653 tree itype = signed_type_for (type);
6654 t1 = fold_convert (fd->iter_type, fold_convert (itype, t1));
6655 t0 = fold_convert (fd->iter_type, fold_convert (itype, t0));
6657 else
6659 t1 = fold_convert (fd->iter_type, t1);
6660 t0 = fold_convert (fd->iter_type, t0);
6662 if (bias)
6664 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias);
6665 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias);
6667 if (fd->iter_type == long_integer_type_node)
6669 if (fd->chunk_size)
6671 t = fold_convert (fd->iter_type, fd->chunk_size);
6672 t = build_call_expr (builtin_decl_explicit (start_fn),
6673 6, t0, t1, t2, t, t3, t4);
6675 else
6676 t = build_call_expr (builtin_decl_explicit (start_fn),
6677 5, t0, t1, t2, t3, t4);
6679 else
6681 tree t5;
6682 tree c_bool_type;
6683 tree bfn_decl;
6685 /* The GOMP_loop_ull_*start functions have additional boolean
6686 argument, true for < loops and false for > loops.
6687 In Fortran, the C bool type can be different from
6688 boolean_type_node. */
6689 bfn_decl = builtin_decl_explicit (start_fn);
6690 c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl));
6691 t5 = build_int_cst (c_bool_type,
6692 fd->loop.cond_code == LT_EXPR ? 1 : 0);
6693 if (fd->chunk_size)
6695 tree bfn_decl = builtin_decl_explicit (start_fn);
6696 t = fold_convert (fd->iter_type, fd->chunk_size);
6697 t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4);
6699 else
6700 t = build_call_expr (builtin_decl_explicit (start_fn),
6701 6, t5, t0, t1, t2, t3, t4);
6704 if (TREE_TYPE (t) != boolean_type_node)
6705 t = fold_build2 (NE_EXPR, boolean_type_node,
6706 t, build_int_cst (TREE_TYPE (t), 0));
6707 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6708 true, GSI_SAME_STMT);
6709 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
6711 /* Remove the GIMPLE_OMP_FOR statement. */
6712 gsi_remove (&gsi, true);
6714 /* Iteration setup for sequential loop goes in L0_BB. */
6715 tree startvar = fd->loop.v;
6716 tree endvar = NULL_TREE;
6718 if (gimple_omp_for_combined_p (fd->for_stmt))
6720 gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR
6721 && gimple_omp_for_kind (inner_stmt)
6722 == GF_OMP_FOR_KIND_SIMD);
6723 tree innerc = find_omp_clause (gimple_omp_for_clauses (inner_stmt),
6724 OMP_CLAUSE__LOOPTEMP_);
6725 gcc_assert (innerc);
6726 startvar = OMP_CLAUSE_DECL (innerc);
6727 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
6728 OMP_CLAUSE__LOOPTEMP_);
6729 gcc_assert (innerc);
6730 endvar = OMP_CLAUSE_DECL (innerc);
6733 gsi = gsi_start_bb (l0_bb);
6734 t = istart0;
6735 if (bias)
6736 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
6737 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
6738 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
6739 t = fold_convert (TREE_TYPE (startvar), t);
6740 t = force_gimple_operand_gsi (&gsi, t,
6741 DECL_P (startvar)
6742 && TREE_ADDRESSABLE (startvar),
6743 NULL_TREE, false, GSI_CONTINUE_LINKING);
6744 stmt = gimple_build_assign (startvar, t);
6745 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6747 t = iend0;
6748 if (bias)
6749 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias);
6750 if (POINTER_TYPE_P (TREE_TYPE (startvar)))
6751 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t);
6752 t = fold_convert (TREE_TYPE (startvar), t);
6753 iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6754 false, GSI_CONTINUE_LINKING);
6755 if (endvar)
6757 stmt = gimple_build_assign (endvar, iend);
6758 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6759 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (iend)))
6760 stmt = gimple_build_assign (fd->loop.v, iend);
6761 else
6762 stmt = gimple_build_assign_with_ops (NOP_EXPR, fd->loop.v, iend,
6763 NULL_TREE);
6764 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6766 if (fd->collapse > 1)
6767 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
6769 if (!broken_loop)
6771 /* Code to control the increment and predicate for the sequential
6772 loop goes in the CONT_BB. */
6773 gsi = gsi_last_bb (cont_bb);
6774 stmt = gsi_stmt (gsi);
6775 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
6776 vmain = gimple_omp_continue_control_use (stmt);
6777 vback = gimple_omp_continue_control_def (stmt);
6779 if (!gimple_omp_for_combined_p (fd->for_stmt))
6781 if (POINTER_TYPE_P (type))
6782 t = fold_build_pointer_plus (vmain, fd->loop.step);
6783 else
6784 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step);
6785 t = force_gimple_operand_gsi (&gsi, t,
6786 DECL_P (vback)
6787 && TREE_ADDRESSABLE (vback),
6788 NULL_TREE, true, GSI_SAME_STMT);
6789 stmt = gimple_build_assign (vback, t);
6790 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
6792 t = build2 (fd->loop.cond_code, boolean_type_node,
6793 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback,
6794 iend);
6795 stmt = gimple_build_cond_empty (t);
6796 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
6799 /* Remove GIMPLE_OMP_CONTINUE. */
6800 gsi_remove (&gsi, true);
6802 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
6803 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, l1_bb);
6805 /* Emit code to get the next parallel iteration in L2_BB. */
6806 gsi = gsi_start_bb (l2_bb);
6808 t = build_call_expr (builtin_decl_explicit (next_fn), 2,
6809 build_fold_addr_expr (istart0),
6810 build_fold_addr_expr (iend0));
6811 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
6812 false, GSI_CONTINUE_LINKING);
6813 if (TREE_TYPE (t) != boolean_type_node)
6814 t = fold_build2 (NE_EXPR, boolean_type_node,
6815 t, build_int_cst (TREE_TYPE (t), 0));
6816 stmt = gimple_build_cond_empty (t);
6817 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
6820 /* Add the loop cleanup function. */
6821 gsi = gsi_last_bb (exit_bb);
6822 if (gimple_omp_return_nowait_p (gsi_stmt (gsi)))
6823 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT);
6824 else if (gimple_omp_return_lhs (gsi_stmt (gsi)))
6825 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL);
6826 else
6827 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END);
6828 stmt = gimple_build_call (t, 0);
6829 if (gimple_omp_return_lhs (gsi_stmt (gsi)))
6830 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (gsi)));
6831 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
6832 gsi_remove (&gsi, true);
6834 /* Connect the new blocks. */
6835 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
6836 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
6838 if (!broken_loop)
6840 gimple_seq phis;
6842 e = find_edge (cont_bb, l3_bb);
6843 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
6845 phis = phi_nodes (l3_bb);
6846 for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi))
6848 gimple phi = gsi_stmt (gsi);
6849 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
6850 PHI_ARG_DEF_FROM_EDGE (phi, e));
6852 remove_edge (e);
6854 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
6855 add_bb_to_loop (l2_bb, cont_bb->loop_father);
6856 e = find_edge (cont_bb, l1_bb);
6857 if (gimple_omp_for_combined_p (fd->for_stmt))
6859 remove_edge (e);
6860 e = NULL;
6862 else if (fd->collapse > 1)
6864 remove_edge (e);
6865 e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
6867 else
6868 e->flags = EDGE_TRUE_VALUE;
6869 if (e)
6871 e->probability = REG_BR_PROB_BASE * 7 / 8;
6872 find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8;
6874 else
6876 e = find_edge (cont_bb, l2_bb);
6877 e->flags = EDGE_FALLTHRU;
6879 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
6881 set_immediate_dominator (CDI_DOMINATORS, l2_bb,
6882 recompute_dominator (CDI_DOMINATORS, l2_bb));
6883 set_immediate_dominator (CDI_DOMINATORS, l3_bb,
6884 recompute_dominator (CDI_DOMINATORS, l3_bb));
6885 set_immediate_dominator (CDI_DOMINATORS, l0_bb,
6886 recompute_dominator (CDI_DOMINATORS, l0_bb));
6887 set_immediate_dominator (CDI_DOMINATORS, l1_bb,
6888 recompute_dominator (CDI_DOMINATORS, l1_bb));
6890 struct loop *outer_loop = alloc_loop ();
6891 outer_loop->header = l0_bb;
6892 outer_loop->latch = l2_bb;
6893 add_loop (outer_loop, l0_bb->loop_father);
6895 if (!gimple_omp_for_combined_p (fd->for_stmt))
6897 struct loop *loop = alloc_loop ();
6898 loop->header = l1_bb;
6899 /* The loop may have multiple latches. */
6900 add_loop (loop, outer_loop);
6906 /* A subroutine of expand_omp_for. Generate code for a parallel
6907 loop with static schedule and no specified chunk size. Given
6908 parameters:
6910 for (V = N1; V cond N2; V += STEP) BODY;
6912 where COND is "<" or ">", we generate pseudocode
6914 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6915 if (cond is <)
6916 adj = STEP - 1;
6917 else
6918 adj = STEP + 1;
6919 if ((__typeof (V)) -1 > 0 && cond is >)
6920 n = -(adj + N2 - N1) / -STEP;
6921 else
6922 n = (adj + N2 - N1) / STEP;
6923 q = n / nthreads;
6924 tt = n % nthreads;
6925 if (threadid < tt) goto L3; else goto L4;
6927 tt = 0;
6928 q = q + 1;
6930 s0 = q * threadid + tt;
6931 e0 = s0 + q;
6932 V = s0 * STEP + N1;
6933 if (s0 >= e0) goto L2; else goto L0;
6935 e = e0 * STEP + N1;
6937 BODY;
6938 V += STEP;
6939 if (V cond e) goto L1;
6943 static void
6944 expand_omp_for_static_nochunk (struct omp_region *region,
6945 struct omp_for_data *fd,
6946 gimple inner_stmt)
6948 tree n, q, s0, e0, e, t, tt, nthreads, threadid;
6949 tree type, itype, vmain, vback;
6950 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb;
6951 basic_block body_bb, cont_bb, collapse_bb = NULL;
6952 basic_block fin_bb;
6953 gimple_stmt_iterator gsi;
6954 gimple stmt;
6955 edge ep;
6956 bool broken_loop = region->cont == NULL;
6957 tree *counts = NULL;
6958 tree n1, n2, step;
6960 gcc_assert ((gimple_omp_for_kind (fd->for_stmt)
6961 != GF_OMP_FOR_KIND_OACC_LOOP)
6962 || !inner_stmt);
6964 itype = type = TREE_TYPE (fd->loop.v);
6965 if (POINTER_TYPE_P (type))
6966 itype = signed_type_for (type);
6968 entry_bb = region->entry;
6969 cont_bb = region->cont;
6970 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
6971 fin_bb = BRANCH_EDGE (entry_bb)->dest;
6972 gcc_assert (broken_loop
6973 || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest));
6974 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
6975 body_bb = single_succ (seq_start_bb);
6976 if (!broken_loop)
6978 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
6979 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
6981 exit_bb = region->exit;
6983 /* Iteration space partitioning goes in ENTRY_BB. */
6984 gsi = gsi_last_bb (entry_bb);
6985 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
6987 if (fd->collapse > 1)
6989 int first_zero_iter = -1;
6990 basic_block l2_dom_bb = NULL;
6992 counts = XALLOCAVEC (tree, fd->collapse);
6993 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
6994 fin_bb, first_zero_iter,
6995 l2_dom_bb);
6996 t = NULL_TREE;
6998 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
6999 t = integer_one_node;
7000 else
7001 t = fold_binary (fd->loop.cond_code, boolean_type_node,
7002 fold_convert (type, fd->loop.n1),
7003 fold_convert (type, fd->loop.n2));
7004 if (fd->collapse == 1
7005 && TYPE_UNSIGNED (type)
7006 && (t == NULL_TREE || !integer_onep (t)))
7008 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
7009 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
7010 true, GSI_SAME_STMT);
7011 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
7012 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
7013 true, GSI_SAME_STMT);
7014 stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
7015 NULL_TREE, NULL_TREE);
7016 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
7017 if (walk_tree (gimple_cond_lhs_ptr (stmt),
7018 expand_omp_regimplify_p, NULL, NULL)
7019 || walk_tree (gimple_cond_rhs_ptr (stmt),
7020 expand_omp_regimplify_p, NULL, NULL))
7022 gsi = gsi_for_stmt (stmt);
7023 gimple_regimplify_operands (stmt, &gsi);
7025 ep = split_block (entry_bb, stmt);
7026 ep->flags = EDGE_TRUE_VALUE;
7027 entry_bb = ep->dest;
7028 ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
7029 ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
7030 ep->probability = REG_BR_PROB_BASE / 2000 - 1;
7031 if (gimple_in_ssa_p (cfun))
7033 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
7034 for (gsi = gsi_start_phis (fin_bb);
7035 !gsi_end_p (gsi); gsi_next (&gsi))
7037 gimple phi = gsi_stmt (gsi);
7038 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
7039 ep, UNKNOWN_LOCATION);
7042 gsi = gsi_last_bb (entry_bb);
7045 switch (gimple_omp_for_kind (fd->for_stmt))
7047 case GF_OMP_FOR_KIND_FOR:
7048 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
7049 nthreads = build_call_expr (nthreads, 0);
7050 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
7051 threadid = build_call_expr (threadid, 0);
7052 break;
7053 case GF_OMP_FOR_KIND_DISTRIBUTE:
7054 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS);
7055 nthreads = build_call_expr (nthreads, 0);
7056 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
7057 threadid = build_call_expr (threadid, 0);
7058 break;
7059 case GF_OMP_FOR_KIND_OACC_LOOP:
7060 nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS);
7061 nthreads = build_call_expr (nthreads, 0);
7062 threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
7063 threadid = build_call_expr (threadid, 0);
7064 break;
7065 default:
7066 gcc_unreachable ();
7068 nthreads = fold_convert (itype, nthreads);
7069 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE,
7070 true, GSI_SAME_STMT);
7071 threadid = fold_convert (itype, threadid);
7072 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
7073 true, GSI_SAME_STMT);
7075 n1 = fd->loop.n1;
7076 n2 = fd->loop.n2;
7077 step = fd->loop.step;
7078 if (gimple_omp_for_combined_into_p (fd->for_stmt))
7080 gcc_assert (gimple_omp_for_kind (fd->for_stmt)
7081 != GF_OMP_FOR_KIND_OACC_LOOP);
7083 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7084 OMP_CLAUSE__LOOPTEMP_);
7085 gcc_assert (innerc);
7086 n1 = OMP_CLAUSE_DECL (innerc);
7087 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7088 OMP_CLAUSE__LOOPTEMP_);
7089 gcc_assert (innerc);
7090 n2 = OMP_CLAUSE_DECL (innerc);
7092 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
7093 true, NULL_TREE, true, GSI_SAME_STMT);
7094 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
7095 true, NULL_TREE, true, GSI_SAME_STMT);
7096 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
7097 true, NULL_TREE, true, GSI_SAME_STMT);
7099 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
7100 t = fold_build2 (PLUS_EXPR, itype, step, t);
7101 t = fold_build2 (PLUS_EXPR, itype, t, n2);
7102 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
7103 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
7104 t = fold_build2 (TRUNC_DIV_EXPR, itype,
7105 fold_build1 (NEGATE_EXPR, itype, t),
7106 fold_build1 (NEGATE_EXPR, itype, step));
7107 else
7108 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
7109 t = fold_convert (itype, t);
7110 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
7112 q = create_tmp_reg (itype, "q");
7113 t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads);
7114 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
7115 gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT);
7117 tt = create_tmp_reg (itype, "tt");
7118 t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads);
7119 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT);
7120 gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT);
7122 t = build2 (LT_EXPR, boolean_type_node, threadid, tt);
7123 stmt = gimple_build_cond_empty (t);
7124 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
7126 second_bb = split_block (entry_bb, stmt)->dest;
7127 gsi = gsi_last_bb (second_bb);
7128 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
7130 gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)),
7131 GSI_SAME_STMT);
7132 stmt = gimple_build_assign_with_ops (PLUS_EXPR, q, q,
7133 build_int_cst (itype, 1));
7134 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
7136 third_bb = split_block (second_bb, stmt)->dest;
7137 gsi = gsi_last_bb (third_bb);
7138 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
7140 t = build2 (MULT_EXPR, itype, q, threadid);
7141 t = build2 (PLUS_EXPR, itype, t, tt);
7142 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
7144 t = fold_build2 (PLUS_EXPR, itype, s0, q);
7145 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT);
7147 t = build2 (GE_EXPR, boolean_type_node, s0, e0);
7148 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
7150 /* Remove the GIMPLE_OMP_FOR statement. */
7151 gsi_remove (&gsi, true);
7153 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7154 gsi = gsi_start_bb (seq_start_bb);
7156 tree startvar = fd->loop.v;
7157 tree endvar = NULL_TREE;
7159 if (gimple_omp_for_combined_p (fd->for_stmt))
7161 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
7162 ? gimple_omp_parallel_clauses (inner_stmt)
7163 : gimple_omp_for_clauses (inner_stmt);
7164 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
7165 gcc_assert (innerc);
7166 startvar = OMP_CLAUSE_DECL (innerc);
7167 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7168 OMP_CLAUSE__LOOPTEMP_);
7169 gcc_assert (innerc);
7170 endvar = OMP_CLAUSE_DECL (innerc);
7172 t = fold_convert (itype, s0);
7173 t = fold_build2 (MULT_EXPR, itype, t, step);
7174 if (POINTER_TYPE_P (type))
7175 t = fold_build_pointer_plus (n1, t);
7176 else
7177 t = fold_build2 (PLUS_EXPR, type, t, n1);
7178 t = fold_convert (TREE_TYPE (startvar), t);
7179 t = force_gimple_operand_gsi (&gsi, t,
7180 DECL_P (startvar)
7181 && TREE_ADDRESSABLE (startvar),
7182 NULL_TREE, false, GSI_CONTINUE_LINKING);
7183 stmt = gimple_build_assign (startvar, t);
7184 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7186 t = fold_convert (itype, e0);
7187 t = fold_build2 (MULT_EXPR, itype, t, step);
7188 if (POINTER_TYPE_P (type))
7189 t = fold_build_pointer_plus (n1, t);
7190 else
7191 t = fold_build2 (PLUS_EXPR, type, t, n1);
7192 t = fold_convert (TREE_TYPE (startvar), t);
7193 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7194 false, GSI_CONTINUE_LINKING);
7195 if (endvar)
7197 stmt = gimple_build_assign (endvar, e);
7198 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7199 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
7200 stmt = gimple_build_assign (fd->loop.v, e);
7201 else
7202 stmt = gimple_build_assign_with_ops (NOP_EXPR, fd->loop.v, e,
7203 NULL_TREE);
7204 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7206 if (fd->collapse > 1)
7207 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
7209 if (!broken_loop)
7211 /* The code controlling the sequential loop replaces the
7212 GIMPLE_OMP_CONTINUE. */
7213 gsi = gsi_last_bb (cont_bb);
7214 stmt = gsi_stmt (gsi);
7215 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
7216 vmain = gimple_omp_continue_control_use (stmt);
7217 vback = gimple_omp_continue_control_def (stmt);
7219 if (!gimple_omp_for_combined_p (fd->for_stmt))
7221 if (POINTER_TYPE_P (type))
7222 t = fold_build_pointer_plus (vmain, step);
7223 else
7224 t = fold_build2 (PLUS_EXPR, type, vmain, step);
7225 t = force_gimple_operand_gsi (&gsi, t,
7226 DECL_P (vback)
7227 && TREE_ADDRESSABLE (vback),
7228 NULL_TREE, true, GSI_SAME_STMT);
7229 stmt = gimple_build_assign (vback, t);
7230 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
7232 t = build2 (fd->loop.cond_code, boolean_type_node,
7233 DECL_P (vback) && TREE_ADDRESSABLE (vback)
7234 ? t : vback, e);
7235 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
7238 /* Remove the GIMPLE_OMP_CONTINUE statement. */
7239 gsi_remove (&gsi, true);
7241 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
7242 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
7245 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7246 gsi = gsi_last_bb (exit_bb);
7247 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
7249 gcc_assert (gimple_omp_for_kind (fd->for_stmt)
7250 != GF_OMP_FOR_KIND_OACC_LOOP);
7252 t = gimple_omp_return_lhs (gsi_stmt (gsi));
7253 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
7255 gsi_remove (&gsi, true);
7257 /* Connect all the blocks. */
7258 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
7259 ep->probability = REG_BR_PROB_BASE / 4 * 3;
7260 ep = find_edge (entry_bb, second_bb);
7261 ep->flags = EDGE_TRUE_VALUE;
7262 ep->probability = REG_BR_PROB_BASE / 4;
7263 find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
7264 find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
7266 if (!broken_loop)
7268 ep = find_edge (cont_bb, body_bb);
7269 if (gimple_omp_for_combined_p (fd->for_stmt))
7271 gcc_assert (gimple_omp_for_kind (fd->for_stmt)
7272 != GF_OMP_FOR_KIND_OACC_LOOP);
7274 remove_edge (ep);
7275 ep = NULL;
7277 else if (fd->collapse > 1)
7279 remove_edge (ep);
7280 ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
7282 else
7283 ep->flags = EDGE_TRUE_VALUE;
7284 find_edge (cont_bb, fin_bb)->flags
7285 = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
7288 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb);
7289 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb);
7290 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb);
7292 set_immediate_dominator (CDI_DOMINATORS, body_bb,
7293 recompute_dominator (CDI_DOMINATORS, body_bb));
7294 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
7295 recompute_dominator (CDI_DOMINATORS, fin_bb));
7297 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt))
7299 struct loop *loop = alloc_loop ();
7300 loop->header = body_bb;
7301 if (collapse_bb == NULL)
7302 loop->latch = cont_bb;
7303 add_loop (loop, body_bb->loop_father);
7308 /* A subroutine of expand_omp_for. Generate code for a parallel
7309 loop with static schedule and a specified chunk size. Given
7310 parameters:
7312 for (V = N1; V cond N2; V += STEP) BODY;
7314 where COND is "<" or ">", we generate pseudocode
7316 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
7317 if (cond is <)
7318 adj = STEP - 1;
7319 else
7320 adj = STEP + 1;
7321 if ((__typeof (V)) -1 > 0 && cond is >)
7322 n = -(adj + N2 - N1) / -STEP;
7323 else
7324 n = (adj + N2 - N1) / STEP;
7325 trip = 0;
7326 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
7327 here so that V is defined
7328 if the loop is not entered
7330 s0 = (trip * nthreads + threadid) * CHUNK;
7331 e0 = min(s0 + CHUNK, n);
7332 if (s0 < n) goto L1; else goto L4;
7334 V = s0 * STEP + N1;
7335 e = e0 * STEP + N1;
7337 BODY;
7338 V += STEP;
7339 if (V cond e) goto L2; else goto L3;
7341 trip += 1;
7342 goto L0;
7346 static void
7347 expand_omp_for_static_chunk (struct omp_region *region,
7348 struct omp_for_data *fd, gimple inner_stmt)
7350 tree n, s0, e0, e, t;
7351 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
7352 tree type, itype, vmain, vback, vextra;
7353 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
7354 basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb;
7355 gimple_stmt_iterator gsi;
7356 gimple stmt;
7357 edge se;
7358 bool broken_loop = region->cont == NULL;
7359 tree *counts = NULL;
7360 tree n1, n2, step;
7362 gcc_assert ((gimple_omp_for_kind (fd->for_stmt)
7363 != GF_OMP_FOR_KIND_OACC_LOOP)
7364 || !inner_stmt);
7366 itype = type = TREE_TYPE (fd->loop.v);
7367 if (POINTER_TYPE_P (type))
7368 itype = signed_type_for (type);
7370 entry_bb = region->entry;
7371 se = split_block (entry_bb, last_stmt (entry_bb));
7372 entry_bb = se->src;
7373 iter_part_bb = se->dest;
7374 cont_bb = region->cont;
7375 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
7376 fin_bb = BRANCH_EDGE (iter_part_bb)->dest;
7377 gcc_assert (broken_loop
7378 || fin_bb == FALLTHRU_EDGE (cont_bb)->dest);
7379 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
7380 body_bb = single_succ (seq_start_bb);
7381 if (!broken_loop)
7383 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
7384 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
7385 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
7387 exit_bb = region->exit;
7389 /* Trip and adjustment setup goes in ENTRY_BB. */
7390 gsi = gsi_last_bb (entry_bb);
7391 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
7393 if (fd->collapse > 1)
7395 int first_zero_iter = -1;
7396 basic_block l2_dom_bb = NULL;
7398 counts = XALLOCAVEC (tree, fd->collapse);
7399 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
7400 fin_bb, first_zero_iter,
7401 l2_dom_bb);
7402 t = NULL_TREE;
7404 else if (gimple_omp_for_combined_into_p (fd->for_stmt))
7405 t = integer_one_node;
7406 else
7407 t = fold_binary (fd->loop.cond_code, boolean_type_node,
7408 fold_convert (type, fd->loop.n1),
7409 fold_convert (type, fd->loop.n2));
7410 if (fd->collapse == 1
7411 && TYPE_UNSIGNED (type)
7412 && (t == NULL_TREE || !integer_onep (t)))
7414 n1 = fold_convert (type, unshare_expr (fd->loop.n1));
7415 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE,
7416 true, GSI_SAME_STMT);
7417 n2 = fold_convert (type, unshare_expr (fd->loop.n2));
7418 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE,
7419 true, GSI_SAME_STMT);
7420 stmt = gimple_build_cond (fd->loop.cond_code, n1, n2,
7421 NULL_TREE, NULL_TREE);
7422 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
7423 if (walk_tree (gimple_cond_lhs_ptr (stmt),
7424 expand_omp_regimplify_p, NULL, NULL)
7425 || walk_tree (gimple_cond_rhs_ptr (stmt),
7426 expand_omp_regimplify_p, NULL, NULL))
7428 gsi = gsi_for_stmt (stmt);
7429 gimple_regimplify_operands (stmt, &gsi);
7431 se = split_block (entry_bb, stmt);
7432 se->flags = EDGE_TRUE_VALUE;
7433 entry_bb = se->dest;
7434 se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
7435 se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
7436 se->probability = REG_BR_PROB_BASE / 2000 - 1;
7437 if (gimple_in_ssa_p (cfun))
7439 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
7440 for (gsi = gsi_start_phis (fin_bb);
7441 !gsi_end_p (gsi); gsi_next (&gsi))
7443 gimple phi = gsi_stmt (gsi);
7444 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx),
7445 se, UNKNOWN_LOCATION);
7448 gsi = gsi_last_bb (entry_bb);
7451 switch (gimple_omp_for_kind (fd->for_stmt))
7453 case GF_OMP_FOR_KIND_FOR:
7454 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
7455 nthreads = build_call_expr (nthreads, 0);
7456 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
7457 threadid = build_call_expr (threadid, 0);
7458 break;
7459 case GF_OMP_FOR_KIND_DISTRIBUTE:
7460 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS);
7461 nthreads = build_call_expr (nthreads, 0);
7462 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM);
7463 threadid = build_call_expr (threadid, 0);
7464 break;
7465 case GF_OMP_FOR_KIND_OACC_LOOP:
7466 nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS);
7467 nthreads = build_call_expr (nthreads, 0);
7468 threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM);
7469 threadid = build_call_expr (threadid, 0);
7470 break;
7471 default:
7472 gcc_unreachable ();
7474 nthreads = fold_convert (itype, nthreads);
7475 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE,
7476 true, GSI_SAME_STMT);
7477 threadid = fold_convert (itype, threadid);
7478 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE,
7479 true, GSI_SAME_STMT);
7481 n1 = fd->loop.n1;
7482 n2 = fd->loop.n2;
7483 step = fd->loop.step;
7484 if (gimple_omp_for_combined_into_p (fd->for_stmt))
7486 gcc_assert (gimple_omp_for_kind (fd->for_stmt)
7487 != GF_OMP_FOR_KIND_OACC_LOOP);
7489 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
7490 OMP_CLAUSE__LOOPTEMP_);
7491 gcc_assert (innerc);
7492 n1 = OMP_CLAUSE_DECL (innerc);
7493 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7494 OMP_CLAUSE__LOOPTEMP_);
7495 gcc_assert (innerc);
7496 n2 = OMP_CLAUSE_DECL (innerc);
7498 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1),
7499 true, NULL_TREE, true, GSI_SAME_STMT);
7500 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2),
7501 true, NULL_TREE, true, GSI_SAME_STMT);
7502 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step),
7503 true, NULL_TREE, true, GSI_SAME_STMT);
7504 fd->chunk_size
7505 = force_gimple_operand_gsi (&gsi, fold_convert (itype, fd->chunk_size),
7506 true, NULL_TREE, true, GSI_SAME_STMT);
7508 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
7509 t = fold_build2 (PLUS_EXPR, itype, step, t);
7510 t = fold_build2 (PLUS_EXPR, itype, t, n2);
7511 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1));
7512 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR)
7513 t = fold_build2 (TRUNC_DIV_EXPR, itype,
7514 fold_build1 (NEGATE_EXPR, itype, t),
7515 fold_build1 (NEGATE_EXPR, itype, step));
7516 else
7517 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step);
7518 t = fold_convert (itype, t);
7519 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7520 true, GSI_SAME_STMT);
7522 trip_var = create_tmp_reg (itype, ".trip");
7523 if (gimple_in_ssa_p (cfun))
7525 trip_init = make_ssa_name (trip_var, NULL);
7526 trip_main = make_ssa_name (trip_var, NULL);
7527 trip_back = make_ssa_name (trip_var, NULL);
7529 else
7531 trip_init = trip_var;
7532 trip_main = trip_var;
7533 trip_back = trip_var;
7536 stmt = gimple_build_assign (trip_init, build_int_cst (itype, 0));
7537 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
7539 t = fold_build2 (MULT_EXPR, itype, threadid, fd->chunk_size);
7540 t = fold_build2 (MULT_EXPR, itype, t, step);
7541 if (POINTER_TYPE_P (type))
7542 t = fold_build_pointer_plus (n1, t);
7543 else
7544 t = fold_build2 (PLUS_EXPR, type, t, n1);
7545 vextra = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7546 true, GSI_SAME_STMT);
7548 /* Remove the GIMPLE_OMP_FOR. */
7549 gsi_remove (&gsi, true);
7551 /* Iteration space partitioning goes in ITER_PART_BB. */
7552 gsi = gsi_last_bb (iter_part_bb);
7554 t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads);
7555 t = fold_build2 (PLUS_EXPR, itype, t, threadid);
7556 t = fold_build2 (MULT_EXPR, itype, t, fd->chunk_size);
7557 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7558 false, GSI_CONTINUE_LINKING);
7560 t = fold_build2 (PLUS_EXPR, itype, s0, fd->chunk_size);
7561 t = fold_build2 (MIN_EXPR, itype, t, n);
7562 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7563 false, GSI_CONTINUE_LINKING);
7565 t = build2 (LT_EXPR, boolean_type_node, s0, n);
7566 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING);
7568 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7569 gsi = gsi_start_bb (seq_start_bb);
7571 tree startvar = fd->loop.v;
7572 tree endvar = NULL_TREE;
7574 if (gimple_omp_for_combined_p (fd->for_stmt))
7576 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL
7577 ? gimple_omp_parallel_clauses (inner_stmt)
7578 : gimple_omp_for_clauses (inner_stmt);
7579 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
7580 gcc_assert (innerc);
7581 startvar = OMP_CLAUSE_DECL (innerc);
7582 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
7583 OMP_CLAUSE__LOOPTEMP_);
7584 gcc_assert (innerc);
7585 endvar = OMP_CLAUSE_DECL (innerc);
7588 t = fold_convert (itype, s0);
7589 t = fold_build2 (MULT_EXPR, itype, t, step);
7590 if (POINTER_TYPE_P (type))
7591 t = fold_build_pointer_plus (n1, t);
7592 else
7593 t = fold_build2 (PLUS_EXPR, type, t, n1);
7594 t = fold_convert (TREE_TYPE (startvar), t);
7595 t = force_gimple_operand_gsi (&gsi, t,
7596 DECL_P (startvar)
7597 && TREE_ADDRESSABLE (startvar),
7598 NULL_TREE, false, GSI_CONTINUE_LINKING);
7599 stmt = gimple_build_assign (startvar, t);
7600 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7602 t = fold_convert (itype, e0);
7603 t = fold_build2 (MULT_EXPR, itype, t, step);
7604 if (POINTER_TYPE_P (type))
7605 t = fold_build_pointer_plus (n1, t);
7606 else
7607 t = fold_build2 (PLUS_EXPR, type, t, n1);
7608 t = fold_convert (TREE_TYPE (startvar), t);
7609 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7610 false, GSI_CONTINUE_LINKING);
7611 if (endvar)
7613 stmt = gimple_build_assign (endvar, e);
7614 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7615 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e)))
7616 stmt = gimple_build_assign (fd->loop.v, e);
7617 else
7618 stmt = gimple_build_assign_with_ops (NOP_EXPR, fd->loop.v, e,
7619 NULL_TREE);
7620 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7622 if (fd->collapse > 1)
7623 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar);
7625 if (!broken_loop)
7627 /* The code controlling the sequential loop goes in CONT_BB,
7628 replacing the GIMPLE_OMP_CONTINUE. */
7629 gsi = gsi_last_bb (cont_bb);
7630 stmt = gsi_stmt (gsi);
7631 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
7632 vmain = gimple_omp_continue_control_use (stmt);
7633 vback = gimple_omp_continue_control_def (stmt);
7635 if (!gimple_omp_for_combined_p (fd->for_stmt))
7637 if (POINTER_TYPE_P (type))
7638 t = fold_build_pointer_plus (vmain, step);
7639 else
7640 t = fold_build2 (PLUS_EXPR, type, vmain, step);
7641 if (DECL_P (vback) && TREE_ADDRESSABLE (vback))
7642 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
7643 true, GSI_SAME_STMT);
7644 stmt = gimple_build_assign (vback, t);
7645 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
7647 t = build2 (fd->loop.cond_code, boolean_type_node,
7648 DECL_P (vback) && TREE_ADDRESSABLE (vback)
7649 ? t : vback, e);
7650 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT);
7653 /* Remove GIMPLE_OMP_CONTINUE. */
7654 gsi_remove (&gsi, true);
7656 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt))
7657 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb);
7659 /* Trip update code goes into TRIP_UPDATE_BB. */
7660 gsi = gsi_start_bb (trip_update_bb);
7662 t = build_int_cst (itype, 1);
7663 t = build2 (PLUS_EXPR, itype, trip_main, t);
7664 stmt = gimple_build_assign (trip_back, t);
7665 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
7668 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7669 gsi = gsi_last_bb (exit_bb);
7670 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi)))
7672 gcc_assert (gimple_omp_for_kind (fd->for_stmt)
7673 != GF_OMP_FOR_KIND_OACC_LOOP);
7675 t = gimple_omp_return_lhs (gsi_stmt (gsi));
7676 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT);
7678 gsi_remove (&gsi, true);
7680 /* Connect the new blocks. */
7681 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
7682 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
7684 if (!broken_loop)
7686 se = find_edge (cont_bb, body_bb);
7687 if (gimple_omp_for_combined_p (fd->for_stmt))
7689 gcc_assert (gimple_omp_for_kind (fd->for_stmt)
7690 != GF_OMP_FOR_KIND_OACC_LOOP);
7692 remove_edge (se);
7693 se = NULL;
7695 else if (fd->collapse > 1)
7697 remove_edge (se);
7698 se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE);
7700 else
7701 se->flags = EDGE_TRUE_VALUE;
7702 find_edge (cont_bb, trip_update_bb)->flags
7703 = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU;
7705 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb);
7708 if (gimple_in_ssa_p (cfun))
7710 gimple_stmt_iterator psi;
7711 gimple phi;
7712 edge re, ene;
7713 edge_var_map *vm;
7714 size_t i;
7716 gcc_assert (fd->collapse == 1 && !broken_loop);
7718 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7719 remove arguments of the phi nodes in fin_bb. We need to create
7720 appropriate phi nodes in iter_part_bb instead. */
7721 se = single_pred_edge (fin_bb);
7722 re = single_succ_edge (trip_update_bb);
7723 vec<edge_var_map> *head = redirect_edge_var_map_vector (re);
7724 ene = single_succ_edge (entry_bb);
7726 psi = gsi_start_phis (fin_bb);
7727 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm);
7728 gsi_next (&psi), ++i)
7730 gimple nphi;
7731 source_location locus;
7733 phi = gsi_stmt (psi);
7734 t = gimple_phi_result (phi);
7735 gcc_assert (t == redirect_edge_var_map_result (vm));
7736 nphi = create_phi_node (t, iter_part_bb);
7738 t = PHI_ARG_DEF_FROM_EDGE (phi, se);
7739 locus = gimple_phi_arg_location_from_edge (phi, se);
7741 /* A special case -- fd->loop.v is not yet computed in
7742 iter_part_bb, we need to use vextra instead. */
7743 if (t == fd->loop.v)
7744 t = vextra;
7745 add_phi_arg (nphi, t, ene, locus);
7746 locus = redirect_edge_var_map_location (vm);
7747 add_phi_arg (nphi, redirect_edge_var_map_def (vm), re, locus);
7749 gcc_assert (!gsi_end_p (psi) && i == head->length ());
7750 redirect_edge_var_map_clear (re);
7751 while (1)
7753 psi = gsi_start_phis (fin_bb);
7754 if (gsi_end_p (psi))
7755 break;
7756 remove_phi_node (&psi, false);
7759 /* Make phi node for trip. */
7760 phi = create_phi_node (trip_main, iter_part_bb);
7761 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb),
7762 UNKNOWN_LOCATION);
7763 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb),
7764 UNKNOWN_LOCATION);
7767 if (!broken_loop)
7768 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
7769 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
7770 recompute_dominator (CDI_DOMINATORS, iter_part_bb));
7771 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
7772 recompute_dominator (CDI_DOMINATORS, fin_bb));
7773 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
7774 recompute_dominator (CDI_DOMINATORS, seq_start_bb));
7775 set_immediate_dominator (CDI_DOMINATORS, body_bb,
7776 recompute_dominator (CDI_DOMINATORS, body_bb));
7778 if (!broken_loop)
7780 struct loop *trip_loop = alloc_loop ();
7781 trip_loop->header = iter_part_bb;
7782 trip_loop->latch = trip_update_bb;
7783 add_loop (trip_loop, iter_part_bb->loop_father);
7785 if (!gimple_omp_for_combined_p (fd->for_stmt))
7787 struct loop *loop = alloc_loop ();
7788 loop->header = body_bb;
7789 if (collapse_bb == NULL)
7790 loop->latch = cont_bb;
7791 add_loop (loop, trip_loop);
7796 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7797 Given parameters:
7798 for (V = N1; V cond N2; V += STEP) BODY;
7800 where COND is "<" or ">" or "!=", we generate pseudocode
7802 for (ind_var = low; ind_var < high; ind_var++)
7804 V = n1 + (ind_var * STEP)
7806 <BODY>
7809 In the above pseudocode, low and high are function parameters of the
7810 child function. In the function below, we are inserting a temp.
7811 variable that will be making a call to two OMP functions that will not be
7812 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7813 with _Cilk_for). These functions are replaced with low and high
7814 by the function that handles taskreg. */
7817 static void
7818 expand_cilk_for (struct omp_region *region, struct omp_for_data *fd)
7820 bool broken_loop = region->cont == NULL;
7821 basic_block entry_bb = region->entry;
7822 basic_block cont_bb = region->cont;
7824 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
7825 gcc_assert (broken_loop
7826 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
7827 basic_block l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
7828 basic_block l1_bb, l2_bb;
7830 if (!broken_loop)
7832 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
7833 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
7834 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
7835 l2_bb = BRANCH_EDGE (entry_bb)->dest;
7837 else
7839 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
7840 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
7841 l2_bb = single_succ (l1_bb);
7843 basic_block exit_bb = region->exit;
7844 basic_block l2_dom_bb = NULL;
7846 gimple_stmt_iterator gsi = gsi_last_bb (entry_bb);
7848 /* Below statements until the "tree high_val = ..." are pseudo statements
7849 used to pass information to be used by expand_omp_taskreg.
7850 low_val and high_val will be replaced by the __low and __high
7851 parameter from the child function.
7853 The call_exprs part is a place-holder, it is mainly used
7854 to distinctly identify to the top-level part that this is
7855 where we should put low and high (reasoning given in header
7856 comment). */
7858 tree child_fndecl
7859 = gimple_omp_parallel_child_fn (last_stmt (region->outer->entry));
7860 tree t, low_val = NULL_TREE, high_val = NULL_TREE;
7861 for (t = DECL_ARGUMENTS (child_fndecl); t; t = TREE_CHAIN (t))
7863 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__high"))
7864 high_val = t;
7865 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__low"))
7866 low_val = t;
7868 gcc_assert (low_val && high_val);
7870 tree type = TREE_TYPE (low_val);
7871 tree ind_var = create_tmp_reg (type, "__cilk_ind_var");
7872 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
7874 /* Not needed in SSA form right now. */
7875 gcc_assert (!gimple_in_ssa_p (cfun));
7876 if (l2_dom_bb == NULL)
7877 l2_dom_bb = l1_bb;
7879 tree n1 = low_val;
7880 tree n2 = high_val;
7882 gimple stmt = gimple_build_assign (ind_var, n1);
7884 /* Replace the GIMPLE_OMP_FOR statement. */
7885 gsi_replace (&gsi, stmt, true);
7887 if (!broken_loop)
7889 /* Code to control the increment goes in the CONT_BB. */
7890 gsi = gsi_last_bb (cont_bb);
7891 stmt = gsi_stmt (gsi);
7892 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
7893 stmt = gimple_build_assign_with_ops (PLUS_EXPR, ind_var, ind_var,
7894 build_one_cst (type));
7896 /* Replace GIMPLE_OMP_CONTINUE. */
7897 gsi_replace (&gsi, stmt, true);
7900 /* Emit the condition in L1_BB. */
7901 gsi = gsi_after_labels (l1_bb);
7902 t = fold_build2 (MULT_EXPR, TREE_TYPE (fd->loop.step),
7903 fold_convert (TREE_TYPE (fd->loop.step), ind_var),
7904 fd->loop.step);
7905 if (POINTER_TYPE_P (TREE_TYPE (fd->loop.n1)))
7906 t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (fd->loop.n1),
7907 fd->loop.n1, fold_convert (sizetype, t));
7908 else
7909 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loop.n1),
7910 fd->loop.n1, fold_convert (TREE_TYPE (fd->loop.n1), t));
7911 t = fold_convert (TREE_TYPE (fd->loop.v), t);
7912 expand_omp_build_assign (&gsi, fd->loop.v, t);
7914 /* The condition is always '<' since the runtime will fill in the low
7915 and high values. */
7916 stmt = gimple_build_cond (LT_EXPR, ind_var, n2, NULL_TREE, NULL_TREE);
7917 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
7919 /* Remove GIMPLE_OMP_RETURN. */
7920 gsi = gsi_last_bb (exit_bb);
7921 gsi_remove (&gsi, true);
7923 /* Connect the new blocks. */
7924 remove_edge (FALLTHRU_EDGE (entry_bb));
7926 edge e, ne;
7927 if (!broken_loop)
7929 remove_edge (BRANCH_EDGE (entry_bb));
7930 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
7932 e = BRANCH_EDGE (l1_bb);
7933 ne = FALLTHRU_EDGE (l1_bb);
7934 e->flags = EDGE_TRUE_VALUE;
7936 else
7938 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
7940 ne = single_succ_edge (l1_bb);
7941 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
7944 ne->flags = EDGE_FALSE_VALUE;
7945 e->probability = REG_BR_PROB_BASE * 7 / 8;
7946 ne->probability = REG_BR_PROB_BASE / 8;
7948 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
7949 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
7950 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
7952 if (!broken_loop)
7954 struct loop *loop = alloc_loop ();
7955 loop->header = l1_bb;
7956 loop->latch = cont_bb;
7957 add_loop (loop, l1_bb->loop_father);
7958 loop->safelen = INT_MAX;
7961 /* Pick the correct library function based on the precision of the
7962 induction variable type. */
7963 tree lib_fun = NULL_TREE;
7964 if (TYPE_PRECISION (type) == 32)
7965 lib_fun = cilk_for_32_fndecl;
7966 else if (TYPE_PRECISION (type) == 64)
7967 lib_fun = cilk_for_64_fndecl;
7968 else
7969 gcc_unreachable ();
7971 gcc_assert (fd->sched_kind == OMP_CLAUSE_SCHEDULE_CILKFOR);
7973 /* WS_ARGS contains the library function flavor to call:
7974 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7975 user-defined grain value. If the user does not define one, then zero
7976 is passed in by the parser. */
7977 vec_alloc (region->ws_args, 2);
7978 region->ws_args->quick_push (lib_fun);
7979 region->ws_args->quick_push (fd->chunk_size);
7982 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7983 loop. Given parameters:
7985 for (V = N1; V cond N2; V += STEP) BODY;
7987 where COND is "<" or ">", we generate pseudocode
7989 V = N1;
7990 goto L1;
7992 BODY;
7993 V += STEP;
7995 if (V cond N2) goto L0; else goto L2;
7998 For collapsed loops, given parameters:
7999 collapse(3)
8000 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8001 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8002 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8003 BODY;
8005 we generate pseudocode
8007 if (cond3 is <)
8008 adj = STEP3 - 1;
8009 else
8010 adj = STEP3 + 1;
8011 count3 = (adj + N32 - N31) / STEP3;
8012 if (cond2 is <)
8013 adj = STEP2 - 1;
8014 else
8015 adj = STEP2 + 1;
8016 count2 = (adj + N22 - N21) / STEP2;
8017 if (cond1 is <)
8018 adj = STEP1 - 1;
8019 else
8020 adj = STEP1 + 1;
8021 count1 = (adj + N12 - N11) / STEP1;
8022 count = count1 * count2 * count3;
8023 V = 0;
8024 V1 = N11;
8025 V2 = N21;
8026 V3 = N31;
8027 goto L1;
8029 BODY;
8030 V += 1;
8031 V3 += STEP3;
8032 V2 += (V3 cond3 N32) ? 0 : STEP2;
8033 V3 = (V3 cond3 N32) ? V3 : N31;
8034 V1 += (V2 cond2 N22) ? 0 : STEP1;
8035 V2 = (V2 cond2 N22) ? V2 : N21;
8037 if (V < count) goto L0; else goto L2;
8042 static void
8043 expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
8045 tree type, t;
8046 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb;
8047 gimple_stmt_iterator gsi;
8048 gimple stmt;
8049 bool broken_loop = region->cont == NULL;
8050 edge e, ne;
8051 tree *counts = NULL;
8052 int i;
8053 tree safelen = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
8054 OMP_CLAUSE_SAFELEN);
8055 tree simduid = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
8056 OMP_CLAUSE__SIMDUID_);
8057 tree n1, n2;
8059 type = TREE_TYPE (fd->loop.v);
8060 entry_bb = region->entry;
8061 cont_bb = region->cont;
8062 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
8063 gcc_assert (broken_loop
8064 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
8065 l0_bb = FALLTHRU_EDGE (entry_bb)->dest;
8066 if (!broken_loop)
8068 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb);
8069 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
8070 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest;
8071 l2_bb = BRANCH_EDGE (entry_bb)->dest;
8073 else
8075 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL;
8076 l1_bb = split_edge (BRANCH_EDGE (entry_bb));
8077 l2_bb = single_succ (l1_bb);
8079 exit_bb = region->exit;
8080 l2_dom_bb = NULL;
8082 gsi = gsi_last_bb (entry_bb);
8084 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR);
8085 /* Not needed in SSA form right now. */
8086 gcc_assert (!gimple_in_ssa_p (cfun));
8087 if (fd->collapse > 1)
8089 int first_zero_iter = -1;
8090 basic_block zero_iter_bb = l2_bb;
8092 counts = XALLOCAVEC (tree, fd->collapse);
8093 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts,
8094 zero_iter_bb, first_zero_iter,
8095 l2_dom_bb);
8097 if (l2_dom_bb == NULL)
8098 l2_dom_bb = l1_bb;
8100 n1 = fd->loop.n1;
8101 n2 = fd->loop.n2;
8102 if (gimple_omp_for_combined_into_p (fd->for_stmt))
8104 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt),
8105 OMP_CLAUSE__LOOPTEMP_);
8106 gcc_assert (innerc);
8107 n1 = OMP_CLAUSE_DECL (innerc);
8108 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc),
8109 OMP_CLAUSE__LOOPTEMP_);
8110 gcc_assert (innerc);
8111 n2 = OMP_CLAUSE_DECL (innerc);
8112 expand_omp_build_assign (&gsi, fd->loop.v,
8113 fold_convert (type, n1));
8114 if (fd->collapse > 1)
8116 gsi_prev (&gsi);
8117 expand_omp_for_init_vars (fd, &gsi, counts, NULL, n1);
8118 gsi_next (&gsi);
8121 else
8123 expand_omp_build_assign (&gsi, fd->loop.v,
8124 fold_convert (type, fd->loop.n1));
8125 if (fd->collapse > 1)
8126 for (i = 0; i < fd->collapse; i++)
8128 tree itype = TREE_TYPE (fd->loops[i].v);
8129 if (POINTER_TYPE_P (itype))
8130 itype = signed_type_for (itype);
8131 t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1);
8132 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
8136 /* Remove the GIMPLE_OMP_FOR statement. */
8137 gsi_remove (&gsi, true);
8139 if (!broken_loop)
8141 /* Code to control the increment goes in the CONT_BB. */
8142 gsi = gsi_last_bb (cont_bb);
8143 stmt = gsi_stmt (gsi);
8144 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE);
8146 if (POINTER_TYPE_P (type))
8147 t = fold_build_pointer_plus (fd->loop.v, fd->loop.step);
8148 else
8149 t = fold_build2 (PLUS_EXPR, type, fd->loop.v, fd->loop.step);
8150 expand_omp_build_assign (&gsi, fd->loop.v, t);
8152 if (fd->collapse > 1)
8154 i = fd->collapse - 1;
8155 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v)))
8157 t = fold_convert (sizetype, fd->loops[i].step);
8158 t = fold_build_pointer_plus (fd->loops[i].v, t);
8160 else
8162 t = fold_convert (TREE_TYPE (fd->loops[i].v),
8163 fd->loops[i].step);
8164 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v),
8165 fd->loops[i].v, t);
8167 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
8169 for (i = fd->collapse - 1; i > 0; i--)
8171 tree itype = TREE_TYPE (fd->loops[i].v);
8172 tree itype2 = TREE_TYPE (fd->loops[i - 1].v);
8173 if (POINTER_TYPE_P (itype2))
8174 itype2 = signed_type_for (itype2);
8175 t = build3 (COND_EXPR, itype2,
8176 build2 (fd->loops[i].cond_code, boolean_type_node,
8177 fd->loops[i].v,
8178 fold_convert (itype, fd->loops[i].n2)),
8179 build_int_cst (itype2, 0),
8180 fold_convert (itype2, fd->loops[i - 1].step));
8181 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i - 1].v)))
8182 t = fold_build_pointer_plus (fd->loops[i - 1].v, t);
8183 else
8184 t = fold_build2 (PLUS_EXPR, itype2, fd->loops[i - 1].v, t);
8185 expand_omp_build_assign (&gsi, fd->loops[i - 1].v, t);
8187 t = build3 (COND_EXPR, itype,
8188 build2 (fd->loops[i].cond_code, boolean_type_node,
8189 fd->loops[i].v,
8190 fold_convert (itype, fd->loops[i].n2)),
8191 fd->loops[i].v,
8192 fold_convert (itype, fd->loops[i].n1));
8193 expand_omp_build_assign (&gsi, fd->loops[i].v, t);
8197 /* Remove GIMPLE_OMP_CONTINUE. */
8198 gsi_remove (&gsi, true);
8201 /* Emit the condition in L1_BB. */
8202 gsi = gsi_start_bb (l1_bb);
8204 t = fold_convert (type, n2);
8205 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
8206 false, GSI_CONTINUE_LINKING);
8207 t = build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t);
8208 stmt = gimple_build_cond_empty (t);
8209 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
8210 if (walk_tree (gimple_cond_lhs_ptr (stmt), expand_omp_regimplify_p,
8211 NULL, NULL)
8212 || walk_tree (gimple_cond_rhs_ptr (stmt), expand_omp_regimplify_p,
8213 NULL, NULL))
8215 gsi = gsi_for_stmt (stmt);
8216 gimple_regimplify_operands (stmt, &gsi);
8219 /* Remove GIMPLE_OMP_RETURN. */
8220 gsi = gsi_last_bb (exit_bb);
8221 gsi_remove (&gsi, true);
8223 /* Connect the new blocks. */
8224 remove_edge (FALLTHRU_EDGE (entry_bb));
8226 if (!broken_loop)
8228 remove_edge (BRANCH_EDGE (entry_bb));
8229 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU);
8231 e = BRANCH_EDGE (l1_bb);
8232 ne = FALLTHRU_EDGE (l1_bb);
8233 e->flags = EDGE_TRUE_VALUE;
8235 else
8237 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8239 ne = single_succ_edge (l1_bb);
8240 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE);
8243 ne->flags = EDGE_FALSE_VALUE;
8244 e->probability = REG_BR_PROB_BASE * 7 / 8;
8245 ne->probability = REG_BR_PROB_BASE / 8;
8247 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
8248 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
8249 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
8251 if (!broken_loop)
8253 struct loop *loop = alloc_loop ();
8254 loop->header = l1_bb;
8255 loop->latch = cont_bb;
8256 add_loop (loop, l1_bb->loop_father);
8257 if (safelen == NULL_TREE)
8258 loop->safelen = INT_MAX;
8259 else
8261 safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen);
8262 if (TREE_CODE (safelen) != INTEGER_CST)
8263 loop->safelen = 0;
8264 else if (!tree_fits_uhwi_p (safelen)
8265 || tree_to_uhwi (safelen) > INT_MAX)
8266 loop->safelen = INT_MAX;
8267 else
8268 loop->safelen = tree_to_uhwi (safelen);
8269 if (loop->safelen == 1)
8270 loop->safelen = 0;
8272 if (simduid)
8274 loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid);
8275 cfun->has_simduid_loops = true;
8277 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
8278 the loop. */
8279 if ((flag_tree_loop_vectorize
8280 || (!global_options_set.x_flag_tree_loop_vectorize
8281 && !global_options_set.x_flag_tree_vectorize))
8282 && flag_tree_loop_optimize
8283 && loop->safelen > 1)
8285 loop->force_vectorize = true;
8286 cfun->has_force_vectorize_loops = true;
8292 /* Expand the OpenMP loop defined by REGION. */
8294 static void
8295 expand_omp_for (struct omp_region *region, gimple inner_stmt)
8297 struct omp_for_data fd;
8298 struct omp_for_data_loop *loops;
8300 loops
8301 = (struct omp_for_data_loop *)
8302 alloca (gimple_omp_for_collapse (last_stmt (region->entry))
8303 * sizeof (struct omp_for_data_loop));
8304 extract_omp_for_data (last_stmt (region->entry), &fd, loops);
8305 region->sched_kind = fd.sched_kind;
8307 gcc_assert (EDGE_COUNT (region->entry->succs) == 2);
8308 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
8309 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL;
8310 if (region->cont)
8312 gcc_assert (EDGE_COUNT (region->cont->succs) == 2);
8313 BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
8314 FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL;
8316 else
8317 /* If there isn't a continue then this is a degerate case where
8318 the introduction of abnormal edges during lowering will prevent
8319 original loops from being detected. Fix that up. */
8320 loops_state_set (LOOPS_NEED_FIXUP);
8322 if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD)
8323 expand_omp_simd (region, &fd);
8324 else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_CILKFOR)
8325 expand_cilk_for (region, &fd);
8326 else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
8327 && !fd.have_ordered)
8329 if (fd.chunk_size == NULL)
8330 expand_omp_for_static_nochunk (region, &fd, inner_stmt);
8331 else
8332 expand_omp_for_static_chunk (region, &fd, inner_stmt);
8334 else
8336 int fn_index, start_ix, next_ix;
8338 gcc_assert (gimple_omp_for_kind (fd.for_stmt)
8339 == GF_OMP_FOR_KIND_FOR);
8340 if (fd.chunk_size == NULL
8341 && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
8342 fd.chunk_size = integer_zero_node;
8343 gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO);
8344 fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
8345 ? 3 : fd.sched_kind;
8346 fn_index += fd.have_ordered * 4;
8347 start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index;
8348 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index;
8349 if (fd.iter_type == long_long_unsigned_type_node)
8351 start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
8352 - (int)BUILT_IN_GOMP_LOOP_STATIC_START);
8353 next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
8354 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT);
8356 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix,
8357 (enum built_in_function) next_ix, inner_stmt);
8360 if (gimple_in_ssa_p (cfun))
8361 update_ssa (TODO_update_ssa_only_virtuals);
8365 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
8367 v = GOMP_sections_start (n);
8369 switch (v)
8371 case 0:
8372 goto L2;
8373 case 1:
8374 section 1;
8375 goto L1;
8376 case 2:
8378 case n:
8380 default:
8381 abort ();
8384 v = GOMP_sections_next ();
8385 goto L0;
8387 reduction;
8389 If this is a combined parallel sections, replace the call to
8390 GOMP_sections_start with call to GOMP_sections_next. */
8392 static void
8393 expand_omp_sections (struct omp_region *region)
8395 tree t, u, vin = NULL, vmain, vnext, l2;
8396 unsigned len;
8397 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
8398 gimple_stmt_iterator si, switch_si;
8399 gimple sections_stmt, stmt, cont;
8400 edge_iterator ei;
8401 edge e;
8402 struct omp_region *inner;
8403 unsigned i, casei;
8404 bool exit_reachable = region->cont != NULL;
8406 gcc_assert (region->exit != NULL);
8407 entry_bb = region->entry;
8408 l0_bb = single_succ (entry_bb);
8409 l1_bb = region->cont;
8410 l2_bb = region->exit;
8411 if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb)
8412 l2 = gimple_block_label (l2_bb);
8413 else
8415 /* This can happen if there are reductions. */
8416 len = EDGE_COUNT (l0_bb->succs);
8417 gcc_assert (len > 0);
8418 e = EDGE_SUCC (l0_bb, len - 1);
8419 si = gsi_last_bb (e->dest);
8420 l2 = NULL_TREE;
8421 if (gsi_end_p (si)
8422 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
8423 l2 = gimple_block_label (e->dest);
8424 else
8425 FOR_EACH_EDGE (e, ei, l0_bb->succs)
8427 si = gsi_last_bb (e->dest);
8428 if (gsi_end_p (si)
8429 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION)
8431 l2 = gimple_block_label (e->dest);
8432 break;
8436 if (exit_reachable)
8437 default_bb = create_empty_bb (l1_bb->prev_bb);
8438 else
8439 default_bb = create_empty_bb (l0_bb);
8441 /* We will build a switch() with enough cases for all the
8442 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
8443 and a default case to abort if something goes wrong. */
8444 len = EDGE_COUNT (l0_bb->succs);
8446 /* Use vec::quick_push on label_vec throughout, since we know the size
8447 in advance. */
8448 auto_vec<tree> label_vec (len);
8450 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
8451 GIMPLE_OMP_SECTIONS statement. */
8452 si = gsi_last_bb (entry_bb);
8453 sections_stmt = gsi_stmt (si);
8454 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS);
8455 vin = gimple_omp_sections_control (sections_stmt);
8456 if (!is_combined_parallel (region))
8458 /* If we are not inside a combined parallel+sections region,
8459 call GOMP_sections_start. */
8460 t = build_int_cst (unsigned_type_node, len - 1);
8461 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START);
8462 stmt = gimple_build_call (u, 1, t);
8464 else
8466 /* Otherwise, call GOMP_sections_next. */
8467 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
8468 stmt = gimple_build_call (u, 0);
8470 gimple_call_set_lhs (stmt, vin);
8471 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8472 gsi_remove (&si, true);
8474 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8475 L0_BB. */
8476 switch_si = gsi_last_bb (l0_bb);
8477 gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH);
8478 if (exit_reachable)
8480 cont = last_stmt (l1_bb);
8481 gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE);
8482 vmain = gimple_omp_continue_control_use (cont);
8483 vnext = gimple_omp_continue_control_def (cont);
8485 else
8487 vmain = vin;
8488 vnext = NULL_TREE;
8491 t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2);
8492 label_vec.quick_push (t);
8493 i = 1;
8495 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8496 for (inner = region->inner, casei = 1;
8497 inner;
8498 inner = inner->next, i++, casei++)
8500 basic_block s_entry_bb, s_exit_bb;
8502 /* Skip optional reduction region. */
8503 if (inner->type == GIMPLE_OMP_ATOMIC_LOAD)
8505 --i;
8506 --casei;
8507 continue;
8510 s_entry_bb = inner->entry;
8511 s_exit_bb = inner->exit;
8513 t = gimple_block_label (s_entry_bb);
8514 u = build_int_cst (unsigned_type_node, casei);
8515 u = build_case_label (u, NULL, t);
8516 label_vec.quick_push (u);
8518 si = gsi_last_bb (s_entry_bb);
8519 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION);
8520 gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si)));
8521 gsi_remove (&si, true);
8522 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
8524 if (s_exit_bb == NULL)
8525 continue;
8527 si = gsi_last_bb (s_exit_bb);
8528 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
8529 gsi_remove (&si, true);
8531 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
8534 /* Error handling code goes in DEFAULT_BB. */
8535 t = gimple_block_label (default_bb);
8536 u = build_case_label (NULL, NULL, t);
8537 make_edge (l0_bb, default_bb, 0);
8538 add_bb_to_loop (default_bb, current_loops->tree_root);
8540 stmt = gimple_build_switch (vmain, u, label_vec);
8541 gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT);
8542 gsi_remove (&switch_si, true);
8544 si = gsi_start_bb (default_bb);
8545 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0);
8546 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING);
8548 if (exit_reachable)
8550 tree bfn_decl;
8552 /* Code to get the next section goes in L1_BB. */
8553 si = gsi_last_bb (l1_bb);
8554 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE);
8556 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT);
8557 stmt = gimple_build_call (bfn_decl, 0);
8558 gimple_call_set_lhs (stmt, vnext);
8559 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8560 gsi_remove (&si, true);
8562 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
8565 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8566 si = gsi_last_bb (l2_bb);
8567 if (gimple_omp_return_nowait_p (gsi_stmt (si)))
8568 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT);
8569 else if (gimple_omp_return_lhs (gsi_stmt (si)))
8570 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL);
8571 else
8572 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END);
8573 stmt = gimple_build_call (t, 0);
8574 if (gimple_omp_return_lhs (gsi_stmt (si)))
8575 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si)));
8576 gsi_insert_after (&si, stmt, GSI_SAME_STMT);
8577 gsi_remove (&si, true);
8579 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
8583 /* Expand code for an OpenMP single directive. We've already expanded
8584 much of the code, here we simply place the GOMP_barrier call. */
8586 static void
8587 expand_omp_single (struct omp_region *region)
8589 basic_block entry_bb, exit_bb;
8590 gimple_stmt_iterator si;
8592 entry_bb = region->entry;
8593 exit_bb = region->exit;
8595 si = gsi_last_bb (entry_bb);
8596 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE);
8597 gsi_remove (&si, true);
8598 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8600 si = gsi_last_bb (exit_bb);
8601 if (!gimple_omp_return_nowait_p (gsi_stmt (si)))
8603 tree t = gimple_omp_return_lhs (gsi_stmt (si));
8604 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT);
8606 gsi_remove (&si, true);
8607 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
8611 /* Generic expansion for OpenMP synchronization directives: master,
8612 ordered and critical. All we need to do here is remove the entry
8613 and exit markers for REGION. */
8615 static void
8616 expand_omp_synch (struct omp_region *region)
8618 basic_block entry_bb, exit_bb;
8619 gimple_stmt_iterator si;
8621 entry_bb = region->entry;
8622 exit_bb = region->exit;
8624 si = gsi_last_bb (entry_bb);
8625 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
8626 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
8627 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
8628 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
8629 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
8630 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS);
8631 gsi_remove (&si, true);
8632 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
8634 if (exit_bb)
8636 si = gsi_last_bb (exit_bb);
8637 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
8638 gsi_remove (&si, true);
8639 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
8643 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8644 operation as a normal volatile load. */
8646 static bool
8647 expand_omp_atomic_load (basic_block load_bb, tree addr,
8648 tree loaded_val, int index)
8650 enum built_in_function tmpbase;
8651 gimple_stmt_iterator gsi;
8652 basic_block store_bb;
8653 location_t loc;
8654 gimple stmt;
8655 tree decl, call, type, itype;
8657 gsi = gsi_last_bb (load_bb);
8658 stmt = gsi_stmt (gsi);
8659 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
8660 loc = gimple_location (stmt);
8662 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8663 is smaller than word size, then expand_atomic_load assumes that the load
8664 is atomic. We could avoid the builtin entirely in this case. */
8666 tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
8667 decl = builtin_decl_explicit (tmpbase);
8668 if (decl == NULL_TREE)
8669 return false;
8671 type = TREE_TYPE (loaded_val);
8672 itype = TREE_TYPE (TREE_TYPE (decl));
8674 call = build_call_expr_loc (loc, decl, 2, addr,
8675 build_int_cst (NULL,
8676 gimple_omp_atomic_seq_cst_p (stmt)
8677 ? MEMMODEL_SEQ_CST
8678 : MEMMODEL_RELAXED));
8679 if (!useless_type_conversion_p (type, itype))
8680 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
8681 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
8683 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
8684 gsi_remove (&gsi, true);
8686 store_bb = single_succ (load_bb);
8687 gsi = gsi_last_bb (store_bb);
8688 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
8689 gsi_remove (&gsi, true);
8691 if (gimple_in_ssa_p (cfun))
8692 update_ssa (TODO_update_ssa_no_phi);
8694 return true;
8697 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8698 operation as a normal volatile store. */
8700 static bool
8701 expand_omp_atomic_store (basic_block load_bb, tree addr,
8702 tree loaded_val, tree stored_val, int index)
8704 enum built_in_function tmpbase;
8705 gimple_stmt_iterator gsi;
8706 basic_block store_bb = single_succ (load_bb);
8707 location_t loc;
8708 gimple stmt;
8709 tree decl, call, type, itype;
8710 machine_mode imode;
8711 bool exchange;
8713 gsi = gsi_last_bb (load_bb);
8714 stmt = gsi_stmt (gsi);
8715 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD);
8717 /* If the load value is needed, then this isn't a store but an exchange. */
8718 exchange = gimple_omp_atomic_need_value_p (stmt);
8720 gsi = gsi_last_bb (store_bb);
8721 stmt = gsi_stmt (gsi);
8722 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE);
8723 loc = gimple_location (stmt);
8725 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8726 is smaller than word size, then expand_atomic_store assumes that the store
8727 is atomic. We could avoid the builtin entirely in this case. */
8729 tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N);
8730 tmpbase = (enum built_in_function) ((int) tmpbase + index + 1);
8731 decl = builtin_decl_explicit (tmpbase);
8732 if (decl == NULL_TREE)
8733 return false;
8735 type = TREE_TYPE (stored_val);
8737 /* Dig out the type of the function's second argument. */
8738 itype = TREE_TYPE (decl);
8739 itype = TYPE_ARG_TYPES (itype);
8740 itype = TREE_CHAIN (itype);
8741 itype = TREE_VALUE (itype);
8742 imode = TYPE_MODE (itype);
8744 if (exchange && !can_atomic_exchange_p (imode, true))
8745 return false;
8747 if (!useless_type_conversion_p (itype, type))
8748 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val);
8749 call = build_call_expr_loc (loc, decl, 3, addr, stored_val,
8750 build_int_cst (NULL,
8751 gimple_omp_atomic_seq_cst_p (stmt)
8752 ? MEMMODEL_SEQ_CST
8753 : MEMMODEL_RELAXED));
8754 if (exchange)
8756 if (!useless_type_conversion_p (type, itype))
8757 call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call);
8758 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call);
8761 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
8762 gsi_remove (&gsi, true);
8764 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8765 gsi = gsi_last_bb (load_bb);
8766 gsi_remove (&gsi, true);
8768 if (gimple_in_ssa_p (cfun))
8769 update_ssa (TODO_update_ssa_no_phi);
8771 return true;
8774 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8775 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8776 size of the data type, and thus usable to find the index of the builtin
8777 decl. Returns false if the expression is not of the proper form. */
8779 static bool
8780 expand_omp_atomic_fetch_op (basic_block load_bb,
8781 tree addr, tree loaded_val,
8782 tree stored_val, int index)
8784 enum built_in_function oldbase, newbase, tmpbase;
8785 tree decl, itype, call;
8786 tree lhs, rhs;
8787 basic_block store_bb = single_succ (load_bb);
8788 gimple_stmt_iterator gsi;
8789 gimple stmt;
8790 location_t loc;
8791 enum tree_code code;
8792 bool need_old, need_new;
8793 machine_mode imode;
8794 bool seq_cst;
8796 /* We expect to find the following sequences:
8798 load_bb:
8799 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8801 store_bb:
8802 val = tmp OP something; (or: something OP tmp)
8803 GIMPLE_OMP_STORE (val)
8805 ???FIXME: Allow a more flexible sequence.
8806 Perhaps use data flow to pick the statements.
8810 gsi = gsi_after_labels (store_bb);
8811 stmt = gsi_stmt (gsi);
8812 loc = gimple_location (stmt);
8813 if (!is_gimple_assign (stmt))
8814 return false;
8815 gsi_next (&gsi);
8816 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
8817 return false;
8818 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi));
8819 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb));
8820 seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb));
8821 gcc_checking_assert (!need_old || !need_new);
8823 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
8824 return false;
8826 /* Check for one of the supported fetch-op operations. */
8827 code = gimple_assign_rhs_code (stmt);
8828 switch (code)
8830 case PLUS_EXPR:
8831 case POINTER_PLUS_EXPR:
8832 oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N;
8833 newbase = BUILT_IN_ATOMIC_ADD_FETCH_N;
8834 break;
8835 case MINUS_EXPR:
8836 oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N;
8837 newbase = BUILT_IN_ATOMIC_SUB_FETCH_N;
8838 break;
8839 case BIT_AND_EXPR:
8840 oldbase = BUILT_IN_ATOMIC_FETCH_AND_N;
8841 newbase = BUILT_IN_ATOMIC_AND_FETCH_N;
8842 break;
8843 case BIT_IOR_EXPR:
8844 oldbase = BUILT_IN_ATOMIC_FETCH_OR_N;
8845 newbase = BUILT_IN_ATOMIC_OR_FETCH_N;
8846 break;
8847 case BIT_XOR_EXPR:
8848 oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N;
8849 newbase = BUILT_IN_ATOMIC_XOR_FETCH_N;
8850 break;
8851 default:
8852 return false;
8855 /* Make sure the expression is of the proper form. */
8856 if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
8857 rhs = gimple_assign_rhs2 (stmt);
8858 else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
8859 && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
8860 rhs = gimple_assign_rhs1 (stmt);
8861 else
8862 return false;
8864 tmpbase = ((enum built_in_function)
8865 ((need_new ? newbase : oldbase) + index + 1));
8866 decl = builtin_decl_explicit (tmpbase);
8867 if (decl == NULL_TREE)
8868 return false;
8869 itype = TREE_TYPE (TREE_TYPE (decl));
8870 imode = TYPE_MODE (itype);
8872 /* We could test all of the various optabs involved, but the fact of the
8873 matter is that (with the exception of i486 vs i586 and xadd) all targets
8874 that support any atomic operaton optab also implements compare-and-swap.
8875 Let optabs.c take care of expanding any compare-and-swap loop. */
8876 if (!can_compare_and_swap_p (imode, true))
8877 return false;
8879 gsi = gsi_last_bb (load_bb);
8880 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
8882 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8883 It only requires that the operation happen atomically. Thus we can
8884 use the RELAXED memory model. */
8885 call = build_call_expr_loc (loc, decl, 3, addr,
8886 fold_convert_loc (loc, itype, rhs),
8887 build_int_cst (NULL,
8888 seq_cst ? MEMMODEL_SEQ_CST
8889 : MEMMODEL_RELAXED));
8891 if (need_old || need_new)
8893 lhs = need_old ? loaded_val : stored_val;
8894 call = fold_convert_loc (loc, TREE_TYPE (lhs), call);
8895 call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call);
8897 else
8898 call = fold_convert_loc (loc, void_type_node, call);
8899 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
8900 gsi_remove (&gsi, true);
8902 gsi = gsi_last_bb (store_bb);
8903 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
8904 gsi_remove (&gsi, true);
8905 gsi = gsi_last_bb (store_bb);
8906 gsi_remove (&gsi, true);
8908 if (gimple_in_ssa_p (cfun))
8909 update_ssa (TODO_update_ssa_no_phi);
8911 return true;
8914 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8916 oldval = *addr;
8917 repeat:
8918 newval = rhs; // with oldval replacing *addr in rhs
8919 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8920 if (oldval != newval)
8921 goto repeat;
8923 INDEX is log2 of the size of the data type, and thus usable to find the
8924 index of the builtin decl. */
8926 static bool
8927 expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
8928 tree addr, tree loaded_val, tree stored_val,
8929 int index)
8931 tree loadedi, storedi, initial, new_storedi, old_vali;
8932 tree type, itype, cmpxchg, iaddr;
8933 gimple_stmt_iterator si;
8934 basic_block loop_header = single_succ (load_bb);
8935 gimple phi, stmt;
8936 edge e;
8937 enum built_in_function fncode;
8939 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8940 order to use the RELAXED memory model effectively. */
8941 fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8942 + index + 1);
8943 cmpxchg = builtin_decl_explicit (fncode);
8944 if (cmpxchg == NULL_TREE)
8945 return false;
8946 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
8947 itype = TREE_TYPE (TREE_TYPE (cmpxchg));
8949 if (!can_compare_and_swap_p (TYPE_MODE (itype), true))
8950 return false;
8952 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8953 si = gsi_last_bb (load_bb);
8954 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
8956 /* For floating-point values, we'll need to view-convert them to integers
8957 so that we can perform the atomic compare and swap. Simplify the
8958 following code by always setting up the "i"ntegral variables. */
8959 if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
8961 tree iaddr_val;
8963 iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode,
8964 true), NULL);
8965 iaddr_val
8966 = force_gimple_operand_gsi (&si,
8967 fold_convert (TREE_TYPE (iaddr), addr),
8968 false, NULL_TREE, true, GSI_SAME_STMT);
8969 stmt = gimple_build_assign (iaddr, iaddr_val);
8970 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
8971 loadedi = create_tmp_var (itype, NULL);
8972 if (gimple_in_ssa_p (cfun))
8973 loadedi = make_ssa_name (loadedi, NULL);
8975 else
8977 iaddr = addr;
8978 loadedi = loaded_val;
8981 fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1);
8982 tree loaddecl = builtin_decl_explicit (fncode);
8983 if (loaddecl)
8984 initial
8985 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr)),
8986 build_call_expr (loaddecl, 2, iaddr,
8987 build_int_cst (NULL_TREE,
8988 MEMMODEL_RELAXED)));
8989 else
8990 initial = build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)), iaddr,
8991 build_int_cst (TREE_TYPE (iaddr), 0));
8993 initial
8994 = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true,
8995 GSI_SAME_STMT);
8997 /* Move the value to the LOADEDI temporary. */
8998 if (gimple_in_ssa_p (cfun))
9000 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
9001 phi = create_phi_node (loadedi, loop_header);
9002 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
9003 initial);
9005 else
9006 gsi_insert_before (&si,
9007 gimple_build_assign (loadedi, initial),
9008 GSI_SAME_STMT);
9009 if (loadedi != loaded_val)
9011 gimple_stmt_iterator gsi2;
9012 tree x;
9014 x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
9015 gsi2 = gsi_start_bb (loop_header);
9016 if (gimple_in_ssa_p (cfun))
9018 gimple stmt;
9019 x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
9020 true, GSI_SAME_STMT);
9021 stmt = gimple_build_assign (loaded_val, x);
9022 gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
9024 else
9026 x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
9027 force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
9028 true, GSI_SAME_STMT);
9031 gsi_remove (&si, true);
9033 si = gsi_last_bb (store_bb);
9034 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
9036 if (iaddr == addr)
9037 storedi = stored_val;
9038 else
9039 storedi =
9040 force_gimple_operand_gsi (&si,
9041 build1 (VIEW_CONVERT_EXPR, itype,
9042 stored_val), true, NULL_TREE, true,
9043 GSI_SAME_STMT);
9045 /* Build the compare&swap statement. */
9046 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
9047 new_storedi = force_gimple_operand_gsi (&si,
9048 fold_convert (TREE_TYPE (loadedi),
9049 new_storedi),
9050 true, NULL_TREE,
9051 true, GSI_SAME_STMT);
9053 if (gimple_in_ssa_p (cfun))
9054 old_vali = loadedi;
9055 else
9057 old_vali = create_tmp_var (TREE_TYPE (loadedi), NULL);
9058 stmt = gimple_build_assign (old_vali, loadedi);
9059 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9061 stmt = gimple_build_assign (loadedi, new_storedi);
9062 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9065 /* Note that we always perform the comparison as an integer, even for
9066 floating point. This allows the atomic operation to properly
9067 succeed even with NaNs and -0.0. */
9068 stmt = gimple_build_cond_empty
9069 (build2 (NE_EXPR, boolean_type_node,
9070 new_storedi, old_vali));
9071 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9073 /* Update cfg. */
9074 e = single_succ_edge (store_bb);
9075 e->flags &= ~EDGE_FALLTHRU;
9076 e->flags |= EDGE_FALSE_VALUE;
9078 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
9080 /* Copy the new value to loadedi (we already did that before the condition
9081 if we are not in SSA). */
9082 if (gimple_in_ssa_p (cfun))
9084 phi = gimple_seq_first_stmt (phi_nodes (loop_header));
9085 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
9088 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
9089 gsi_remove (&si, true);
9091 struct loop *loop = alloc_loop ();
9092 loop->header = loop_header;
9093 loop->latch = store_bb;
9094 add_loop (loop, loop_header->loop_father);
9096 if (gimple_in_ssa_p (cfun))
9097 update_ssa (TODO_update_ssa_no_phi);
9099 return true;
9102 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
9104 GOMP_atomic_start ();
9105 *addr = rhs;
9106 GOMP_atomic_end ();
9108 The result is not globally atomic, but works so long as all parallel
9109 references are within #pragma omp atomic directives. According to
9110 responses received from omp@openmp.org, appears to be within spec.
9111 Which makes sense, since that's how several other compilers handle
9112 this situation as well.
9113 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
9114 expanding. STORED_VAL is the operand of the matching
9115 GIMPLE_OMP_ATOMIC_STORE.
9117 We replace
9118 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
9119 loaded_val = *addr;
9121 and replace
9122 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
9123 *addr = stored_val;
9126 static bool
9127 expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
9128 tree addr, tree loaded_val, tree stored_val)
9130 gimple_stmt_iterator si;
9131 gimple stmt;
9132 tree t;
9134 si = gsi_last_bb (load_bb);
9135 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
9137 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
9138 t = build_call_expr (t, 0);
9139 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
9141 stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr));
9142 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9143 gsi_remove (&si, true);
9145 si = gsi_last_bb (store_bb);
9146 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
9148 stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)),
9149 stored_val);
9150 gsi_insert_before (&si, stmt, GSI_SAME_STMT);
9152 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
9153 t = build_call_expr (t, 0);
9154 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
9155 gsi_remove (&si, true);
9157 if (gimple_in_ssa_p (cfun))
9158 update_ssa (TODO_update_ssa_no_phi);
9159 return true;
9162 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
9163 using expand_omp_atomic_fetch_op. If it failed, we try to
9164 call expand_omp_atomic_pipeline, and if it fails too, the
9165 ultimate fallback is wrapping the operation in a mutex
9166 (expand_omp_atomic_mutex). REGION is the atomic region built
9167 by build_omp_regions_1(). */
9169 static void
9170 expand_omp_atomic (struct omp_region *region)
9172 basic_block load_bb = region->entry, store_bb = region->exit;
9173 gimple load = last_stmt (load_bb), store = last_stmt (store_bb);
9174 tree loaded_val = gimple_omp_atomic_load_lhs (load);
9175 tree addr = gimple_omp_atomic_load_rhs (load);
9176 tree stored_val = gimple_omp_atomic_store_val (store);
9177 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
9178 HOST_WIDE_INT index;
9180 /* Make sure the type is one of the supported sizes. */
9181 index = tree_to_uhwi (TYPE_SIZE_UNIT (type));
9182 index = exact_log2 (index);
9183 if (index >= 0 && index <= 4)
9185 unsigned int align = TYPE_ALIGN_UNIT (type);
9187 /* __sync builtins require strict data alignment. */
9188 if (exact_log2 (align) >= index)
9190 /* Atomic load. */
9191 if (loaded_val == stored_val
9192 && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
9193 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
9194 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
9195 && expand_omp_atomic_load (load_bb, addr, loaded_val, index))
9196 return;
9198 /* Atomic store. */
9199 if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT
9200 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT)
9201 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD
9202 && store_bb == single_succ (load_bb)
9203 && first_stmt (store_bb) == store
9204 && expand_omp_atomic_store (load_bb, addr, loaded_val,
9205 stored_val, index))
9206 return;
9208 /* When possible, use specialized atomic update functions. */
9209 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
9210 && store_bb == single_succ (load_bb)
9211 && expand_omp_atomic_fetch_op (load_bb, addr,
9212 loaded_val, stored_val, index))
9213 return;
9215 /* If we don't have specialized __sync builtins, try and implement
9216 as a compare and swap loop. */
9217 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
9218 loaded_val, stored_val, index))
9219 return;
9223 /* The ultimate fallback is wrapping the operation in a mutex. */
9224 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
9228 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
9230 static void
9231 expand_omp_target (struct omp_region *region)
9233 basic_block entry_bb, exit_bb, new_bb;
9234 struct function *child_cfun = NULL;
9235 tree child_fn = NULL_TREE, block, t;
9236 gimple_stmt_iterator gsi;
9237 gimple entry_stmt, stmt;
9238 edge e;
9240 entry_stmt = last_stmt (region->entry);
9241 new_bb = region->entry;
9242 int kind = gimple_omp_target_kind (entry_stmt);
9243 if (kind == GF_OMP_TARGET_KIND_REGION)
9245 child_fn = gimple_omp_target_child_fn (entry_stmt);
9246 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
9249 entry_bb = region->entry;
9250 exit_bb = region->exit;
9252 if (kind == GF_OMP_TARGET_KIND_REGION)
9254 unsigned srcidx, dstidx, num;
9256 /* If the target region needs data sent from the parent
9257 function, then the very first statement (except possible
9258 tree profile counter updates) of the parallel body
9259 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
9260 &.OMP_DATA_O is passed as an argument to the child function,
9261 we need to replace it with the argument as seen by the child
9262 function.
9264 In most cases, this will end up being the identity assignment
9265 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
9266 a function call that has been inlined, the original PARM_DECL
9267 .OMP_DATA_I may have been converted into a different local
9268 variable. In which case, we need to keep the assignment. */
9269 if (gimple_omp_target_data_arg (entry_stmt))
9271 basic_block entry_succ_bb = single_succ (entry_bb);
9272 gimple_stmt_iterator gsi;
9273 tree arg;
9274 gimple tgtcopy_stmt = NULL;
9275 tree sender
9276 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt), 0);
9278 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi))
9280 gcc_assert (!gsi_end_p (gsi));
9281 stmt = gsi_stmt (gsi);
9282 if (gimple_code (stmt) != GIMPLE_ASSIGN)
9283 continue;
9285 if (gimple_num_ops (stmt) == 2)
9287 tree arg = gimple_assign_rhs1 (stmt);
9289 /* We're ignoring the subcode because we're
9290 effectively doing a STRIP_NOPS. */
9292 if (TREE_CODE (arg) == ADDR_EXPR
9293 && TREE_OPERAND (arg, 0) == sender)
9295 tgtcopy_stmt = stmt;
9296 break;
9301 gcc_assert (tgtcopy_stmt != NULL);
9302 arg = DECL_ARGUMENTS (child_fn);
9304 gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg);
9305 gsi_remove (&gsi, true);
9308 /* Declare local variables needed in CHILD_CFUN. */
9309 block = DECL_INITIAL (child_fn);
9310 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls);
9311 /* The gimplifier could record temporaries in target block
9312 rather than in containing function's local_decls chain,
9313 which would mean cgraph missed finalizing them. Do it now. */
9314 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t))
9315 if (TREE_CODE (t) == VAR_DECL
9316 && TREE_STATIC (t)
9317 && !DECL_EXTERNAL (t))
9318 varpool_node::finalize_decl (t);
9319 DECL_SAVED_TREE (child_fn) = NULL;
9320 /* We'll create a CFG for child_fn, so no gimple body is needed. */
9321 gimple_set_body (child_fn, NULL);
9322 TREE_USED (block) = 1;
9324 /* Reset DECL_CONTEXT on function arguments. */
9325 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
9326 DECL_CONTEXT (t) = child_fn;
9328 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
9329 so that it can be moved to the child function. */
9330 gsi = gsi_last_bb (entry_bb);
9331 stmt = gsi_stmt (gsi);
9332 gcc_assert (stmt && gimple_code (stmt) == GIMPLE_OMP_TARGET
9333 && gimple_omp_target_kind (stmt)
9334 == GF_OMP_TARGET_KIND_REGION);
9335 gsi_remove (&gsi, true);
9336 e = split_block (entry_bb, stmt);
9337 entry_bb = e->dest;
9338 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
9340 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
9341 if (exit_bb)
9343 gsi = gsi_last_bb (exit_bb);
9344 gcc_assert (!gsi_end_p (gsi)
9345 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN);
9346 stmt = gimple_build_return (NULL);
9347 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
9348 gsi_remove (&gsi, true);
9351 /* Move the target region into CHILD_CFUN. */
9353 block = gimple_block (entry_stmt);
9355 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block);
9356 if (exit_bb)
9357 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
9358 /* When the OMP expansion process cannot guarantee an up-to-date
9359 loop tree arrange for the child function to fixup loops. */
9360 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
9361 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP;
9363 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
9364 num = vec_safe_length (child_cfun->local_decls);
9365 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++)
9367 t = (*child_cfun->local_decls)[srcidx];
9368 if (DECL_CONTEXT (t) == cfun->decl)
9369 continue;
9370 if (srcidx != dstidx)
9371 (*child_cfun->local_decls)[dstidx] = t;
9372 dstidx++;
9374 if (dstidx != num)
9375 vec_safe_truncate (child_cfun->local_decls, dstidx);
9377 /* Inform the callgraph about the new function. */
9378 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties;
9379 cgraph_node::add_new_function (child_fn, true);
9381 /* Add the new function to the offload table. */
9382 vec_safe_push (offload_funcs, child_fn);
9384 /* Fix the callgraph edges for child_cfun. Those for cfun will be
9385 fixed in a following pass. */
9386 push_cfun (child_cfun);
9387 cgraph_edge::rebuild_edges ();
9389 /* Some EH regions might become dead, see PR34608. If
9390 pass_cleanup_cfg isn't the first pass to happen with the
9391 new child, these dead EH edges might cause problems.
9392 Clean them up now. */
9393 if (flag_exceptions)
9395 basic_block bb;
9396 bool changed = false;
9398 FOR_EACH_BB_FN (bb, cfun)
9399 changed |= gimple_purge_dead_eh_edges (bb);
9400 if (changed)
9401 cleanup_tree_cfg ();
9403 pop_cfun ();
9406 /* Emit a library call to launch the target region, or do data
9407 transfers. */
9408 tree t1, t2, t3, t4, device, cond, c, clauses;
9409 enum built_in_function start_ix;
9410 location_t clause_loc;
9412 clauses = gimple_omp_target_clauses (entry_stmt);
9414 switch (kind)
9416 case GF_OMP_TARGET_KIND_REGION:
9417 start_ix = BUILT_IN_GOMP_TARGET;
9418 break;
9419 case GF_OMP_TARGET_KIND_DATA:
9420 start_ix = BUILT_IN_GOMP_TARGET_DATA;
9421 break;
9422 case GF_OMP_TARGET_KIND_UPDATE:
9423 start_ix = BUILT_IN_GOMP_TARGET_UPDATE;
9424 break;
9425 case GF_OMP_TARGET_KIND_OACC_DATA:
9426 start_ix = BUILT_IN_GOACC_DATA_START;
9427 break;
9428 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
9429 start_ix = BUILT_IN_GOACC_ENTER_EXIT_DATA;
9430 break;
9431 case GF_OMP_TARGET_KIND_OACC_UPDATE:
9432 start_ix = BUILT_IN_GOACC_UPDATE;
9433 break;
9434 default:
9435 gcc_unreachable ();
9438 /* By default, the value of DEVICE is -1 (let runtime library choose)
9439 and there is no conditional. */
9440 cond = NULL_TREE;
9441 device = build_int_cst (integer_type_node, -1);
9443 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
9444 if (c)
9445 cond = OMP_CLAUSE_IF_EXPR (c);
9447 c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE);
9448 if (c)
9450 gcc_assert (kind != GF_OMP_TARGET_KIND_OACC_DATA
9451 && kind != GF_OMP_TARGET_KIND_OACC_UPDATE);
9453 device = OMP_CLAUSE_DEVICE_ID (c);
9454 clause_loc = OMP_CLAUSE_LOCATION (c);
9456 else
9457 clause_loc = gimple_location (entry_stmt);
9459 /* Ensure 'device' is of the correct type. */
9460 device = fold_convert_loc (clause_loc, integer_type_node, device);
9462 /* If we found the clause 'if (cond)', build
9463 (cond ? device : -2). */
9464 if (cond)
9466 cond = gimple_boolify (cond);
9468 basic_block cond_bb, then_bb, else_bb;
9469 edge e;
9470 tree tmp_var;
9472 tmp_var = create_tmp_var (TREE_TYPE (device), NULL);
9473 if (kind != GF_OMP_TARGET_KIND_REGION)
9475 gsi = gsi_last_bb (new_bb);
9476 gsi_prev (&gsi);
9477 e = split_block (new_bb, gsi_stmt (gsi));
9479 else
9480 e = split_block (new_bb, NULL);
9481 cond_bb = e->src;
9482 new_bb = e->dest;
9483 remove_edge (e);
9485 then_bb = create_empty_bb (cond_bb);
9486 else_bb = create_empty_bb (then_bb);
9487 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
9488 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
9490 stmt = gimple_build_cond_empty (cond);
9491 gsi = gsi_last_bb (cond_bb);
9492 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
9494 gsi = gsi_start_bb (then_bb);
9495 stmt = gimple_build_assign (tmp_var, device);
9496 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
9498 gsi = gsi_start_bb (else_bb);
9499 stmt = gimple_build_assign (tmp_var,
9500 build_int_cst (integer_type_node, -2));
9501 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
9503 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
9504 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
9505 add_bb_to_loop (then_bb, cond_bb->loop_father);
9506 add_bb_to_loop (else_bb, cond_bb->loop_father);
9507 make_edge (then_bb, new_bb, EDGE_FALLTHRU);
9508 make_edge (else_bb, new_bb, EDGE_FALLTHRU);
9510 device = tmp_var;
9513 gsi = gsi_last_bb (new_bb);
9514 t = gimple_omp_target_data_arg (entry_stmt);
9515 if (t == NULL)
9517 t1 = size_zero_node;
9518 t2 = build_zero_cst (ptr_type_node);
9519 t3 = t2;
9520 t4 = t2;
9522 else
9524 t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1))));
9525 t1 = size_binop (PLUS_EXPR, t1, size_int (1));
9526 t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0));
9527 t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1));
9528 t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2));
9531 gimple g;
9532 tree openmp_target = get_offload_symbol_decl ();
9533 vec<tree> *args;
9534 unsigned int argcnt = 6;
9536 if (kind == GF_OMP_TARGET_KIND_REGION)
9537 argcnt++;
9538 else if (kind == GF_OMP_TARGET_KIND_OACC_DATA
9539 || kind == GF_OMP_TARGET_KIND_OACC_UPDATE)
9540 argcnt += 2;
9542 c = find_omp_clause (clauses, OMP_CLAUSE_WAIT);
9543 if (c)
9545 for (t = c; t; t = OMP_CLAUSE_CHAIN (t))
9547 if (OMP_CLAUSE_CODE (t) == OMP_CLAUSE_WAIT)
9548 argcnt++;
9552 vec_alloc (args, argcnt);
9553 args->quick_push (device);
9555 if (kind == GF_OMP_TARGET_KIND_REGION)
9556 args->quick_push (build_fold_addr_expr (child_fn));
9558 args->quick_push (build_fold_addr_expr (openmp_target));
9559 args->quick_push (t1);
9560 args->quick_push (t2);
9561 args->quick_push (t3);
9562 args->quick_push (t4);
9564 if (kind == GF_OMP_TARGET_KIND_OACC_DATA
9565 || kind == GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
9566 || kind == GF_OMP_TARGET_KIND_OACC_UPDATE)
9568 int idx;
9570 c = find_omp_clause (clauses, OMP_CLAUSE_ASYNC);
9571 if (c)
9572 t1 = fold_convert_loc (OMP_CLAUSE_LOCATION (c), integer_type_node,
9573 OMP_CLAUSE_ASYNC_EXPR (c));
9574 else /* TODO: XXX FIX -2. */
9575 t1 = fold_convert_loc (gimple_location (entry_stmt),
9576 integer_type_node, build_int_cst (integer_type_node, -2));
9578 args->safe_push (t1);
9579 idx = args->length ();
9580 args->safe_push (fold_convert_loc (gimple_location (entry_stmt),
9581 integer_type_node, integer_minus_one_node));
9583 c = find_omp_clause (clauses, OMP_CLAUSE_WAIT);
9584 if (c)
9586 int n = 0;
9588 for (t = c; t; t = OMP_CLAUSE_CHAIN (t))
9590 if (OMP_CLAUSE_CODE (t) == OMP_CLAUSE_WAIT)
9592 args->safe_push (fold_convert (integer_type_node,
9593 OMP_CLAUSE_WAIT_EXPR (t)));
9594 n++;
9598 args->ordered_remove (idx);
9599 args->quick_insert (idx,
9600 fold_convert_loc (gimple_location (entry_stmt),
9601 integer_type_node,
9602 build_int_cst (integer_type_node, n)));
9606 g = gimple_build_call_vec (builtin_decl_explicit (start_ix), *args);
9607 args->release ();
9608 gimple_set_location (g, gimple_location (entry_stmt));
9609 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
9610 if (kind != GF_OMP_TARGET_KIND_REGION)
9612 g = gsi_stmt (gsi);
9613 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET);
9614 gsi_remove (&gsi, true);
9616 if ((kind == GF_OMP_TARGET_KIND_DATA
9617 || kind == GF_OMP_TARGET_KIND_OACC_DATA)
9618 && region->exit)
9620 gsi = gsi_last_bb (region->exit);
9621 g = gsi_stmt (gsi);
9622 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN);
9623 gsi_remove (&gsi, true);
9628 /* Expand the parallel region tree rooted at REGION. Expansion
9629 proceeds in depth-first order. Innermost regions are expanded
9630 first. This way, parallel regions that require a new function to
9631 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9632 internal dependencies in their body. */
9634 static void
9635 expand_omp (struct omp_region *region)
9637 while (region)
9639 location_t saved_location;
9640 gimple inner_stmt = NULL;
9642 /* First, determine whether this is a combined parallel+workshare
9643 region. */
9644 if (region->type == GIMPLE_OMP_PARALLEL)
9645 determine_parallel_type (region);
9647 if (region->type == GIMPLE_OMP_FOR
9648 && gimple_omp_for_combined_p (last_stmt (region->entry)))
9649 inner_stmt = last_stmt (region->inner->entry);
9651 if (region->inner)
9652 expand_omp (region->inner);
9654 saved_location = input_location;
9655 if (gimple_has_location (last_stmt (region->entry)))
9656 input_location = gimple_location (last_stmt (region->entry));
9658 switch (region->type)
9660 case GIMPLE_OACC_KERNELS:
9661 case GIMPLE_OACC_PARALLEL:
9662 expand_oacc_offload (region);
9663 break;
9665 case GIMPLE_OMP_PARALLEL:
9666 case GIMPLE_OMP_TASK:
9667 expand_omp_taskreg (region);
9668 break;
9670 case GIMPLE_OMP_FOR:
9671 expand_omp_for (region, inner_stmt);
9672 break;
9674 case GIMPLE_OMP_SECTIONS:
9675 expand_omp_sections (region);
9676 break;
9678 case GIMPLE_OMP_SECTION:
9679 /* Individual omp sections are handled together with their
9680 parent GIMPLE_OMP_SECTIONS region. */
9681 break;
9683 case GIMPLE_OMP_SINGLE:
9684 expand_omp_single (region);
9685 break;
9687 case GIMPLE_OMP_MASTER:
9688 case GIMPLE_OMP_TASKGROUP:
9689 case GIMPLE_OMP_ORDERED:
9690 case GIMPLE_OMP_CRITICAL:
9691 case GIMPLE_OMP_TEAMS:
9692 expand_omp_synch (region);
9693 break;
9695 case GIMPLE_OMP_ATOMIC_LOAD:
9696 expand_omp_atomic (region);
9697 break;
9699 case GIMPLE_OMP_TARGET:
9700 expand_omp_target (region);
9701 break;
9703 default:
9704 gcc_unreachable ();
9707 input_location = saved_location;
9708 region = region->next;
9713 /* Helper for build_omp_regions. Scan the dominator tree starting at
9714 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9715 true, the function ends once a single tree is built (otherwise, whole
9716 forest of OMP constructs may be built). */
9718 static void
9719 build_omp_regions_1 (basic_block bb, struct omp_region *parent,
9720 bool single_tree)
9722 gimple_stmt_iterator gsi;
9723 gimple stmt;
9724 basic_block son;
9726 gsi = gsi_last_bb (bb);
9727 if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi)))
9729 struct omp_region *region;
9730 enum gimple_code code;
9732 stmt = gsi_stmt (gsi);
9733 code = gimple_code (stmt);
9734 if (code == GIMPLE_OMP_RETURN)
9736 /* STMT is the return point out of region PARENT. Mark it
9737 as the exit point and make PARENT the immediately
9738 enclosing region. */
9739 gcc_assert (parent);
9740 region = parent;
9741 region->exit = bb;
9742 parent = parent->outer;
9744 else if (code == GIMPLE_OMP_ATOMIC_STORE)
9746 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9747 GIMPLE_OMP_RETURN, but matches with
9748 GIMPLE_OMP_ATOMIC_LOAD. */
9749 gcc_assert (parent);
9750 gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD);
9751 region = parent;
9752 region->exit = bb;
9753 parent = parent->outer;
9756 else if (code == GIMPLE_OMP_CONTINUE)
9758 gcc_assert (parent);
9759 parent->cont = bb;
9761 else if (code == GIMPLE_OMP_SECTIONS_SWITCH)
9763 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9764 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9767 else if (code == GIMPLE_OMP_TARGET
9768 && (gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_UPDATE
9769 || (gimple_omp_target_kind (stmt)
9770 == GF_OMP_TARGET_KIND_OACC_UPDATE)))
9771 new_omp_region (bb, code, parent);
9772 else
9774 /* Otherwise, this directive becomes the parent for a new
9775 region. */
9776 region = new_omp_region (bb, code, parent);
9777 parent = region;
9781 if (single_tree && !parent)
9782 return;
9784 for (son = first_dom_son (CDI_DOMINATORS, bb);
9785 son;
9786 son = next_dom_son (CDI_DOMINATORS, son))
9787 build_omp_regions_1 (son, parent, single_tree);
9790 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9791 root_omp_region. */
9793 static void
9794 build_omp_regions_root (basic_block root)
9796 gcc_assert (root_omp_region == NULL);
9797 build_omp_regions_1 (root, NULL, true);
9798 gcc_assert (root_omp_region != NULL);
9801 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9803 void
9804 omp_expand_local (basic_block head)
9806 build_omp_regions_root (head);
9807 if (dump_file && (dump_flags & TDF_DETAILS))
9809 fprintf (dump_file, "\nOMP region tree\n\n");
9810 dump_omp_region (dump_file, root_omp_region, 0);
9811 fprintf (dump_file, "\n");
9814 remove_exit_barriers (root_omp_region);
9815 expand_omp (root_omp_region);
9817 free_omp_regions ();
9820 /* Scan the CFG and build a tree of OMP regions. Return the root of
9821 the OMP region tree. */
9823 static void
9824 build_omp_regions (void)
9826 gcc_assert (root_omp_region == NULL);
9827 calculate_dominance_info (CDI_DOMINATORS);
9828 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false);
9831 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9833 static unsigned int
9834 execute_expand_omp (void)
9836 build_omp_regions ();
9838 if (!root_omp_region)
9839 return 0;
9841 if (dump_file)
9843 fprintf (dump_file, "\nOMP region tree\n\n");
9844 dump_omp_region (dump_file, root_omp_region, 0);
9845 fprintf (dump_file, "\n");
9848 remove_exit_barriers (root_omp_region);
9850 expand_omp (root_omp_region);
9852 cleanup_tree_cfg ();
9854 free_omp_regions ();
9856 return 0;
9859 /* OMP expansion -- the default pass, run before creation of SSA form. */
9861 namespace {
9863 const pass_data pass_data_expand_omp =
9865 GIMPLE_PASS, /* type */
9866 "ompexp", /* name */
9867 OPTGROUP_NONE, /* optinfo_flags */
9868 TV_NONE, /* tv_id */
9869 PROP_gimple_any, /* properties_required */
9870 0, /* properties_provided */
9871 0, /* properties_destroyed */
9872 0, /* todo_flags_start */
9873 0, /* todo_flags_finish */
9876 class pass_expand_omp : public gimple_opt_pass
9878 public:
9879 pass_expand_omp (gcc::context *ctxt)
9880 : gimple_opt_pass (pass_data_expand_omp, ctxt)
9883 /* opt_pass methods: */
9884 virtual bool gate (function *)
9886 return ((flag_openacc != 0 || flag_openmp != 0 || flag_openmp_simd != 0
9887 || flag_cilkplus != 0) && !seen_error ());
9890 virtual unsigned int execute (function *) { return execute_expand_omp (); }
9892 }; // class pass_expand_omp
9894 } // anon namespace
9896 gimple_opt_pass *
9897 make_pass_expand_omp (gcc::context *ctxt)
9899 return new pass_expand_omp (ctxt);
9902 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9903 convert it to gimple. */
9904 static void
9905 omp_gimple_assign_with_ops (tree_code op, tree dest, tree src, gimple_seq *seq)
9907 gimple stmt;
9909 if (TREE_CODE (TREE_TYPE (dest)) != COMPLEX_TYPE)
9911 stmt = gimple_build_assign_with_ops (op, dest, dest, src);
9912 gimple_seq_add_stmt (seq, stmt);
9913 return;
9916 tree t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)), NULL);
9917 tree rdest = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest);
9918 gimplify_assign (t, rdest, seq);
9919 rdest = t;
9921 t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)), NULL);
9922 tree idest = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest);
9923 gimplify_assign (t, idest, seq);
9924 idest = t;
9926 t = create_tmp_var (TREE_TYPE (TREE_TYPE (src)), NULL);
9927 tree rsrc = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src);
9928 gimplify_assign (t, rsrc, seq);
9929 rsrc = t;
9931 t = create_tmp_var (TREE_TYPE (TREE_TYPE (src)), NULL);
9932 tree isrc = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src);
9933 gimplify_assign (t, isrc, seq);
9934 isrc = t;
9936 tree r = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)), NULL);
9937 tree i = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)), NULL);
9938 tree result;
9940 gcc_assert (op == PLUS_EXPR || op == MULT_EXPR);
9942 if (op == PLUS_EXPR)
9944 stmt = gimple_build_assign_with_ops (op, r, rdest, rsrc);
9945 gimple_seq_add_stmt (seq, stmt);
9947 stmt = gimple_build_assign_with_ops (op, i, idest, isrc);
9948 gimple_seq_add_stmt (seq, stmt);
9950 else if (op == MULT_EXPR)
9952 /* Let x = a + ib = dest, y = c + id = src.
9953 x * y = (ac - bd) + i(ad + bc) */
9954 tree ac = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)), NULL);
9955 tree bd = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)), NULL);
9956 tree ad = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)), NULL);
9957 tree bc = create_tmp_var (TREE_TYPE (TREE_TYPE (dest)), NULL);
9959 stmt = gimple_build_assign_with_ops (MULT_EXPR, ac, rdest, rsrc);
9960 gimple_seq_add_stmt (seq, stmt);
9962 stmt = gimple_build_assign_with_ops (MULT_EXPR, bd, idest, isrc);
9963 gimple_seq_add_stmt (seq, stmt);
9965 stmt = gimple_build_assign_with_ops (MINUS_EXPR, r, ac, bd);
9966 gimple_seq_add_stmt (seq, stmt);
9968 stmt = gimple_build_assign_with_ops (MULT_EXPR, ad, rdest, isrc);
9969 gimple_seq_add_stmt (seq, stmt);
9971 stmt = gimple_build_assign_with_ops (MULT_EXPR, bd, idest, rsrc);
9972 gimple_seq_add_stmt (seq, stmt);
9974 stmt = gimple_build_assign_with_ops (PLUS_EXPR, i, ad, bc);
9975 gimple_seq_add_stmt (seq, stmt);
9978 result = build2 (COMPLEX_EXPR, TREE_TYPE (dest), r, i);
9979 gimplify_assign (dest, result, seq);
9982 /* Helper function to initialize local data for the reduction arrays.
9983 The reduction arrays need to be placed inside the calling function
9984 for accelerators, or else the host won't be able to preform the final
9985 reduction. FIXME: This function assumes that there are
9986 vector_length threads in total. */
9988 static void
9989 initialize_reduction_data (tree clauses, tree nthreads, gimple_seq *stmt_seqp,
9990 omp_context *ctx)
9992 gcc_assert (is_gimple_omp_oacc_specifically (ctx->stmt));
9994 tree c, t, oc;
9995 gimple stmt;
9996 omp_context *octx;
9997 tree (*gimple_omp_clauses) (const_gimple);
9998 void (*gimple_omp_set_clauses) (gimple, tree);
10000 /* Find the innermost PARALLEL openmp context. FIXME: OpenACC kernels
10001 may require extra care unless they are converted to openmp for loops. */
10003 if (gimple_code (ctx->stmt) == GIMPLE_OACC_PARALLEL)
10004 octx = ctx;
10005 else
10006 octx = ctx->outer;
10008 gimple_omp_clauses = gimple_oacc_parallel_clauses;
10009 gimple_omp_set_clauses = gimple_oacc_parallel_set_clauses;
10011 /* Extract the clauses. */
10012 oc = gimple_omp_clauses (octx->stmt);
10014 /* Find the last outer clause. */
10015 for (; oc && OMP_CLAUSE_CHAIN (oc); oc = OMP_CLAUSE_CHAIN (oc))
10018 /* Allocate arrays for each reduction variable. */
10019 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
10021 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
10022 continue;
10024 tree var = OMP_CLAUSE_DECL (c);
10025 tree type = get_base_type (var);
10026 tree array = lookup_reduction (omp_get_id (var), ctx);
10027 tree size, call;
10029 /* Calculate size of the reduction array. */
10030 t = create_tmp_var (TREE_TYPE (nthreads), NULL);
10031 stmt = gimple_build_assign_with_ops (MULT_EXPR, t, nthreads,
10032 fold_convert (TREE_TYPE (nthreads),
10033 TYPE_SIZE_UNIT (type)));
10034 gimple_seq_add_stmt (stmt_seqp, stmt);
10036 size = create_tmp_var (sizetype, NULL);
10037 gimplify_assign (size, fold_build1 (NOP_EXPR, sizetype, t), stmt_seqp);
10039 /* Now allocate memory for it. FIXME: Allocating memory for the
10040 reduction array may be unnecessary once the final reduction is able
10041 to be preformed on the accelerator. Instead of allocating memory on
10042 the host side, it could just be allocated on the accelerator. */
10043 call = unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA));
10044 stmt = gimple_build_call (call, 1, size);
10045 gimple_call_set_lhs (stmt, array);
10046 gimple_seq_add_stmt (stmt_seqp, stmt);
10048 /* Map this array into the accelerator. */
10050 /* Add the reduction array to the list of clauses. */
10051 /* FIXME: Currently, these variables must be placed in the outer
10052 most clause so that copy-out works. */
10053 tree x = array;
10054 t = build_omp_clause (gimple_location (ctx->stmt), OMP_CLAUSE_MAP);
10055 OMP_CLAUSE_MAP_KIND (t) = OMP_CLAUSE_MAP_FORCE_FROM;
10056 OMP_CLAUSE_DECL (t) = x;
10057 OMP_CLAUSE_CHAIN (t) = NULL;
10058 if (oc)
10059 OMP_CLAUSE_CHAIN (oc) = t;
10060 else
10061 gimple_omp_set_clauses (octx->stmt, t);
10062 OMP_CLAUSE_SIZE (t) = size;
10063 oc = t;
10067 /* Helper function to process the array of partial reductions. Nthreads
10068 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
10069 cannot be used here, because nthreads on the host may be different than
10070 on the accelerator. */
10072 static void
10073 finalize_reduction_data (tree clauses, tree nthreads, gimple_seq *stmt_seqp,
10074 omp_context *ctx)
10076 gcc_assert (is_gimple_omp_oacc_specifically (ctx->stmt));
10078 tree c, x, var, array, loop_header, loop_body, loop_exit, type;
10079 gimple stmt;
10081 /* Create for loop.
10083 let var = the original reduction variable
10084 let array = reduction variable array
10086 for (i = 0; i < nthreads; i++)
10087 var op= array[i]
10090 loop_header = create_artificial_label (UNKNOWN_LOCATION);
10091 loop_body = create_artificial_label (UNKNOWN_LOCATION);
10092 loop_exit = create_artificial_label (UNKNOWN_LOCATION);
10094 /* Create and initialize an index variable. */
10095 tree ix = create_tmp_var (sizetype, NULL);
10096 gimplify_assign (ix, fold_build1 (NOP_EXPR, sizetype, integer_zero_node),
10097 stmt_seqp);
10099 /* Insert the loop header label here. */
10100 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_header));
10102 /* Exit loop if ix >= nthreads. */
10103 x = create_tmp_var (sizetype, NULL);
10104 gimplify_assign (x, fold_build1 (NOP_EXPR, sizetype, nthreads), stmt_seqp);
10105 stmt = gimple_build_cond (GE_EXPR, ix, x, loop_exit, loop_body);
10106 gimple_seq_add_stmt (stmt_seqp, stmt);
10108 /* Insert the loop body label here. */
10109 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_body));
10111 /* Collapse each reduction array, one element at a time. */
10112 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
10114 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
10115 continue;
10117 tree_code reduction_code = OMP_CLAUSE_REDUCTION_CODE (c);
10119 /* reduction(-:var) sums up the partial results, so it acts
10120 identically to reduction(+:var). */
10121 if (reduction_code == MINUS_EXPR)
10122 reduction_code = PLUS_EXPR;
10124 /* Set up reduction variable var. */
10125 var = OMP_CLAUSE_DECL (c);
10126 type = get_base_type (var);
10127 array = lookup_reduction (omp_get_id (OMP_CLAUSE_DECL (c)), ctx);
10129 /* Calculate the array offset. */
10130 tree offset = create_tmp_var (sizetype, NULL);
10131 gimplify_assign (offset, TYPE_SIZE_UNIT (type), stmt_seqp);
10132 stmt = gimple_build_assign_with_ops (MULT_EXPR, offset, offset, ix);
10133 gimple_seq_add_stmt (stmt_seqp, stmt);
10135 tree ptr = create_tmp_var (TREE_TYPE (array), NULL);
10136 stmt = gimple_build_assign_with_ops (POINTER_PLUS_EXPR, ptr, array,
10137 offset);
10138 gimple_seq_add_stmt (stmt_seqp, stmt);
10140 /* Extract array[ix] into mem. */
10141 tree mem = create_tmp_var (type, NULL);
10142 gimplify_assign (mem, build_simple_mem_ref (ptr), stmt_seqp);
10144 /* Find the original reduction variable. */
10145 if (is_reference (var))
10146 var = build_simple_mem_ref (var);
10148 tree t = create_tmp_var (type, NULL);
10150 x = lang_hooks.decls.omp_clause_assign_op (c, t, var);
10151 gimplify_and_add (unshare_expr(x), stmt_seqp);
10153 /* var = var op mem */
10154 switch (OMP_CLAUSE_REDUCTION_CODE (c))
10156 case TRUTH_ANDIF_EXPR:
10157 case TRUTH_ORIF_EXPR:
10158 t = fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c), integer_type_node,
10159 t, mem);
10160 gimplify_and_add (t, stmt_seqp);
10161 break;
10162 default:
10163 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
10164 omp_gimple_assign_with_ops (OMP_CLAUSE_REDUCTION_CODE (c),
10165 t, mem, stmt_seqp);
10168 t = fold_build1 (NOP_EXPR, TREE_TYPE (var), t);
10169 x = lang_hooks.decls.omp_clause_assign_op (c, var, t);
10170 gimplify_and_add (unshare_expr(x), stmt_seqp);
10173 /* Increment the induction variable. */
10174 tree one = fold_build1 (NOP_EXPR, sizetype, integer_one_node);
10175 stmt = gimple_build_assign_with_ops (PLUS_EXPR, ix, ix, one);
10176 gimple_seq_add_stmt (stmt_seqp, stmt);
10178 /* Go back to the top of the loop. */
10179 gimple_seq_add_stmt (stmt_seqp, gimple_build_goto (loop_header));
10181 /* Place the loop exit label here. */
10182 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_exit));
10185 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
10186 scan that for reductions. */
10188 static void
10189 process_reduction_data (gimple_seq *body, gimple_seq *in_stmt_seqp,
10190 gimple_seq *out_stmt_seqp, omp_context *ctx)
10192 gcc_assert (is_gimple_omp_oacc_specifically (ctx->stmt));
10194 gimple_stmt_iterator gsi;
10195 gimple_seq inner = NULL;
10196 gimple stmt;
10198 /* A collapse clause may have inserted a new bind block. */
10199 gsi = gsi_start (*body);
10200 while (!gsi_end_p (gsi))
10202 stmt = gsi_stmt (gsi);
10203 if (gimple_code (stmt) == GIMPLE_BIND)
10205 inner = gimple_bind_body (stmt);
10206 body = &inner;
10207 gsi = gsi_start (*body);
10209 else if (gimple_code (stmt) == GIMPLE_OMP_FOR)
10210 break;
10211 else
10212 gsi_next (&gsi);
10215 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
10217 tree clauses, nthreads, t, c, acc_device, acc_device_host, call,
10218 enter, exit;
10219 bool reduction_found = false;
10221 stmt = gsi_stmt (gsi);
10223 switch (gimple_code (stmt))
10225 /* FIXME: A reduction may also appear in an oacc parallel. */
10226 case GIMPLE_OMP_FOR:
10227 clauses = gimple_omp_for_clauses (stmt);
10229 /* Search for a reduction clause. */
10230 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
10231 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
10233 reduction_found = true;
10234 break;
10237 if (!reduction_found)
10238 break;
10240 ctx = maybe_lookup_ctx (stmt);
10241 t = NULL_TREE;
10243 /* Extract the number of threads. */
10244 nthreads = create_tmp_var (sizetype, NULL);
10245 t = oacc_max_threads (ctx);
10246 gimplify_assign (nthreads, t, in_stmt_seqp);
10248 /* Determine if this is kernel will be executed on the host. */
10249 call = builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE);
10250 acc_device = create_tmp_var (integer_type_node, ".acc_device_type");
10251 stmt = gimple_build_call (call, 0);
10252 gimple_call_set_lhs (stmt, acc_device);
10253 gimple_seq_add_stmt (in_stmt_seqp, stmt);
10255 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
10256 acc_device_host = create_tmp_var (integer_type_node,
10257 ".acc_device_host");
10258 gimplify_assign (acc_device_host, build_int_cst (integer_type_node,
10260 in_stmt_seqp);
10262 enter = create_artificial_label (UNKNOWN_LOCATION);
10263 exit = create_artificial_label (UNKNOWN_LOCATION);
10265 stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host,
10266 enter, exit);
10267 gimple_seq_add_stmt (in_stmt_seqp, stmt);
10268 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter));
10269 gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype,
10270 integer_one_node),
10271 in_stmt_seqp);
10272 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit));
10274 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
10275 gimplify_assign (acc_device_host, build_int_cst (integer_type_node,
10277 in_stmt_seqp);
10279 enter = create_artificial_label (UNKNOWN_LOCATION);
10280 exit = create_artificial_label (UNKNOWN_LOCATION);
10282 stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host,
10283 enter, exit);
10284 gimple_seq_add_stmt (in_stmt_seqp, stmt);
10285 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter));
10286 gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype,
10287 integer_one_node),
10288 in_stmt_seqp);
10289 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit));
10291 initialize_reduction_data (clauses, nthreads, in_stmt_seqp, ctx);
10292 finalize_reduction_data (clauses, nthreads, out_stmt_seqp, ctx);
10293 break;
10294 default:
10295 // Scan for other directives which support reduction here.
10296 break;
10301 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
10303 /* Lower the OpenACC offload directive in the current statement
10304 in GSI_P. CTX holds context information for the directive. */
10306 static void
10307 lower_oacc_offload (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10309 tree clauses;
10310 tree child_fn, t, c;
10311 gimple stmt = gsi_stmt (*gsi_p);
10312 gimple par_bind, bind;
10313 gimple_seq par_body, olist, ilist, orlist, irlist, new_body;
10314 location_t loc = gimple_location (stmt);
10315 unsigned int map_cnt = 0;
10316 tree (*gimple_omp_clauses) (const_gimple);
10317 void (*gimple_omp_set_data_arg) (gimple, tree);
10318 switch (gimple_code (stmt))
10320 case GIMPLE_OACC_KERNELS:
10321 gimple_omp_clauses = gimple_oacc_kernels_clauses;
10322 gimple_omp_set_data_arg = gimple_oacc_kernels_set_data_arg;
10323 break;
10324 case GIMPLE_OACC_PARALLEL:
10325 gimple_omp_clauses = gimple_oacc_parallel_clauses;
10326 gimple_omp_set_data_arg = gimple_oacc_parallel_set_data_arg;
10327 break;
10328 default:
10329 gcc_unreachable ();
10332 clauses = gimple_omp_clauses (stmt);
10333 par_bind = gimple_seq_first_stmt (gimple_omp_body (stmt));
10334 par_body = gimple_bind_body (par_bind);
10335 child_fn = ctx->cb.dst_fn;
10337 push_gimplify_context ();
10339 irlist = NULL;
10340 orlist = NULL;
10341 process_reduction_data (&par_body, &irlist, &orlist, ctx);
10343 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
10344 switch (OMP_CLAUSE_CODE (c))
10346 tree var, x;
10348 default:
10349 break;
10350 case OMP_CLAUSE_MAP:
10351 var = OMP_CLAUSE_DECL (c);
10352 if (!DECL_P (var))
10354 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
10355 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
10356 map_cnt++;
10357 continue;
10360 if (DECL_SIZE (var)
10361 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
10363 tree var2 = DECL_VALUE_EXPR (var);
10364 gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
10365 var2 = TREE_OPERAND (var2, 0);
10366 gcc_assert (DECL_P (var2));
10367 var = var2;
10370 if (!maybe_lookup_field (var, ctx))
10371 continue;
10373 /* Preserve indentation of lower_omp_target. */
10374 if (1)
10376 x = build_receiver_ref (var, true, ctx);
10377 tree new_var = lookup_decl (var, ctx);
10378 gcc_assert (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
10379 || (OMP_CLAUSE_MAP_KIND (c)
10380 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
10381 || TREE_CODE (TREE_TYPE (var)) != ARRAY_TYPE);
10382 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
10383 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
10384 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
10385 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
10386 x = build_simple_mem_ref (x);
10387 SET_DECL_VALUE_EXPR (new_var, x);
10388 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
10390 map_cnt++;
10393 target_nesting_level++;
10394 lower_omp (&par_body, ctx);
10395 target_nesting_level--;
10397 /* Declare all the variables created by mapping and the variables
10398 declared in the scope of the body. */
10399 record_vars_into (ctx->block_vars, child_fn);
10400 record_vars_into (gimple_bind_vars (par_bind), child_fn);
10402 olist = NULL;
10403 ilist = NULL;
10404 if (ctx->record_type)
10406 ctx->sender_decl
10407 = create_tmp_var (ctx->record_type, ".omp_data_arr");
10408 DECL_NAMELESS (ctx->sender_decl) = 1;
10409 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
10410 t = make_tree_vec (3);
10411 TREE_VEC_ELT (t, 0) = ctx->sender_decl;
10412 TREE_VEC_ELT (t, 1)
10413 = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt),
10414 ".omp_data_sizes");
10415 DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
10416 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
10417 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
10418 TREE_VEC_ELT (t, 2)
10419 = create_tmp_var (build_array_type_nelts (short_unsigned_type_node,
10420 map_cnt),
10421 ".omp_data_kinds");
10422 DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
10423 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
10424 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
10425 gimple_omp_set_data_arg (stmt, t);
10427 vec<constructor_elt, va_gc> *vsize;
10428 vec<constructor_elt, va_gc> *vkind;
10429 vec_alloc (vsize, map_cnt);
10430 vec_alloc (vkind, map_cnt);
10431 unsigned int map_idx = 0;
10433 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
10434 switch (OMP_CLAUSE_CODE (c))
10436 tree ovar, nc;
10438 default:
10439 break;
10440 case OMP_CLAUSE_MAP:
10441 nc = c;
10442 ovar = OMP_CLAUSE_DECL (c);
10443 if (!DECL_P (ovar))
10445 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
10446 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
10448 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c))
10449 == get_base_address (ovar));
10450 nc = OMP_CLAUSE_CHAIN (c);
10451 ovar = OMP_CLAUSE_DECL (nc);
10453 else
10455 tree x = build_sender_ref (ovar, ctx);
10456 tree v
10457 = build_fold_addr_expr_with_type (ovar, ptr_type_node);
10458 gimplify_assign (x, v, &ilist);
10459 nc = NULL_TREE;
10462 else
10464 if (DECL_SIZE (ovar)
10465 && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST)
10467 tree ovar2 = DECL_VALUE_EXPR (ovar);
10468 gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
10469 ovar2 = TREE_OPERAND (ovar2, 0);
10470 gcc_assert (DECL_P (ovar2));
10471 ovar = ovar2;
10473 if (!maybe_lookup_field (ovar, ctx))
10474 continue;
10477 unsigned int talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
10478 if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
10479 talign = DECL_ALIGN_UNIT (ovar);
10480 if (nc)
10482 tree var = lookup_decl_in_outer_ctx (ovar, ctx);
10483 tree x = build_sender_ref (ovar, ctx);
10484 gcc_assert (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
10485 || (OMP_CLAUSE_MAP_KIND (c)
10486 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
10487 || TREE_CODE (TREE_TYPE (ovar)) != ARRAY_TYPE);
10488 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
10489 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
10490 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
10491 && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
10493 tree avar
10494 = create_tmp_var (TREE_TYPE (TREE_TYPE (x)), NULL);
10495 mark_addressable (avar);
10496 gimplify_assign (avar, build_fold_addr_expr (var), &ilist);
10497 talign = DECL_ALIGN_UNIT (avar);
10498 avar = build_fold_addr_expr (avar);
10499 gimplify_assign (x, avar, &ilist);
10501 else if (is_gimple_reg (var)
10502 && !maybe_lookup_reduction (var, ctx))
10504 tree avar = create_tmp_var (TREE_TYPE (var), NULL);
10505 mark_addressable (avar);
10506 enum omp_clause_map_kind map_kind
10507 = OMP_CLAUSE_MAP_KIND (c);
10508 if ((!(map_kind & OMP_CLAUSE_MAP_SPECIAL)
10509 && (map_kind & OMP_CLAUSE_MAP_TO))
10510 || map_kind == OMP_CLAUSE_MAP_POINTER
10511 || map_kind == OMP_CLAUSE_MAP_TO_PSET
10512 || map_kind == OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
10513 gimplify_assign (avar, var, &ilist);
10514 avar = build_fold_addr_expr (avar);
10515 gimplify_assign (x, avar, &ilist);
10516 if (((!(map_kind & OMP_CLAUSE_MAP_SPECIAL)
10517 && (map_kind & OMP_CLAUSE_MAP_FROM))
10518 || map_kind == OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
10519 && !TYPE_READONLY (TREE_TYPE (var)))
10521 x = build_sender_ref (ovar, ctx);
10522 x = build_simple_mem_ref (x);
10523 gimplify_assign (var, x, &olist);
10526 else
10528 if (!maybe_lookup_reduction (var, ctx))
10529 var = build_fold_addr_expr (var);
10530 gimplify_assign (x, var, &ilist);
10533 tree s = OMP_CLAUSE_SIZE (c);
10534 if (s == NULL_TREE)
10535 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
10536 s = fold_convert (size_type_node, s);
10537 tree purpose = size_int (map_idx++);
10538 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
10539 if (TREE_CODE (s) != INTEGER_CST)
10540 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
10542 unsigned short tkind = 0;
10543 switch (OMP_CLAUSE_CODE (c))
10545 case OMP_CLAUSE_MAP:
10546 tkind = OMP_CLAUSE_MAP_KIND (c);
10547 break;
10548 default:
10549 gcc_unreachable ();
10551 talign = ceil_log2 (talign);
10552 tkind |= talign << 8;
10553 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
10554 build_int_cst (short_unsigned_type_node,
10555 tkind));
10556 if (nc && nc != c)
10557 c = nc;
10560 gcc_assert (map_idx == map_cnt);
10562 DECL_INITIAL (TREE_VEC_ELT (t, 1))
10563 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
10564 DECL_INITIAL (TREE_VEC_ELT (t, 2))
10565 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
10566 if (!TREE_STATIC (TREE_VEC_ELT (t, 1)))
10568 gimple_seq initlist = NULL;
10569 force_gimple_operand (build1 (DECL_EXPR, void_type_node,
10570 TREE_VEC_ELT (t, 1)),
10571 &initlist, true, NULL_TREE);
10572 gimple_seq_add_seq (&ilist, initlist);
10574 tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)),
10575 NULL);
10576 TREE_THIS_VOLATILE (clobber) = 1;
10577 gimple_seq_add_stmt (&olist,
10578 gimple_build_assign (TREE_VEC_ELT (t, 1),
10579 clobber));
10582 tree clobber = build_constructor (ctx->record_type, NULL);
10583 TREE_THIS_VOLATILE (clobber) = 1;
10584 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
10585 clobber));
10588 /* Once all the expansions are done, sequence all the different
10589 fragments inside gimple_omp_body. */
10591 new_body = NULL;
10593 if (ctx->record_type)
10595 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
10596 /* fixup_child_record_type might have changed receiver_decl's type. */
10597 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
10598 gimple_seq_add_stmt (&new_body,
10599 gimple_build_assign (ctx->receiver_decl, t));
10602 gimple_seq_add_seq (&new_body, par_body);
10603 gcc_assert (!ctx->cancellable);
10604 new_body = maybe_catch_exception (new_body);
10605 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
10606 gimple_omp_set_body (stmt, new_body);
10608 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
10609 gsi_replace (gsi_p, bind, true);
10610 gimple_bind_add_seq (bind, irlist);
10611 gimple_bind_add_seq (bind, ilist);
10612 gimple_bind_add_stmt (bind, stmt);
10613 gimple_bind_add_seq (bind, olist);
10614 gimple_bind_add_seq (bind, orlist);
10616 pop_gimplify_context (NULL);
10619 /* If ctx is a worksharing context inside of a cancellable parallel
10620 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
10621 and conditional branch to parallel's cancel_label to handle
10622 cancellation in the implicit barrier. */
10624 static void
10625 maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body)
10627 gimple omp_return = gimple_seq_last_stmt (*body);
10628 gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
10629 if (gimple_omp_return_nowait_p (omp_return))
10630 return;
10631 if (ctx->outer
10632 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL
10633 && ctx->outer->cancellable)
10635 tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
10636 tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
10637 tree lhs = create_tmp_var (c_bool_type, NULL);
10638 gimple_omp_return_set_lhs (omp_return, lhs);
10639 tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
10640 gimple g = gimple_build_cond (NE_EXPR, lhs,
10641 fold_convert (c_bool_type,
10642 boolean_false_node),
10643 ctx->outer->cancel_label, fallthru_label);
10644 gimple_seq_add_stmt (body, g);
10645 gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
10649 /* Lower the OpenMP sections directive in the current statement in GSI_P.
10650 CTX is the enclosing OMP context for the current statement. */
10652 static void
10653 lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10655 tree block, control;
10656 gimple_stmt_iterator tgsi;
10657 gimple stmt, new_stmt, bind, t;
10658 gimple_seq ilist, dlist, olist, new_body;
10660 stmt = gsi_stmt (*gsi_p);
10662 push_gimplify_context ();
10664 dlist = NULL;
10665 ilist = NULL;
10666 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
10667 &ilist, &dlist, ctx, NULL);
10669 new_body = gimple_omp_body (stmt);
10670 gimple_omp_set_body (stmt, NULL);
10671 tgsi = gsi_start (new_body);
10672 for (; !gsi_end_p (tgsi); gsi_next (&tgsi))
10674 omp_context *sctx;
10675 gimple sec_start;
10677 sec_start = gsi_stmt (tgsi);
10678 sctx = maybe_lookup_ctx (sec_start);
10679 gcc_assert (sctx);
10681 lower_omp (gimple_omp_body_ptr (sec_start), sctx);
10682 gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start),
10683 GSI_CONTINUE_LINKING);
10684 gimple_omp_set_body (sec_start, NULL);
10686 if (gsi_one_before_end_p (tgsi))
10688 gimple_seq l = NULL;
10689 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
10690 &l, ctx);
10691 gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
10692 gimple_omp_section_set_last (sec_start);
10695 gsi_insert_after (&tgsi, gimple_build_omp_return (false),
10696 GSI_CONTINUE_LINKING);
10699 block = make_node (BLOCK);
10700 bind = gimple_build_bind (NULL, new_body, block);
10702 olist = NULL;
10703 lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx);
10705 block = make_node (BLOCK);
10706 new_stmt = gimple_build_bind (NULL, NULL, block);
10707 gsi_replace (gsi_p, new_stmt, true);
10709 pop_gimplify_context (new_stmt);
10710 gimple_bind_append_vars (new_stmt, ctx->block_vars);
10711 BLOCK_VARS (block) = gimple_bind_vars (bind);
10712 if (BLOCK_VARS (block))
10713 TREE_USED (block) = 1;
10715 new_body = NULL;
10716 gimple_seq_add_seq (&new_body, ilist);
10717 gimple_seq_add_stmt (&new_body, stmt);
10718 gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
10719 gimple_seq_add_stmt (&new_body, bind);
10721 control = create_tmp_var (unsigned_type_node, ".section");
10722 t = gimple_build_omp_continue (control, control);
10723 gimple_omp_sections_set_control (stmt, control);
10724 gimple_seq_add_stmt (&new_body, t);
10726 gimple_seq_add_seq (&new_body, olist);
10727 if (ctx->cancellable)
10728 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
10729 gimple_seq_add_seq (&new_body, dlist);
10731 new_body = maybe_catch_exception (new_body);
10733 t = gimple_build_omp_return
10734 (!!find_omp_clause (gimple_omp_sections_clauses (stmt),
10735 OMP_CLAUSE_NOWAIT));
10736 gimple_seq_add_stmt (&new_body, t);
10737 maybe_add_implicit_barrier_cancel (ctx, &new_body);
10739 gimple_bind_set_body (new_stmt, new_body);
10743 /* A subroutine of lower_omp_single. Expand the simple form of
10744 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10746 if (GOMP_single_start ())
10747 BODY;
10748 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10750 FIXME. It may be better to delay expanding the logic of this until
10751 pass_expand_omp. The expanded logic may make the job more difficult
10752 to a synchronization analysis pass. */
10754 static void
10755 lower_omp_single_simple (gimple single_stmt, gimple_seq *pre_p)
10757 location_t loc = gimple_location (single_stmt);
10758 tree tlabel = create_artificial_label (loc);
10759 tree flabel = create_artificial_label (loc);
10760 gimple call, cond;
10761 tree lhs, decl;
10763 decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START);
10764 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl)), NULL);
10765 call = gimple_build_call (decl, 0);
10766 gimple_call_set_lhs (call, lhs);
10767 gimple_seq_add_stmt (pre_p, call);
10769 cond = gimple_build_cond (EQ_EXPR, lhs,
10770 fold_convert_loc (loc, TREE_TYPE (lhs),
10771 boolean_true_node),
10772 tlabel, flabel);
10773 gimple_seq_add_stmt (pre_p, cond);
10774 gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel));
10775 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
10776 gimple_seq_add_stmt (pre_p, gimple_build_label (flabel));
10780 /* A subroutine of lower_omp_single. Expand the simple form of
10781 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10783 #pragma omp single copyprivate (a, b, c)
10785 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10788 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10790 BODY;
10791 copyout.a = a;
10792 copyout.b = b;
10793 copyout.c = c;
10794 GOMP_single_copy_end (&copyout);
10796 else
10798 a = copyout_p->a;
10799 b = copyout_p->b;
10800 c = copyout_p->c;
10802 GOMP_barrier ();
10805 FIXME. It may be better to delay expanding the logic of this until
10806 pass_expand_omp. The expanded logic may make the job more difficult
10807 to a synchronization analysis pass. */
10809 static void
10810 lower_omp_single_copy (gimple single_stmt, gimple_seq *pre_p, omp_context *ctx)
10812 tree ptr_type, t, l0, l1, l2, bfn_decl;
10813 gimple_seq copyin_seq;
10814 location_t loc = gimple_location (single_stmt);
10816 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
10818 ptr_type = build_pointer_type (ctx->record_type);
10819 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
10821 l0 = create_artificial_label (loc);
10822 l1 = create_artificial_label (loc);
10823 l2 = create_artificial_label (loc);
10825 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START);
10826 t = build_call_expr_loc (loc, bfn_decl, 0);
10827 t = fold_convert_loc (loc, ptr_type, t);
10828 gimplify_assign (ctx->receiver_decl, t, pre_p);
10830 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
10831 build_int_cst (ptr_type, 0));
10832 t = build3 (COND_EXPR, void_type_node, t,
10833 build_and_jump (&l0), build_and_jump (&l1));
10834 gimplify_and_add (t, pre_p);
10836 gimple_seq_add_stmt (pre_p, gimple_build_label (l0));
10838 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt));
10840 copyin_seq = NULL;
10841 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p,
10842 &copyin_seq, ctx);
10844 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
10845 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END);
10846 t = build_call_expr_loc (loc, bfn_decl, 1, t);
10847 gimplify_and_add (t, pre_p);
10849 t = build_and_jump (&l2);
10850 gimplify_and_add (t, pre_p);
10852 gimple_seq_add_stmt (pre_p, gimple_build_label (l1));
10854 gimple_seq_add_seq (pre_p, copyin_seq);
10856 gimple_seq_add_stmt (pre_p, gimple_build_label (l2));
10860 /* Expand code for an OpenMP single directive. */
10862 static void
10863 lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10865 tree block;
10866 gimple t, bind, single_stmt = gsi_stmt (*gsi_p);
10867 gimple_seq bind_body, bind_body_tail = NULL, dlist;
10869 push_gimplify_context ();
10871 block = make_node (BLOCK);
10872 bind = gimple_build_bind (NULL, NULL, block);
10873 gsi_replace (gsi_p, bind, true);
10874 bind_body = NULL;
10875 dlist = NULL;
10876 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt),
10877 &bind_body, &dlist, ctx, NULL);
10878 lower_omp (gimple_omp_body_ptr (single_stmt), ctx);
10880 gimple_seq_add_stmt (&bind_body, single_stmt);
10882 if (ctx->record_type)
10883 lower_omp_single_copy (single_stmt, &bind_body, ctx);
10884 else
10885 lower_omp_single_simple (single_stmt, &bind_body);
10887 gimple_omp_set_body (single_stmt, NULL);
10889 gimple_seq_add_seq (&bind_body, dlist);
10891 bind_body = maybe_catch_exception (bind_body);
10893 t = gimple_build_omp_return
10894 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt),
10895 OMP_CLAUSE_NOWAIT));
10896 gimple_seq_add_stmt (&bind_body_tail, t);
10897 maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail);
10898 if (ctx->record_type)
10900 gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
10901 tree clobber = build_constructor (ctx->record_type, NULL);
10902 TREE_THIS_VOLATILE (clobber) = 1;
10903 gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
10904 clobber), GSI_SAME_STMT);
10906 gimple_seq_add_seq (&bind_body, bind_body_tail);
10907 gimple_bind_set_body (bind, bind_body);
10909 pop_gimplify_context (bind);
10911 gimple_bind_append_vars (bind, ctx->block_vars);
10912 BLOCK_VARS (block) = ctx->block_vars;
10913 if (BLOCK_VARS (block))
10914 TREE_USED (block) = 1;
10918 /* Expand code for an OpenMP master directive. */
10920 static void
10921 lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10923 tree block, lab = NULL, x, bfn_decl;
10924 gimple stmt = gsi_stmt (*gsi_p), bind;
10925 location_t loc = gimple_location (stmt);
10926 gimple_seq tseq;
10928 push_gimplify_context ();
10930 block = make_node (BLOCK);
10931 bind = gimple_build_bind (NULL, NULL, block);
10932 gsi_replace (gsi_p, bind, true);
10933 gimple_bind_add_stmt (bind, stmt);
10935 bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
10936 x = build_call_expr_loc (loc, bfn_decl, 0);
10937 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node);
10938 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
10939 tseq = NULL;
10940 gimplify_and_add (x, &tseq);
10941 gimple_bind_add_seq (bind, tseq);
10943 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10944 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
10945 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10946 gimple_omp_set_body (stmt, NULL);
10948 gimple_bind_add_stmt (bind, gimple_build_label (lab));
10950 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10952 pop_gimplify_context (bind);
10954 gimple_bind_append_vars (bind, ctx->block_vars);
10955 BLOCK_VARS (block) = ctx->block_vars;
10959 /* Expand code for an OpenMP taskgroup directive. */
10961 static void
10962 lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10964 gimple stmt = gsi_stmt (*gsi_p), bind, x;
10965 tree block = make_node (BLOCK);
10967 bind = gimple_build_bind (NULL, NULL, block);
10968 gsi_replace (gsi_p, bind, true);
10969 gimple_bind_add_stmt (bind, stmt);
10971 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
10973 gimple_bind_add_stmt (bind, x);
10975 lower_omp (gimple_omp_body_ptr (stmt), ctx);
10976 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
10977 gimple_omp_set_body (stmt, NULL);
10979 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
10981 gimple_bind_append_vars (bind, ctx->block_vars);
10982 BLOCK_VARS (block) = ctx->block_vars;
10986 /* Expand code for an OpenMP ordered directive. */
10988 static void
10989 lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx)
10991 tree block;
10992 gimple stmt = gsi_stmt (*gsi_p), bind, x;
10994 push_gimplify_context ();
10996 block = make_node (BLOCK);
10997 bind = gimple_build_bind (NULL, NULL, block);
10998 gsi_replace (gsi_p, bind, true);
10999 gimple_bind_add_stmt (bind, stmt);
11001 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START),
11003 gimple_bind_add_stmt (bind, x);
11005 lower_omp (gimple_omp_body_ptr (stmt), ctx);
11006 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
11007 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
11008 gimple_omp_set_body (stmt, NULL);
11010 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END), 0);
11011 gimple_bind_add_stmt (bind, x);
11013 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
11015 pop_gimplify_context (bind);
11017 gimple_bind_append_vars (bind, ctx->block_vars);
11018 BLOCK_VARS (block) = gimple_bind_vars (bind);
11022 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
11023 substitution of a couple of function calls. But in the NAMED case,
11024 requires that languages coordinate a symbol name. It is therefore
11025 best put here in common code. */
11027 static GTY((param1_is (tree), param2_is (tree)))
11028 splay_tree critical_name_mutexes;
11030 static void
11031 lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11033 tree block;
11034 tree name, lock, unlock;
11035 gimple stmt = gsi_stmt (*gsi_p), bind;
11036 location_t loc = gimple_location (stmt);
11037 gimple_seq tbody;
11039 name = gimple_omp_critical_name (stmt);
11040 if (name)
11042 tree decl;
11043 splay_tree_node n;
11045 if (!critical_name_mutexes)
11046 critical_name_mutexes
11047 = splay_tree_new_ggc (splay_tree_compare_pointers,
11048 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s,
11049 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s);
11051 n = splay_tree_lookup (critical_name_mutexes, (splay_tree_key) name);
11052 if (n == NULL)
11054 char *new_str;
11056 decl = create_tmp_var_raw (ptr_type_node, NULL);
11058 new_str = ACONCAT ((".gomp_critical_user_",
11059 IDENTIFIER_POINTER (name), NULL));
11060 DECL_NAME (decl) = get_identifier (new_str);
11061 TREE_PUBLIC (decl) = 1;
11062 TREE_STATIC (decl) = 1;
11063 DECL_COMMON (decl) = 1;
11064 DECL_ARTIFICIAL (decl) = 1;
11065 DECL_IGNORED_P (decl) = 1;
11066 varpool_node::finalize_decl (decl);
11068 splay_tree_insert (critical_name_mutexes, (splay_tree_key) name,
11069 (splay_tree_value) decl);
11071 else
11072 decl = (tree) n->value;
11074 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START);
11075 lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl));
11077 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END);
11078 unlock = build_call_expr_loc (loc, unlock, 1,
11079 build_fold_addr_expr_loc (loc, decl));
11081 else
11083 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START);
11084 lock = build_call_expr_loc (loc, lock, 0);
11086 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END);
11087 unlock = build_call_expr_loc (loc, unlock, 0);
11090 push_gimplify_context ();
11092 block = make_node (BLOCK);
11093 bind = gimple_build_bind (NULL, NULL, block);
11094 gsi_replace (gsi_p, bind, true);
11095 gimple_bind_add_stmt (bind, stmt);
11097 tbody = gimple_bind_body (bind);
11098 gimplify_and_add (lock, &tbody);
11099 gimple_bind_set_body (bind, tbody);
11101 lower_omp (gimple_omp_body_ptr (stmt), ctx);
11102 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt)));
11103 gimple_bind_add_seq (bind, gimple_omp_body (stmt));
11104 gimple_omp_set_body (stmt, NULL);
11106 tbody = gimple_bind_body (bind);
11107 gimplify_and_add (unlock, &tbody);
11108 gimple_bind_set_body (bind, tbody);
11110 gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
11112 pop_gimplify_context (bind);
11113 gimple_bind_append_vars (bind, ctx->block_vars);
11114 BLOCK_VARS (block) = gimple_bind_vars (bind);
11118 /* A subroutine of lower_omp_for. Generate code to emit the predicate
11119 for a lastprivate clause. Given a loop control predicate of (V
11120 cond N2), we gate the clause on (!(V cond N2)). The lowered form
11121 is appended to *DLIST, iterator initialization is appended to
11122 *BODY_P. */
11124 static void
11125 lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
11126 gimple_seq *dlist, struct omp_context *ctx)
11128 tree clauses, cond, vinit;
11129 enum tree_code cond_code;
11130 gimple_seq stmts;
11132 cond_code = fd->loop.cond_code;
11133 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
11135 /* When possible, use a strict equality expression. This can let VRP
11136 type optimizations deduce the value and remove a copy. */
11137 if (tree_fits_shwi_p (fd->loop.step))
11139 HOST_WIDE_INT step = tree_to_shwi (fd->loop.step);
11140 if (step == 1 || step == -1)
11141 cond_code = EQ_EXPR;
11144 cond = build2 (cond_code, boolean_type_node, fd->loop.v, fd->loop.n2);
11146 clauses = gimple_omp_for_clauses (fd->for_stmt);
11147 stmts = NULL;
11148 lower_lastprivate_clauses (clauses, cond, &stmts, ctx);
11149 if (!gimple_seq_empty_p (stmts))
11151 gimple_seq_add_seq (&stmts, *dlist);
11152 *dlist = stmts;
11154 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
11155 vinit = fd->loop.n1;
11156 if (cond_code == EQ_EXPR
11157 && tree_fits_shwi_p (fd->loop.n2)
11158 && ! integer_zerop (fd->loop.n2))
11159 vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
11160 else
11161 vinit = unshare_expr (vinit);
11163 /* Initialize the iterator variable, so that threads that don't execute
11164 any iterations don't execute the lastprivate clauses by accident. */
11165 gimplify_assign (fd->loop.v, vinit, body_p);
11170 /* Lower code for an OpenMP loop directive. */
11172 static void
11173 lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11175 tree *rhs_p, block;
11176 struct omp_for_data fd, *fdp = NULL;
11177 gimple stmt = gsi_stmt (*gsi_p), new_stmt;
11178 gimple_seq omp_for_body, body, dlist;
11179 size_t i;
11181 push_gimplify_context ();
11183 lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx);
11185 block = make_node (BLOCK);
11186 new_stmt = gimple_build_bind (NULL, NULL, block);
11187 /* Replace at gsi right away, so that 'stmt' is no member
11188 of a sequence anymore as we're going to add to to a different
11189 one below. */
11190 gsi_replace (gsi_p, new_stmt, true);
11192 /* Move declaration of temporaries in the loop body before we make
11193 it go away. */
11194 omp_for_body = gimple_omp_body (stmt);
11195 if (!gimple_seq_empty_p (omp_for_body)
11196 && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND)
11198 gimple inner_bind = gimple_seq_first_stmt (omp_for_body);
11199 tree vars = gimple_bind_vars (inner_bind);
11200 gimple_bind_append_vars (new_stmt, vars);
11201 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
11202 keep them on the inner_bind and it's block. */
11203 gimple_bind_set_vars (inner_bind, NULL_TREE);
11204 if (gimple_bind_block (inner_bind))
11205 BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE;
11208 if (gimple_omp_for_combined_into_p (stmt))
11210 gcc_assert (gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_OACC_LOOP);
11212 extract_omp_for_data (stmt, &fd, NULL);
11213 fdp = &fd;
11215 /* We need two temporaries with fd.loop.v type (istart/iend)
11216 and then (fd.collapse - 1) temporaries with the same
11217 type for count2 ... countN-1 vars if not constant. */
11218 size_t count = 2;
11219 tree type = fd.iter_type;
11220 if (fd.collapse > 1
11221 && TREE_CODE (fd.loop.n2) != INTEGER_CST)
11222 count += fd.collapse - 1;
11223 bool parallel_for = gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR;
11224 tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
11225 tree clauses = *pc;
11226 if (parallel_for)
11227 outerc
11228 = find_omp_clause (gimple_omp_parallel_clauses (ctx->outer->stmt),
11229 OMP_CLAUSE__LOOPTEMP_);
11230 for (i = 0; i < count; i++)
11232 tree temp;
11233 if (parallel_for)
11235 gcc_assert (outerc);
11236 temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer);
11237 outerc = find_omp_clause (OMP_CLAUSE_CHAIN (outerc),
11238 OMP_CLAUSE__LOOPTEMP_);
11240 else
11242 temp = create_tmp_var (type, NULL);
11243 insert_decl_map (&ctx->outer->cb, temp, temp);
11245 *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
11246 OMP_CLAUSE_DECL (*pc) = temp;
11247 pc = &OMP_CLAUSE_CHAIN (*pc);
11249 *pc = clauses;
11252 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
11253 dlist = NULL;
11254 body = NULL;
11255 lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
11256 fdp);
11257 gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt));
11259 lower_omp (gimple_omp_body_ptr (stmt), ctx);
11261 /* Lower the header expressions. At this point, we can assume that
11262 the header is of the form:
11264 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
11266 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
11267 using the .omp_data_s mapping, if needed. */
11268 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
11270 rhs_p = gimple_omp_for_initial_ptr (stmt, i);
11271 if (!is_gimple_min_invariant (*rhs_p))
11272 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
11274 rhs_p = gimple_omp_for_final_ptr (stmt, i);
11275 if (!is_gimple_min_invariant (*rhs_p))
11276 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
11278 rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
11279 if (!is_gimple_min_invariant (*rhs_p))
11280 *rhs_p = get_formal_tmp_var (*rhs_p, &body);
11283 /* Once lowered, extract the bounds and clauses. */
11284 extract_omp_for_data (stmt, &fd, NULL);
11286 lower_omp_for_lastprivate (&fd, &body, &dlist, ctx);
11288 gimple_seq_add_stmt (&body, stmt);
11289 gimple_seq_add_seq (&body, gimple_omp_body (stmt));
11291 gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
11292 fd.loop.v));
11294 /* After the loop, add exit clauses. */
11295 lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx);
11297 if (ctx->cancellable)
11298 gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
11300 gimple_seq_add_seq (&body, dlist);
11302 body = maybe_catch_exception (body);
11304 /* Region exit marker goes at the end of the loop body. */
11305 gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait));
11306 maybe_add_implicit_barrier_cancel (ctx, &body);
11307 pop_gimplify_context (new_stmt);
11309 gimple_bind_append_vars (new_stmt, ctx->block_vars);
11310 BLOCK_VARS (block) = gimple_bind_vars (new_stmt);
11311 if (BLOCK_VARS (block))
11312 TREE_USED (block) = 1;
11314 gimple_bind_set_body (new_stmt, body);
11315 gimple_omp_set_body (stmt, NULL);
11316 gimple_omp_for_set_pre_body (stmt, NULL);
11319 /* Callback for walk_stmts. Check if the current statement only contains
11320 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
11322 static tree
11323 check_combined_parallel (gimple_stmt_iterator *gsi_p,
11324 bool *handled_ops_p,
11325 struct walk_stmt_info *wi)
11327 int *info = (int *) wi->info;
11328 gimple stmt = gsi_stmt (*gsi_p);
11330 *handled_ops_p = true;
11331 switch (gimple_code (stmt))
11333 WALK_SUBSTMTS;
11335 case GIMPLE_OMP_FOR:
11336 case GIMPLE_OMP_SECTIONS:
11337 *info = *info == 0 ? 1 : -1;
11338 break;
11339 default:
11340 *info = -1;
11341 break;
11343 return NULL;
11346 struct omp_taskcopy_context
11348 /* This field must be at the beginning, as we do "inheritance": Some
11349 callback functions for tree-inline.c (e.g., omp_copy_decl)
11350 receive a copy_body_data pointer that is up-casted to an
11351 omp_context pointer. */
11352 copy_body_data cb;
11353 omp_context *ctx;
11356 static tree
11357 task_copyfn_copy_decl (tree var, copy_body_data *cb)
11359 struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb;
11361 if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var))
11362 return create_tmp_var (TREE_TYPE (var), NULL);
11364 return var;
11367 static tree
11368 task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type)
11370 tree name, new_fields = NULL, type, f;
11372 type = lang_hooks.types.make_type (RECORD_TYPE);
11373 name = DECL_NAME (TYPE_NAME (orig_type));
11374 name = build_decl (gimple_location (tcctx->ctx->stmt),
11375 TYPE_DECL, name, type);
11376 TYPE_NAME (type) = name;
11378 for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f))
11380 tree new_f = copy_node (f);
11381 DECL_CONTEXT (new_f) = type;
11382 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb);
11383 TREE_CHAIN (new_f) = new_fields;
11384 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL);
11385 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL);
11386 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r,
11387 &tcctx->cb, NULL);
11388 new_fields = new_f;
11389 tcctx->cb.decl_map->put (f, new_f);
11391 TYPE_FIELDS (type) = nreverse (new_fields);
11392 layout_type (type);
11393 return type;
11396 /* Create task copyfn. */
11398 static void
11399 create_task_copyfn (gimple task_stmt, omp_context *ctx)
11401 gcc_assert (!is_gimple_omp_oacc_specifically (ctx->stmt));
11403 struct function *child_cfun;
11404 tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl;
11405 tree record_type, srecord_type, bind, list;
11406 bool record_needs_remap = false, srecord_needs_remap = false;
11407 splay_tree_node n;
11408 struct omp_taskcopy_context tcctx;
11409 location_t loc = gimple_location (task_stmt);
11411 child_fn = gimple_omp_task_copy_fn (task_stmt);
11412 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
11413 gcc_assert (child_cfun->cfg == NULL);
11414 DECL_SAVED_TREE (child_fn) = alloc_stmt_list ();
11416 /* Reset DECL_CONTEXT on function arguments. */
11417 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t))
11418 DECL_CONTEXT (t) = child_fn;
11420 /* Populate the function. */
11421 push_gimplify_context ();
11422 push_cfun (child_cfun);
11424 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
11425 TREE_SIDE_EFFECTS (bind) = 1;
11426 list = NULL;
11427 DECL_SAVED_TREE (child_fn) = bind;
11428 DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt);
11430 /* Remap src and dst argument types if needed. */
11431 record_type = ctx->record_type;
11432 srecord_type = ctx->srecord_type;
11433 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f))
11434 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
11436 record_needs_remap = true;
11437 break;
11439 for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f))
11440 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
11442 srecord_needs_remap = true;
11443 break;
11446 if (record_needs_remap || srecord_needs_remap)
11448 memset (&tcctx, '\0', sizeof (tcctx));
11449 tcctx.cb.src_fn = ctx->cb.src_fn;
11450 tcctx.cb.dst_fn = child_fn;
11451 tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn);
11452 gcc_checking_assert (tcctx.cb.src_node);
11453 tcctx.cb.dst_node = tcctx.cb.src_node;
11454 tcctx.cb.src_cfun = ctx->cb.src_cfun;
11455 tcctx.cb.copy_decl = task_copyfn_copy_decl;
11456 tcctx.cb.eh_lp_nr = 0;
11457 tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE;
11458 tcctx.cb.decl_map = new hash_map<tree, tree>;
11459 tcctx.ctx = ctx;
11461 if (record_needs_remap)
11462 record_type = task_copyfn_remap_type (&tcctx, record_type);
11463 if (srecord_needs_remap)
11464 srecord_type = task_copyfn_remap_type (&tcctx, srecord_type);
11466 else
11467 tcctx.cb.decl_map = NULL;
11469 arg = DECL_ARGUMENTS (child_fn);
11470 TREE_TYPE (arg) = build_pointer_type (record_type);
11471 sarg = DECL_CHAIN (arg);
11472 TREE_TYPE (sarg) = build_pointer_type (srecord_type);
11474 /* First pass: initialize temporaries used in record_type and srecord_type
11475 sizes and field offsets. */
11476 if (tcctx.cb.decl_map)
11477 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
11478 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
11480 tree *p;
11482 decl = OMP_CLAUSE_DECL (c);
11483 p = tcctx.cb.decl_map->get (decl);
11484 if (p == NULL)
11485 continue;
11486 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
11487 sf = (tree) n->value;
11488 sf = *tcctx.cb.decl_map->get (sf);
11489 src = build_simple_mem_ref_loc (loc, sarg);
11490 src = omp_build_component_ref (src, sf);
11491 t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src);
11492 append_to_statement_list (t, &list);
11495 /* Second pass: copy shared var pointers and copy construct non-VLA
11496 firstprivate vars. */
11497 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
11498 switch (OMP_CLAUSE_CODE (c))
11500 case OMP_CLAUSE_SHARED:
11501 decl = OMP_CLAUSE_DECL (c);
11502 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
11503 if (n == NULL)
11504 break;
11505 f = (tree) n->value;
11506 if (tcctx.cb.decl_map)
11507 f = *tcctx.cb.decl_map->get (f);
11508 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
11509 sf = (tree) n->value;
11510 if (tcctx.cb.decl_map)
11511 sf = *tcctx.cb.decl_map->get (sf);
11512 src = build_simple_mem_ref_loc (loc, sarg);
11513 src = omp_build_component_ref (src, sf);
11514 dst = build_simple_mem_ref_loc (loc, arg);
11515 dst = omp_build_component_ref (dst, f);
11516 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
11517 append_to_statement_list (t, &list);
11518 break;
11519 case OMP_CLAUSE_FIRSTPRIVATE:
11520 decl = OMP_CLAUSE_DECL (c);
11521 if (is_variable_sized (decl))
11522 break;
11523 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
11524 if (n == NULL)
11525 break;
11526 f = (tree) n->value;
11527 if (tcctx.cb.decl_map)
11528 f = *tcctx.cb.decl_map->get (f);
11529 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
11530 if (n != NULL)
11532 sf = (tree) n->value;
11533 if (tcctx.cb.decl_map)
11534 sf = *tcctx.cb.decl_map->get (sf);
11535 src = build_simple_mem_ref_loc (loc, sarg);
11536 src = omp_build_component_ref (src, sf);
11537 if (use_pointer_for_field (decl, NULL) || is_reference (decl))
11538 src = build_simple_mem_ref_loc (loc, src);
11540 else
11541 src = decl;
11542 dst = build_simple_mem_ref_loc (loc, arg);
11543 dst = omp_build_component_ref (dst, f);
11544 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
11545 append_to_statement_list (t, &list);
11546 break;
11547 case OMP_CLAUSE_PRIVATE:
11548 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c))
11549 break;
11550 decl = OMP_CLAUSE_DECL (c);
11551 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
11552 f = (tree) n->value;
11553 if (tcctx.cb.decl_map)
11554 f = *tcctx.cb.decl_map->get (f);
11555 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl);
11556 if (n != NULL)
11558 sf = (tree) n->value;
11559 if (tcctx.cb.decl_map)
11560 sf = *tcctx.cb.decl_map->get (sf);
11561 src = build_simple_mem_ref_loc (loc, sarg);
11562 src = omp_build_component_ref (src, sf);
11563 if (use_pointer_for_field (decl, NULL))
11564 src = build_simple_mem_ref_loc (loc, src);
11566 else
11567 src = decl;
11568 dst = build_simple_mem_ref_loc (loc, arg);
11569 dst = omp_build_component_ref (dst, f);
11570 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
11571 append_to_statement_list (t, &list);
11572 break;
11573 default:
11574 break;
11577 /* Last pass: handle VLA firstprivates. */
11578 if (tcctx.cb.decl_map)
11579 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c))
11580 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
11582 tree ind, ptr, df;
11584 decl = OMP_CLAUSE_DECL (c);
11585 if (!is_variable_sized (decl))
11586 continue;
11587 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl);
11588 if (n == NULL)
11589 continue;
11590 f = (tree) n->value;
11591 f = *tcctx.cb.decl_map->get (f);
11592 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl));
11593 ind = DECL_VALUE_EXPR (decl);
11594 gcc_assert (TREE_CODE (ind) == INDIRECT_REF);
11595 gcc_assert (DECL_P (TREE_OPERAND (ind, 0)));
11596 n = splay_tree_lookup (ctx->sfield_map,
11597 (splay_tree_key) TREE_OPERAND (ind, 0));
11598 sf = (tree) n->value;
11599 sf = *tcctx.cb.decl_map->get (sf);
11600 src = build_simple_mem_ref_loc (loc, sarg);
11601 src = omp_build_component_ref (src, sf);
11602 src = build_simple_mem_ref_loc (loc, src);
11603 dst = build_simple_mem_ref_loc (loc, arg);
11604 dst = omp_build_component_ref (dst, f);
11605 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
11606 append_to_statement_list (t, &list);
11607 n = splay_tree_lookup (ctx->field_map,
11608 (splay_tree_key) TREE_OPERAND (ind, 0));
11609 df = (tree) n->value;
11610 df = *tcctx.cb.decl_map->get (df);
11611 ptr = build_simple_mem_ref_loc (loc, arg);
11612 ptr = omp_build_component_ref (ptr, df);
11613 t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr,
11614 build_fold_addr_expr_loc (loc, dst));
11615 append_to_statement_list (t, &list);
11618 t = build1 (RETURN_EXPR, void_type_node, NULL);
11619 append_to_statement_list (t, &list);
11621 if (tcctx.cb.decl_map)
11622 delete tcctx.cb.decl_map;
11623 pop_gimplify_context (NULL);
11624 BIND_EXPR_BODY (bind) = list;
11625 pop_cfun ();
11628 static void
11629 lower_depend_clauses (gimple stmt, gimple_seq *iseq, gimple_seq *oseq)
11631 tree c, clauses;
11632 gimple g;
11633 size_t n_in = 0, n_out = 0, idx = 2, i;
11635 clauses = find_omp_clause (gimple_omp_task_clauses (stmt),
11636 OMP_CLAUSE_DEPEND);
11637 gcc_assert (clauses);
11638 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
11639 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
11640 switch (OMP_CLAUSE_DEPEND_KIND (c))
11642 case OMP_CLAUSE_DEPEND_IN:
11643 n_in++;
11644 break;
11645 case OMP_CLAUSE_DEPEND_OUT:
11646 case OMP_CLAUSE_DEPEND_INOUT:
11647 n_out++;
11648 break;
11649 default:
11650 gcc_unreachable ();
11652 tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2);
11653 tree array = create_tmp_var (type, NULL);
11654 tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
11655 NULL_TREE);
11656 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out));
11657 gimple_seq_add_stmt (iseq, g);
11658 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
11659 NULL_TREE);
11660 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out));
11661 gimple_seq_add_stmt (iseq, g);
11662 for (i = 0; i < 2; i++)
11664 if ((i ? n_in : n_out) == 0)
11665 continue;
11666 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
11667 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
11668 && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i))
11670 tree t = OMP_CLAUSE_DECL (c);
11671 t = fold_convert (ptr_type_node, t);
11672 gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
11673 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++),
11674 NULL_TREE, NULL_TREE);
11675 g = gimple_build_assign (r, t);
11676 gimple_seq_add_stmt (iseq, g);
11679 tree *p = gimple_omp_task_clauses_ptr (stmt);
11680 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
11681 OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
11682 OMP_CLAUSE_CHAIN (c) = *p;
11683 *p = c;
11684 tree clobber = build_constructor (type, NULL);
11685 TREE_THIS_VOLATILE (clobber) = 1;
11686 g = gimple_build_assign (array, clobber);
11687 gimple_seq_add_stmt (oseq, g);
11690 /* Lower the OpenMP parallel or task directive in the current statement
11691 in GSI_P. CTX holds context information for the directive. */
11693 static void
11694 lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11696 tree clauses;
11697 tree child_fn, t;
11698 gimple stmt = gsi_stmt (*gsi_p);
11699 gimple par_bind, bind, dep_bind = NULL;
11700 gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body;
11701 location_t loc = gimple_location (stmt);
11703 clauses = gimple_omp_taskreg_clauses (stmt);
11704 par_bind = gimple_seq_first_stmt (gimple_omp_body (stmt));
11705 par_body = gimple_bind_body (par_bind);
11706 child_fn = ctx->cb.dst_fn;
11707 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
11708 && !gimple_omp_parallel_combined_p (stmt))
11710 struct walk_stmt_info wi;
11711 int ws_num = 0;
11713 memset (&wi, 0, sizeof (wi));
11714 wi.info = &ws_num;
11715 wi.val_only = true;
11716 walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi);
11717 if (ws_num == 1)
11718 gimple_omp_parallel_set_combined_p (stmt, true);
11720 gimple_seq dep_ilist = NULL;
11721 gimple_seq dep_olist = NULL;
11722 if (gimple_code (stmt) == GIMPLE_OMP_TASK
11723 && find_omp_clause (clauses, OMP_CLAUSE_DEPEND))
11725 push_gimplify_context ();
11726 dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
11727 lower_depend_clauses (stmt, &dep_ilist, &dep_olist);
11730 if (ctx->srecord_type)
11731 create_task_copyfn (stmt, ctx);
11733 push_gimplify_context ();
11735 par_olist = NULL;
11736 par_ilist = NULL;
11737 par_rlist = NULL;
11738 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
11739 lower_omp (&par_body, ctx);
11740 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL)
11741 lower_reduction_clauses (clauses, &par_rlist, ctx);
11743 /* Declare all the variables created by mapping and the variables
11744 declared in the scope of the parallel body. */
11745 record_vars_into (ctx->block_vars, child_fn);
11746 record_vars_into (gimple_bind_vars (par_bind), child_fn);
11748 if (ctx->record_type)
11750 ctx->sender_decl
11751 = create_tmp_var (ctx->srecord_type ? ctx->srecord_type
11752 : ctx->record_type, ".omp_data_o");
11753 DECL_NAMELESS (ctx->sender_decl) = 1;
11754 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
11755 gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
11758 olist = NULL;
11759 ilist = NULL;
11760 lower_send_clauses (clauses, &ilist, &olist, ctx);
11761 lower_send_shared_vars (&ilist, &olist, ctx);
11763 if (ctx->record_type)
11765 tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL);
11766 TREE_THIS_VOLATILE (clobber) = 1;
11767 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
11768 clobber));
11771 /* Once all the expansions are done, sequence all the different
11772 fragments inside gimple_omp_body. */
11774 new_body = NULL;
11776 if (ctx->record_type)
11778 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
11779 /* fixup_child_record_type might have changed receiver_decl's type. */
11780 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
11781 gimple_seq_add_stmt (&new_body,
11782 gimple_build_assign (ctx->receiver_decl, t));
11785 gimple_seq_add_seq (&new_body, par_ilist);
11786 gimple_seq_add_seq (&new_body, par_body);
11787 gimple_seq_add_seq (&new_body, par_rlist);
11788 if (ctx->cancellable)
11789 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label));
11790 gimple_seq_add_seq (&new_body, par_olist);
11791 new_body = maybe_catch_exception (new_body);
11792 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
11793 gimple_omp_set_body (stmt, new_body);
11795 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
11796 gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
11797 gimple_bind_add_seq (bind, ilist);
11798 gimple_bind_add_stmt (bind, stmt);
11799 gimple_bind_add_seq (bind, olist);
11801 pop_gimplify_context (NULL);
11803 if (dep_bind)
11805 gimple_bind_add_seq (dep_bind, dep_ilist);
11806 gimple_bind_add_stmt (dep_bind, bind);
11807 gimple_bind_add_seq (dep_bind, dep_olist);
11808 pop_gimplify_context (dep_bind);
11812 /* Lower the GIMPLE_OMP_TARGET in the current statement
11813 in GSI_P. CTX holds context information for the directive. */
11815 static void
11816 lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx)
11818 tree clauses;
11819 tree child_fn, t, c;
11820 gimple stmt = gsi_stmt (*gsi_p);
11821 gimple tgt_bind = NULL, bind;
11822 gimple_seq tgt_body = NULL, olist, ilist, new_body;
11823 location_t loc = gimple_location (stmt);
11824 int kind = gimple_omp_target_kind (stmt);
11825 unsigned int map_cnt = 0;
11827 clauses = gimple_omp_target_clauses (stmt);
11828 if (kind == GF_OMP_TARGET_KIND_REGION)
11830 tgt_bind = gimple_seq_first_stmt (gimple_omp_body (stmt));
11831 tgt_body = gimple_bind_body (tgt_bind);
11833 else if (kind == GF_OMP_TARGET_KIND_DATA
11834 || kind == GF_OMP_TARGET_KIND_OACC_DATA)
11835 tgt_body = gimple_omp_body (stmt);
11836 child_fn = ctx->cb.dst_fn;
11838 push_gimplify_context ();
11840 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
11841 switch (OMP_CLAUSE_CODE (c))
11843 tree var, x;
11845 default:
11846 break;
11847 case OMP_CLAUSE_MAP:
11848 #ifdef ENABLE_CHECKING
11849 /* First check what we're prepared to handle in the following. */
11850 switch (OMP_CLAUSE_MAP_KIND (c))
11852 case OMP_CLAUSE_MAP_ALLOC:
11853 case OMP_CLAUSE_MAP_TO:
11854 case OMP_CLAUSE_MAP_FROM:
11855 case OMP_CLAUSE_MAP_TOFROM:
11856 case OMP_CLAUSE_MAP_POINTER:
11857 case OMP_CLAUSE_MAP_TO_PSET:
11858 break;
11859 case OMP_CLAUSE_MAP_FORCE_ALLOC:
11860 case OMP_CLAUSE_MAP_FORCE_TO:
11861 case OMP_CLAUSE_MAP_FORCE_FROM:
11862 case OMP_CLAUSE_MAP_FORCE_TOFROM:
11863 case OMP_CLAUSE_MAP_FORCE_PRESENT:
11864 case OMP_CLAUSE_MAP_FORCE_DEALLOC:
11865 case OMP_CLAUSE_MAP_FORCE_DEVICEPTR:
11866 gcc_assert (kind == GF_OMP_TARGET_KIND_OACC_DATA
11867 || kind == GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
11868 || kind == GF_OMP_TARGET_KIND_OACC_UPDATE);
11869 break;
11870 default:
11871 gcc_unreachable ();
11873 #endif
11874 /* FALLTHRU */
11876 case OMP_CLAUSE_TO:
11877 case OMP_CLAUSE_FROM:
11878 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP)
11879 gcc_assert (kind != GF_OMP_TARGET_KIND_OACC_DATA
11880 && kind != GF_OMP_TARGET_KIND_OACC_UPDATE);
11881 var = OMP_CLAUSE_DECL (c);
11882 if (!DECL_P (var))
11884 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
11885 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
11886 map_cnt++;
11887 continue;
11890 if (DECL_SIZE (var)
11891 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST)
11893 tree var2 = DECL_VALUE_EXPR (var);
11894 gcc_assert (TREE_CODE (var2) == INDIRECT_REF);
11895 var2 = TREE_OPERAND (var2, 0);
11896 gcc_assert (DECL_P (var2));
11897 var = var2;
11900 if (!maybe_lookup_field (var, ctx))
11901 continue;
11903 if (kind == GF_OMP_TARGET_KIND_REGION)
11905 x = build_receiver_ref (var, true, ctx);
11906 tree new_var = lookup_decl (var, ctx);
11907 gcc_assert (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
11908 || (OMP_CLAUSE_MAP_KIND (c)
11909 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
11910 || TREE_CODE (TREE_TYPE (var)) != ARRAY_TYPE);
11911 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
11912 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
11913 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
11914 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
11915 x = build_simple_mem_ref (x);
11916 SET_DECL_VALUE_EXPR (new_var, x);
11917 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
11919 map_cnt++;
11922 if (kind == GF_OMP_TARGET_KIND_REGION)
11924 target_nesting_level++;
11925 lower_omp (&tgt_body, ctx);
11926 target_nesting_level--;
11928 else if (kind == GF_OMP_TARGET_KIND_DATA
11929 || kind == GF_OMP_TARGET_KIND_OACC_DATA)
11930 lower_omp (&tgt_body, ctx);
11932 if (kind == GF_OMP_TARGET_KIND_REGION)
11934 /* Declare all the variables created by mapping and the variables
11935 declared in the scope of the target body. */
11936 record_vars_into (ctx->block_vars, child_fn);
11937 record_vars_into (gimple_bind_vars (tgt_bind), child_fn);
11940 olist = NULL;
11941 ilist = NULL;
11942 if (ctx->record_type)
11944 ctx->sender_decl
11945 = create_tmp_var (ctx->record_type, ".omp_data_arr");
11946 DECL_NAMELESS (ctx->sender_decl) = 1;
11947 TREE_ADDRESSABLE (ctx->sender_decl) = 1;
11948 t = make_tree_vec (3);
11949 TREE_VEC_ELT (t, 0) = ctx->sender_decl;
11950 TREE_VEC_ELT (t, 1)
11951 = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt),
11952 ".omp_data_sizes");
11953 DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1;
11954 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1;
11955 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1;
11956 tree tkind_type;
11957 int talign_shift;
11958 switch (kind)
11960 case GF_OMP_TARGET_KIND_REGION:
11961 case GF_OMP_TARGET_KIND_DATA:
11962 case GF_OMP_TARGET_KIND_UPDATE:
11963 tkind_type = unsigned_char_type_node;
11964 talign_shift = 3;
11965 break;
11966 case GF_OMP_TARGET_KIND_OACC_DATA:
11967 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
11968 case GF_OMP_TARGET_KIND_OACC_UPDATE:
11969 tkind_type = short_unsigned_type_node;
11970 talign_shift = 8;
11971 break;
11972 default:
11973 gcc_unreachable ();
11975 TREE_VEC_ELT (t, 2)
11976 = create_tmp_var (build_array_type_nelts (tkind_type, map_cnt),
11977 ".omp_data_kinds");
11978 DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1;
11979 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1;
11980 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1;
11981 gimple_omp_target_set_data_arg (stmt, t);
11983 vec<constructor_elt, va_gc> *vsize;
11984 vec<constructor_elt, va_gc> *vkind;
11985 vec_alloc (vsize, map_cnt);
11986 vec_alloc (vkind, map_cnt);
11987 unsigned int map_idx = 0;
11989 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
11990 switch (OMP_CLAUSE_CODE (c))
11992 tree ovar, nc;
11994 default:
11995 break;
11996 case OMP_CLAUSE_MAP:
11997 case OMP_CLAUSE_TO:
11998 case OMP_CLAUSE_FROM:
11999 nc = c;
12000 ovar = OMP_CLAUSE_DECL (c);
12001 if (!DECL_P (ovar))
12003 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
12004 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
12006 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c))
12007 == get_base_address (ovar));
12008 nc = OMP_CLAUSE_CHAIN (c);
12009 ovar = OMP_CLAUSE_DECL (nc);
12011 else
12013 tree x = build_sender_ref (ovar, ctx);
12014 tree v
12015 = build_fold_addr_expr_with_type (ovar, ptr_type_node);
12016 gimplify_assign (x, v, &ilist);
12017 nc = NULL_TREE;
12020 else
12022 if (DECL_SIZE (ovar)
12023 && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST)
12025 tree ovar2 = DECL_VALUE_EXPR (ovar);
12026 gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF);
12027 ovar2 = TREE_OPERAND (ovar2, 0);
12028 gcc_assert (DECL_P (ovar2));
12029 ovar = ovar2;
12031 if (!maybe_lookup_field (ovar, ctx))
12032 continue;
12035 unsigned int talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
12036 if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
12037 talign = DECL_ALIGN_UNIT (ovar);
12038 if (nc)
12040 tree var = lookup_decl_in_outer_ctx (ovar, ctx);
12041 tree x = build_sender_ref (ovar, ctx);
12042 gcc_assert (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP
12043 || (OMP_CLAUSE_MAP_KIND (c)
12044 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
12045 || TREE_CODE (TREE_TYPE (ovar)) != ARRAY_TYPE);
12046 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
12047 && OMP_CLAUSE_MAP_KIND (c) == OMP_CLAUSE_MAP_POINTER
12048 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)
12049 && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE)
12051 tree avar
12052 = create_tmp_var (TREE_TYPE (TREE_TYPE (x)), NULL);
12053 mark_addressable (avar);
12054 gimplify_assign (avar, build_fold_addr_expr (var), &ilist);
12055 talign = DECL_ALIGN_UNIT (avar);
12056 avar = build_fold_addr_expr (avar);
12057 gimplify_assign (x, avar, &ilist);
12059 else if (is_gimple_reg (var))
12061 tree avar = create_tmp_var (TREE_TYPE (var), NULL);
12062 mark_addressable (avar);
12063 enum omp_clause_map_kind map_kind
12064 = OMP_CLAUSE_MAP_KIND (c);
12065 if ((!(map_kind & OMP_CLAUSE_MAP_SPECIAL)
12066 && (map_kind & OMP_CLAUSE_MAP_TO))
12067 || map_kind == OMP_CLAUSE_MAP_POINTER
12068 || map_kind == OMP_CLAUSE_MAP_TO_PSET
12069 || map_kind == OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
12070 gimplify_assign (avar, var, &ilist);
12071 avar = build_fold_addr_expr (avar);
12072 gimplify_assign (x, avar, &ilist);
12073 if (((!(map_kind & OMP_CLAUSE_MAP_SPECIAL)
12074 && (map_kind & OMP_CLAUSE_MAP_FROM))
12075 || map_kind == OMP_CLAUSE_MAP_FORCE_DEVICEPTR)
12076 && !TYPE_READONLY (TREE_TYPE (var)))
12078 x = build_sender_ref (ovar, ctx);
12079 x = build_simple_mem_ref (x);
12080 gimplify_assign (var, x, &olist);
12083 else
12085 var = build_fold_addr_expr (var);
12086 gimplify_assign (x, var, &ilist);
12089 tree s = OMP_CLAUSE_SIZE (c);
12090 if (s == NULL_TREE)
12091 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar));
12092 s = fold_convert (size_type_node, s);
12093 tree purpose = size_int (map_idx++);
12094 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
12095 if (TREE_CODE (s) != INTEGER_CST)
12096 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0;
12098 unsigned HOST_WIDE_INT tkind;
12099 switch (OMP_CLAUSE_CODE (c))
12101 case OMP_CLAUSE_MAP:
12102 tkind = OMP_CLAUSE_MAP_KIND (c);
12103 break;
12104 case OMP_CLAUSE_TO:
12105 tkind = OMP_CLAUSE_MAP_TO;
12106 break;
12107 case OMP_CLAUSE_FROM:
12108 tkind = OMP_CLAUSE_MAP_FROM;
12109 break;
12110 default:
12111 gcc_unreachable ();
12113 gcc_assert (tkind < (HOST_WIDE_INT_C (1U) << talign_shift));
12114 talign = ceil_log2 (talign);
12115 tkind |= talign << talign_shift;
12116 gcc_assert (tkind <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type)));
12117 CONSTRUCTOR_APPEND_ELT (vkind, purpose,
12118 build_int_cstu (tkind_type, tkind));
12119 if (nc && nc != c)
12120 c = nc;
12123 gcc_assert (map_idx == map_cnt);
12125 DECL_INITIAL (TREE_VEC_ELT (t, 1))
12126 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize);
12127 DECL_INITIAL (TREE_VEC_ELT (t, 2))
12128 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind);
12129 if (!TREE_STATIC (TREE_VEC_ELT (t, 1)))
12131 gimple_seq initlist = NULL;
12132 force_gimple_operand (build1 (DECL_EXPR, void_type_node,
12133 TREE_VEC_ELT (t, 1)),
12134 &initlist, true, NULL_TREE);
12135 gimple_seq_add_seq (&ilist, initlist);
12137 tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)),
12138 NULL);
12139 TREE_THIS_VOLATILE (clobber) = 1;
12140 gimple_seq_add_stmt (&olist,
12141 gimple_build_assign (TREE_VEC_ELT (t, 1),
12142 clobber));
12145 tree clobber = build_constructor (ctx->record_type, NULL);
12146 TREE_THIS_VOLATILE (clobber) = 1;
12147 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
12148 clobber));
12151 /* Once all the expansions are done, sequence all the different
12152 fragments inside gimple_omp_body. */
12154 new_body = NULL;
12156 if (ctx->record_type && kind == GF_OMP_TARGET_KIND_REGION)
12158 t = build_fold_addr_expr_loc (loc, ctx->sender_decl);
12159 /* fixup_child_record_type might have changed receiver_decl's type. */
12160 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t);
12161 gimple_seq_add_stmt (&new_body,
12162 gimple_build_assign (ctx->receiver_decl, t));
12165 if (kind == GF_OMP_TARGET_KIND_REGION)
12167 gimple_seq_add_seq (&new_body, tgt_body);
12168 new_body = maybe_catch_exception (new_body);
12170 else if (kind == GF_OMP_TARGET_KIND_DATA
12171 || kind == GF_OMP_TARGET_KIND_OACC_DATA)
12172 new_body = tgt_body;
12173 if (kind != GF_OMP_TARGET_KIND_UPDATE
12174 && kind != GF_OMP_TARGET_KIND_OACC_UPDATE
12175 && kind != GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA)
12177 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
12178 gimple_omp_set_body (stmt, new_body);
12181 bind = gimple_build_bind (NULL, NULL,
12182 tgt_bind ? gimple_bind_block (tgt_bind)
12183 : NULL_TREE);
12184 gsi_replace (gsi_p, bind, true);
12185 gimple_bind_add_seq (bind, ilist);
12186 gimple_bind_add_stmt (bind, stmt);
12187 gimple_bind_add_seq (bind, olist);
12189 pop_gimplify_context (NULL);
12192 /* Expand code for an OpenMP teams directive. */
12194 static void
12195 lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12197 gimple teams_stmt = gsi_stmt (*gsi_p);
12198 push_gimplify_context ();
12200 tree block = make_node (BLOCK);
12201 gimple bind = gimple_build_bind (NULL, NULL, block);
12202 gsi_replace (gsi_p, bind, true);
12203 gimple_seq bind_body = NULL;
12204 gimple_seq dlist = NULL;
12205 gimple_seq olist = NULL;
12207 tree num_teams = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
12208 OMP_CLAUSE_NUM_TEAMS);
12209 if (num_teams == NULL_TREE)
12210 num_teams = build_int_cst (unsigned_type_node, 0);
12211 else
12213 num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams);
12214 num_teams = fold_convert (unsigned_type_node, num_teams);
12215 gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue);
12217 tree thread_limit = find_omp_clause (gimple_omp_teams_clauses (teams_stmt),
12218 OMP_CLAUSE_THREAD_LIMIT);
12219 if (thread_limit == NULL_TREE)
12220 thread_limit = build_int_cst (unsigned_type_node, 0);
12221 else
12223 thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit);
12224 thread_limit = fold_convert (unsigned_type_node, thread_limit);
12225 gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val,
12226 fb_rvalue);
12229 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
12230 &bind_body, &dlist, ctx, NULL);
12231 lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
12232 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx);
12233 gimple_seq_add_stmt (&bind_body, teams_stmt);
12235 location_t loc = gimple_location (teams_stmt);
12236 tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS);
12237 gimple call = gimple_build_call (decl, 2, num_teams, thread_limit);
12238 gimple_set_location (call, loc);
12239 gimple_seq_add_stmt (&bind_body, call);
12241 gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
12242 gimple_omp_set_body (teams_stmt, NULL);
12243 gimple_seq_add_seq (&bind_body, olist);
12244 gimple_seq_add_seq (&bind_body, dlist);
12245 gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
12246 gimple_bind_set_body (bind, bind_body);
12248 pop_gimplify_context (bind);
12250 gimple_bind_append_vars (bind, ctx->block_vars);
12251 BLOCK_VARS (block) = ctx->block_vars;
12252 if (BLOCK_VARS (block))
12253 TREE_USED (block) = 1;
12257 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
12258 regimplified. If DATA is non-NULL, lower_omp_1 is outside
12259 of OpenMP context, but with task_shared_vars set. */
12261 static tree
12262 lower_omp_regimplify_p (tree *tp, int *walk_subtrees,
12263 void *data)
12265 tree t = *tp;
12267 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
12268 if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t))
12269 return t;
12271 if (task_shared_vars
12272 && DECL_P (t)
12273 && bitmap_bit_p (task_shared_vars, DECL_UID (t)))
12274 return t;
12276 /* If a global variable has been privatized, TREE_CONSTANT on
12277 ADDR_EXPR might be wrong. */
12278 if (data == NULL && TREE_CODE (t) == ADDR_EXPR)
12279 recompute_tree_invariant_for_addr_expr (t);
12281 *walk_subtrees = !TYPE_P (t) && !DECL_P (t);
12282 return NULL_TREE;
12285 static void
12286 lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx)
12288 gimple stmt = gsi_stmt (*gsi_p);
12289 struct walk_stmt_info wi;
12291 if (gimple_has_location (stmt))
12292 input_location = gimple_location (stmt);
12294 if (task_shared_vars)
12295 memset (&wi, '\0', sizeof (wi));
12297 /* If we have issued syntax errors, avoid doing any heavy lifting.
12298 Just replace the OpenMP directives with a NOP to avoid
12299 confusing RTL expansion. */
12300 if (seen_error () && is_gimple_omp (stmt))
12302 gsi_replace (gsi_p, gimple_build_nop (), true);
12303 return;
12306 switch (gimple_code (stmt))
12308 case GIMPLE_COND:
12309 if ((ctx || task_shared_vars)
12310 && (walk_tree (gimple_cond_lhs_ptr (stmt), lower_omp_regimplify_p,
12311 ctx ? NULL : &wi, NULL)
12312 || walk_tree (gimple_cond_rhs_ptr (stmt), lower_omp_regimplify_p,
12313 ctx ? NULL : &wi, NULL)))
12314 gimple_regimplify_operands (stmt, gsi_p);
12315 break;
12316 case GIMPLE_CATCH:
12317 lower_omp (gimple_catch_handler_ptr (stmt), ctx);
12318 break;
12319 case GIMPLE_EH_FILTER:
12320 lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx);
12321 break;
12322 case GIMPLE_TRY:
12323 lower_omp (gimple_try_eval_ptr (stmt), ctx);
12324 lower_omp (gimple_try_cleanup_ptr (stmt), ctx);
12325 break;
12326 case GIMPLE_TRANSACTION:
12327 lower_omp (gimple_transaction_body_ptr (stmt), ctx);
12328 break;
12329 case GIMPLE_BIND:
12330 lower_omp (gimple_bind_body_ptr (stmt), ctx);
12331 break;
12332 case GIMPLE_OACC_KERNELS:
12333 case GIMPLE_OACC_PARALLEL:
12334 ctx = maybe_lookup_ctx (stmt);
12335 gcc_assert (ctx);
12336 gcc_assert (!ctx->cancellable);
12337 lower_oacc_offload (gsi_p, ctx);
12338 break;
12339 case GIMPLE_OMP_PARALLEL:
12340 case GIMPLE_OMP_TASK:
12341 ctx = maybe_lookup_ctx (stmt);
12342 gcc_assert (ctx);
12343 if (ctx->cancellable)
12344 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
12345 lower_omp_taskreg (gsi_p, ctx);
12346 break;
12347 case GIMPLE_OMP_FOR:
12348 ctx = maybe_lookup_ctx (stmt);
12349 gcc_assert (ctx);
12350 if (ctx->cancellable)
12351 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
12352 lower_omp_for (gsi_p, ctx);
12353 break;
12354 case GIMPLE_OMP_SECTIONS:
12355 ctx = maybe_lookup_ctx (stmt);
12356 gcc_assert (ctx);
12357 if (ctx->cancellable)
12358 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION);
12359 lower_omp_sections (gsi_p, ctx);
12360 break;
12361 case GIMPLE_OMP_SINGLE:
12362 ctx = maybe_lookup_ctx (stmt);
12363 gcc_assert (ctx);
12364 lower_omp_single (gsi_p, ctx);
12365 break;
12366 case GIMPLE_OMP_MASTER:
12367 ctx = maybe_lookup_ctx (stmt);
12368 gcc_assert (ctx);
12369 lower_omp_master (gsi_p, ctx);
12370 break;
12371 case GIMPLE_OMP_TASKGROUP:
12372 ctx = maybe_lookup_ctx (stmt);
12373 gcc_assert (ctx);
12374 lower_omp_taskgroup (gsi_p, ctx);
12375 break;
12376 case GIMPLE_OMP_ORDERED:
12377 ctx = maybe_lookup_ctx (stmt);
12378 gcc_assert (ctx);
12379 lower_omp_ordered (gsi_p, ctx);
12380 break;
12381 case GIMPLE_OMP_CRITICAL:
12382 ctx = maybe_lookup_ctx (stmt);
12383 gcc_assert (ctx);
12384 lower_omp_critical (gsi_p, ctx);
12385 break;
12386 case GIMPLE_OMP_ATOMIC_LOAD:
12387 if ((ctx || task_shared_vars)
12388 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt),
12389 lower_omp_regimplify_p, ctx ? NULL : &wi, NULL))
12390 gimple_regimplify_operands (stmt, gsi_p);
12391 break;
12392 case GIMPLE_OMP_TARGET:
12393 ctx = maybe_lookup_ctx (stmt);
12394 gcc_assert (ctx);
12395 if (gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_OACC_DATA
12396 || gimple_omp_target_kind (stmt) == GF_OMP_TARGET_KIND_OACC_UPDATE)
12397 gcc_assert (!ctx->cancellable);
12398 lower_omp_target (gsi_p, ctx);
12399 break;
12400 case GIMPLE_OMP_TEAMS:
12401 ctx = maybe_lookup_ctx (stmt);
12402 gcc_assert (ctx);
12403 lower_omp_teams (gsi_p, ctx);
12404 break;
12405 case GIMPLE_CALL:
12406 tree fndecl;
12407 fndecl = gimple_call_fndecl (stmt);
12408 if (fndecl
12409 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
12410 switch (DECL_FUNCTION_CODE (fndecl))
12412 case BUILT_IN_GOMP_BARRIER:
12413 if (ctx == NULL)
12414 break;
12415 /* FALLTHRU */
12416 case BUILT_IN_GOMP_CANCEL:
12417 case BUILT_IN_GOMP_CANCELLATION_POINT:
12418 omp_context *cctx;
12419 cctx = ctx;
12420 if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION)
12421 cctx = cctx->outer;
12422 gcc_assert (gimple_call_lhs (stmt) == NULL_TREE);
12423 if (!cctx->cancellable)
12425 if (DECL_FUNCTION_CODE (fndecl)
12426 == BUILT_IN_GOMP_CANCELLATION_POINT)
12428 stmt = gimple_build_nop ();
12429 gsi_replace (gsi_p, stmt, false);
12431 break;
12433 if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER)
12435 fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL);
12436 gimple_call_set_fndecl (stmt, fndecl);
12437 gimple_call_set_fntype (stmt, TREE_TYPE (fndecl));
12439 tree lhs;
12440 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl)), NULL);
12441 gimple_call_set_lhs (stmt, lhs);
12442 tree fallthru_label;
12443 fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
12444 gimple g;
12445 g = gimple_build_label (fallthru_label);
12446 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
12447 g = gimple_build_cond (NE_EXPR, lhs,
12448 fold_convert (TREE_TYPE (lhs),
12449 boolean_false_node),
12450 cctx->cancel_label, fallthru_label);
12451 gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
12452 break;
12453 default:
12454 break;
12456 /* FALLTHRU */
12457 default:
12458 if ((ctx || task_shared_vars)
12459 && walk_gimple_op (stmt, lower_omp_regimplify_p,
12460 ctx ? NULL : &wi))
12462 /* Just remove clobbers, this should happen only if we have
12463 "privatized" local addressable variables in SIMD regions,
12464 the clobber isn't needed in that case and gimplifying address
12465 of the ARRAY_REF into a pointer and creating MEM_REF based
12466 clobber would create worse code than we get with the clobber
12467 dropped. */
12468 if (gimple_clobber_p (stmt))
12470 gsi_replace (gsi_p, gimple_build_nop (), true);
12471 break;
12473 gimple_regimplify_operands (stmt, gsi_p);
12475 break;
12479 static void
12480 lower_omp (gimple_seq *body, omp_context *ctx)
12482 location_t saved_location = input_location;
12483 gimple_stmt_iterator gsi;
12484 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
12485 lower_omp_1 (&gsi, ctx);
12486 /* During gimplification, we haven't folded statments inside offloading
12487 regions (gimplify.c:maybe_fold_stmt); do that now. */
12488 if (target_nesting_level)
12489 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi))
12490 fold_stmt (&gsi);
12491 input_location = saved_location;
12494 /* Main entry point. */
12496 static unsigned int
12497 execute_lower_omp (void)
12499 gimple_seq body;
12500 int i;
12501 omp_context *ctx;
12503 /* This pass always runs, to provide PROP_gimple_lomp.
12504 But often, there is nothing to do. */
12505 if (flag_openacc == 0 && flag_openmp == 0 && flag_openmp_simd == 0
12506 && flag_cilkplus == 0)
12507 return 0;
12509 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
12510 delete_omp_context);
12512 body = gimple_body (current_function_decl);
12513 scan_omp (&body, NULL);
12514 gcc_assert (taskreg_nesting_level == 0);
12515 FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
12516 finish_taskreg_scan (ctx);
12517 taskreg_contexts.release ();
12519 if (all_contexts->root)
12521 if (task_shared_vars)
12522 push_gimplify_context ();
12523 lower_omp (&body, NULL);
12524 if (task_shared_vars)
12525 pop_gimplify_context (NULL);
12528 if (all_contexts)
12530 splay_tree_delete (all_contexts);
12531 all_contexts = NULL;
12533 BITMAP_FREE (task_shared_vars);
12534 return 0;
12537 namespace {
12539 const pass_data pass_data_lower_omp =
12541 GIMPLE_PASS, /* type */
12542 "omplower", /* name */
12543 OPTGROUP_NONE, /* optinfo_flags */
12544 TV_NONE, /* tv_id */
12545 PROP_gimple_any, /* properties_required */
12546 PROP_gimple_lomp, /* properties_provided */
12547 0, /* properties_destroyed */
12548 0, /* todo_flags_start */
12549 0, /* todo_flags_finish */
12552 class pass_lower_omp : public gimple_opt_pass
12554 public:
12555 pass_lower_omp (gcc::context *ctxt)
12556 : gimple_opt_pass (pass_data_lower_omp, ctxt)
12559 /* opt_pass methods: */
12560 virtual unsigned int execute (function *) { return execute_lower_omp (); }
12562 }; // class pass_lower_omp
12564 } // anon namespace
12566 gimple_opt_pass *
12567 make_pass_lower_omp (gcc::context *ctxt)
12569 return new pass_lower_omp (ctxt);
12572 /* The following is a utility to diagnose structured block violations.
12573 It is not part of the "omplower" pass, as that's invoked too late. It
12574 should be invoked by the respective front ends after gimplification. */
12576 static splay_tree all_labels;
12578 /* Check for mismatched contexts and generate an error if needed. Return
12579 true if an error is detected. */
12581 static bool
12582 diagnose_sb_0 (gimple_stmt_iterator *gsi_p,
12583 gimple branch_ctx, gimple label_ctx)
12585 gcc_assert (!branch_ctx || is_gimple_omp (branch_ctx));
12586 gcc_assert (!label_ctx || is_gimple_omp (label_ctx));
12588 if (label_ctx == branch_ctx)
12589 return false;
12591 const char* kind = NULL;
12593 if (flag_cilkplus)
12595 if ((branch_ctx
12596 && gimple_code (branch_ctx) == GIMPLE_OMP_FOR
12597 && gimple_omp_for_kind (branch_ctx) == GF_OMP_FOR_KIND_CILKSIMD)
12598 || (label_ctx
12599 && gimple_code (label_ctx) == GIMPLE_OMP_FOR
12600 && gimple_omp_for_kind (label_ctx) == GF_OMP_FOR_KIND_CILKSIMD))
12601 kind = "Cilk Plus";
12603 if (flag_openacc)
12605 if ((branch_ctx && is_gimple_omp_oacc_specifically (branch_ctx))
12606 || (label_ctx && is_gimple_omp_oacc_specifically (label_ctx)))
12608 gcc_assert (kind == NULL);
12609 kind = "OpenACC";
12612 if (kind == NULL)
12614 gcc_assert (flag_openmp);
12615 kind = "OpenMP";
12619 Previously we kept track of the label's entire context in diagnose_sb_[12]
12620 so we could traverse it and issue a correct "exit" or "enter" error
12621 message upon a structured block violation.
12623 We built the context by building a list with tree_cons'ing, but there is
12624 no easy counterpart in gimple tuples. It seems like far too much work
12625 for issuing exit/enter error messages. If someone really misses the
12626 distinct error message... patches welcome.
12629 #if 0
12630 /* Try to avoid confusing the user by producing and error message
12631 with correct "exit" or "enter" verbiage. We prefer "exit"
12632 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12633 if (branch_ctx == NULL)
12634 exit_p = false;
12635 else
12637 while (label_ctx)
12639 if (TREE_VALUE (label_ctx) == branch_ctx)
12641 exit_p = false;
12642 break;
12644 label_ctx = TREE_CHAIN (label_ctx);
12648 if (exit_p)
12649 error ("invalid exit from %s structured block", kind);
12650 else
12651 error ("invalid entry to %s structured block", kind);
12652 #endif
12654 /* If it's obvious we have an invalid entry, be specific about the error. */
12655 if (branch_ctx == NULL)
12656 error ("invalid entry to %s structured block", kind);
12657 else
12659 /* Otherwise, be vague and lazy, but efficient. */
12660 error ("invalid branch to/from %s structured block", kind);
12663 gsi_replace (gsi_p, gimple_build_nop (), false);
12664 return true;
12667 /* Pass 1: Create a minimal tree of structured blocks, and record
12668 where each label is found. */
12670 static tree
12671 diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
12672 struct walk_stmt_info *wi)
12674 gimple context = (gimple) wi->info;
12675 gimple inner_context;
12676 gimple stmt = gsi_stmt (*gsi_p);
12678 *handled_ops_p = true;
12680 switch (gimple_code (stmt))
12682 WALK_SUBSTMTS;
12684 case GIMPLE_OACC_KERNELS:
12685 case GIMPLE_OACC_PARALLEL:
12686 case GIMPLE_OMP_PARALLEL:
12687 case GIMPLE_OMP_TASK:
12688 case GIMPLE_OMP_SECTIONS:
12689 case GIMPLE_OMP_SINGLE:
12690 case GIMPLE_OMP_SECTION:
12691 case GIMPLE_OMP_MASTER:
12692 case GIMPLE_OMP_ORDERED:
12693 case GIMPLE_OMP_CRITICAL:
12694 case GIMPLE_OMP_TARGET:
12695 case GIMPLE_OMP_TEAMS:
12696 case GIMPLE_OMP_TASKGROUP:
12697 /* The minimal context here is just the current construct. */
12698 inner_context = stmt;
12699 wi->info = inner_context;
12700 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
12701 wi->info = context;
12702 break;
12704 case GIMPLE_OMP_FOR:
12705 inner_context = stmt;
12706 wi->info = inner_context;
12707 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12708 walk them. */
12709 walk_gimple_seq (gimple_omp_for_pre_body (stmt),
12710 diagnose_sb_1, NULL, wi);
12711 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi);
12712 wi->info = context;
12713 break;
12715 case GIMPLE_LABEL:
12716 splay_tree_insert (all_labels, (splay_tree_key) gimple_label_label (stmt),
12717 (splay_tree_value) context);
12718 break;
12720 default:
12721 break;
12724 return NULL_TREE;
12727 /* Pass 2: Check each branch and see if its context differs from that of
12728 the destination label's context. */
12730 static tree
12731 diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
12732 struct walk_stmt_info *wi)
12734 gimple context = (gimple) wi->info;
12735 splay_tree_node n;
12736 gimple stmt = gsi_stmt (*gsi_p);
12738 *handled_ops_p = true;
12740 switch (gimple_code (stmt))
12742 WALK_SUBSTMTS;
12744 case GIMPLE_OACC_KERNELS:
12745 case GIMPLE_OACC_PARALLEL:
12746 case GIMPLE_OMP_PARALLEL:
12747 case GIMPLE_OMP_TASK:
12748 case GIMPLE_OMP_SECTIONS:
12749 case GIMPLE_OMP_SINGLE:
12750 case GIMPLE_OMP_SECTION:
12751 case GIMPLE_OMP_MASTER:
12752 case GIMPLE_OMP_ORDERED:
12753 case GIMPLE_OMP_CRITICAL:
12754 case GIMPLE_OMP_TARGET:
12755 case GIMPLE_OMP_TEAMS:
12756 case GIMPLE_OMP_TASKGROUP:
12757 wi->info = stmt;
12758 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
12759 wi->info = context;
12760 break;
12762 case GIMPLE_OMP_FOR:
12763 wi->info = stmt;
12764 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12765 walk them. */
12766 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt),
12767 diagnose_sb_2, NULL, wi);
12768 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi);
12769 wi->info = context;
12770 break;
12772 case GIMPLE_COND:
12774 tree lab = gimple_cond_true_label (stmt);
12775 if (lab)
12777 n = splay_tree_lookup (all_labels,
12778 (splay_tree_key) lab);
12779 diagnose_sb_0 (gsi_p, context,
12780 n ? (gimple) n->value : NULL);
12782 lab = gimple_cond_false_label (stmt);
12783 if (lab)
12785 n = splay_tree_lookup (all_labels,
12786 (splay_tree_key) lab);
12787 diagnose_sb_0 (gsi_p, context,
12788 n ? (gimple) n->value : NULL);
12791 break;
12793 case GIMPLE_GOTO:
12795 tree lab = gimple_goto_dest (stmt);
12796 if (TREE_CODE (lab) != LABEL_DECL)
12797 break;
12799 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
12800 diagnose_sb_0 (gsi_p, context, n ? (gimple) n->value : NULL);
12802 break;
12804 case GIMPLE_SWITCH:
12806 unsigned int i;
12807 for (i = 0; i < gimple_switch_num_labels (stmt); ++i)
12809 tree lab = CASE_LABEL (gimple_switch_label (stmt, i));
12810 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
12811 if (n && diagnose_sb_0 (gsi_p, context, (gimple) n->value))
12812 break;
12815 break;
12817 case GIMPLE_RETURN:
12818 diagnose_sb_0 (gsi_p, context, NULL);
12819 break;
12821 default:
12822 break;
12825 return NULL_TREE;
12828 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12829 GIMPLE codes. */
12830 bool
12831 make_gimple_omp_edges (basic_block bb, struct omp_region **region,
12832 int *region_idx)
12834 gimple last = last_stmt (bb);
12835 enum gimple_code code = gimple_code (last);
12836 struct omp_region *cur_region = *region;
12837 bool fallthru = false;
12839 switch (code)
12841 case GIMPLE_OACC_KERNELS:
12842 case GIMPLE_OACC_PARALLEL:
12843 case GIMPLE_OMP_PARALLEL:
12844 case GIMPLE_OMP_TASK:
12845 case GIMPLE_OMP_FOR:
12846 case GIMPLE_OMP_SINGLE:
12847 case GIMPLE_OMP_TEAMS:
12848 case GIMPLE_OMP_MASTER:
12849 case GIMPLE_OMP_TASKGROUP:
12850 case GIMPLE_OMP_ORDERED:
12851 case GIMPLE_OMP_CRITICAL:
12852 case GIMPLE_OMP_SECTION:
12853 cur_region = new_omp_region (bb, code, cur_region);
12854 fallthru = true;
12855 break;
12857 case GIMPLE_OMP_TARGET:
12858 cur_region = new_omp_region (bb, code, cur_region);
12859 fallthru = true;
12860 if (gimple_omp_target_kind (last) == GF_OMP_TARGET_KIND_UPDATE
12861 || gimple_omp_target_kind (last) == GF_OMP_TARGET_KIND_OACC_UPDATE)
12862 cur_region = cur_region->outer;
12863 break;
12865 case GIMPLE_OMP_SECTIONS:
12866 cur_region = new_omp_region (bb, code, cur_region);
12867 fallthru = true;
12868 break;
12870 case GIMPLE_OMP_SECTIONS_SWITCH:
12871 fallthru = false;
12872 break;
12874 case GIMPLE_OMP_ATOMIC_LOAD:
12875 case GIMPLE_OMP_ATOMIC_STORE:
12876 fallthru = true;
12877 break;
12879 case GIMPLE_OMP_RETURN:
12880 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12881 somewhere other than the next block. This will be
12882 created later. */
12883 cur_region->exit = bb;
12884 fallthru = cur_region->type != GIMPLE_OMP_SECTION;
12885 cur_region = cur_region->outer;
12886 break;
12888 case GIMPLE_OMP_CONTINUE:
12889 cur_region->cont = bb;
12890 switch (cur_region->type)
12892 case GIMPLE_OMP_FOR:
12893 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12894 succs edges as abnormal to prevent splitting
12895 them. */
12896 single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL;
12897 /* Make the loopback edge. */
12898 make_edge (bb, single_succ (cur_region->entry),
12899 EDGE_ABNORMAL);
12901 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12902 corresponds to the case that the body of the loop
12903 is not executed at all. */
12904 make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL);
12905 make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL);
12906 fallthru = false;
12907 break;
12909 case GIMPLE_OMP_SECTIONS:
12910 /* Wire up the edges into and out of the nested sections. */
12912 basic_block switch_bb = single_succ (cur_region->entry);
12914 struct omp_region *i;
12915 for (i = cur_region->inner; i ; i = i->next)
12917 gcc_assert (i->type == GIMPLE_OMP_SECTION);
12918 make_edge (switch_bb, i->entry, 0);
12919 make_edge (i->exit, bb, EDGE_FALLTHRU);
12922 /* Make the loopback edge to the block with
12923 GIMPLE_OMP_SECTIONS_SWITCH. */
12924 make_edge (bb, switch_bb, 0);
12926 /* Make the edge from the switch to exit. */
12927 make_edge (switch_bb, bb->next_bb, 0);
12928 fallthru = false;
12930 break;
12932 default:
12933 gcc_unreachable ();
12935 break;
12937 default:
12938 gcc_unreachable ();
12941 if (*region != cur_region)
12943 *region = cur_region;
12944 if (cur_region)
12945 *region_idx = cur_region->entry->index;
12946 else
12947 *region_idx = 0;
12950 return fallthru;
12953 static unsigned int
12954 diagnose_omp_structured_block_errors (void)
12956 struct walk_stmt_info wi;
12957 gimple_seq body = gimple_body (current_function_decl);
12959 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
12961 memset (&wi, 0, sizeof (wi));
12962 walk_gimple_seq (body, diagnose_sb_1, NULL, &wi);
12964 memset (&wi, 0, sizeof (wi));
12965 wi.want_locations = true;
12966 walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi);
12968 gimple_set_body (current_function_decl, body);
12970 splay_tree_delete (all_labels);
12971 all_labels = NULL;
12973 return 0;
12976 namespace {
12978 const pass_data pass_data_diagnose_omp_blocks =
12980 GIMPLE_PASS, /* type */
12981 "*diagnose_omp_blocks", /* name */
12982 OPTGROUP_NONE, /* optinfo_flags */
12983 TV_NONE, /* tv_id */
12984 PROP_gimple_any, /* properties_required */
12985 0, /* properties_provided */
12986 0, /* properties_destroyed */
12987 0, /* todo_flags_start */
12988 0, /* todo_flags_finish */
12991 class pass_diagnose_omp_blocks : public gimple_opt_pass
12993 public:
12994 pass_diagnose_omp_blocks (gcc::context *ctxt)
12995 : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt)
12998 /* opt_pass methods: */
12999 virtual bool gate (function *)
13001 return flag_openacc || flag_openmp || flag_cilkplus;
13003 virtual unsigned int execute (function *)
13005 return diagnose_omp_structured_block_errors ();
13008 }; // class pass_diagnose_omp_blocks
13010 } // anon namespace
13012 gimple_opt_pass *
13013 make_pass_diagnose_omp_blocks (gcc::context *ctxt)
13015 return new pass_diagnose_omp_blocks (ctxt);
13018 /* SIMD clone supporting code. */
13020 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
13021 of arguments to reserve space for. */
13023 static struct cgraph_simd_clone *
13024 simd_clone_struct_alloc (int nargs)
13026 struct cgraph_simd_clone *clone_info;
13027 size_t len = (sizeof (struct cgraph_simd_clone)
13028 + nargs * sizeof (struct cgraph_simd_clone_arg));
13029 clone_info = (struct cgraph_simd_clone *)
13030 ggc_internal_cleared_alloc (len);
13031 return clone_info;
13034 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
13036 static inline void
13037 simd_clone_struct_copy (struct cgraph_simd_clone *to,
13038 struct cgraph_simd_clone *from)
13040 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
13041 + ((from->nargs - from->inbranch)
13042 * sizeof (struct cgraph_simd_clone_arg))));
13045 /* Return vector of parameter types of function FNDECL. This uses
13046 TYPE_ARG_TYPES if available, otherwise falls back to types of
13047 DECL_ARGUMENTS types. */
13049 vec<tree>
13050 simd_clone_vector_of_formal_parm_types (tree fndecl)
13052 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
13053 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
13054 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
13055 unsigned int i;
13056 tree arg;
13057 FOR_EACH_VEC_ELT (args, i, arg)
13058 args[i] = TREE_TYPE (args[i]);
13059 return args;
13062 /* Given a simd function in NODE, extract the simd specific
13063 information from the OMP clauses passed in CLAUSES, and return
13064 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
13065 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
13066 otherwise set to FALSE. */
13068 static struct cgraph_simd_clone *
13069 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
13070 bool *inbranch_specified)
13072 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
13073 tree t;
13074 int n;
13075 *inbranch_specified = false;
13077 n = args.length ();
13078 if (n > 0 && args.last () == void_type_node)
13079 n--;
13081 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
13082 be cloned have a distinctive artificial label in addition to "omp
13083 declare simd". */
13084 bool cilk_clone
13085 = (flag_cilkplus
13086 && lookup_attribute ("cilk simd function",
13087 DECL_ATTRIBUTES (node->decl)));
13089 /* Allocate one more than needed just in case this is an in-branch
13090 clone which will require a mask argument. */
13091 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
13092 clone_info->nargs = n;
13093 clone_info->cilk_elemental = cilk_clone;
13095 if (!clauses)
13097 args.release ();
13098 return clone_info;
13100 clauses = TREE_VALUE (clauses);
13101 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
13102 return clone_info;
13104 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
13106 switch (OMP_CLAUSE_CODE (t))
13108 case OMP_CLAUSE_INBRANCH:
13109 clone_info->inbranch = 1;
13110 *inbranch_specified = true;
13111 break;
13112 case OMP_CLAUSE_NOTINBRANCH:
13113 clone_info->inbranch = 0;
13114 *inbranch_specified = true;
13115 break;
13116 case OMP_CLAUSE_SIMDLEN:
13117 clone_info->simdlen
13118 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
13119 break;
13120 case OMP_CLAUSE_LINEAR:
13122 tree decl = OMP_CLAUSE_DECL (t);
13123 tree step = OMP_CLAUSE_LINEAR_STEP (t);
13124 int argno = TREE_INT_CST_LOW (decl);
13125 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
13127 clone_info->args[argno].arg_type
13128 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
13129 clone_info->args[argno].linear_step = tree_to_shwi (step);
13130 gcc_assert (clone_info->args[argno].linear_step >= 0
13131 && clone_info->args[argno].linear_step < n);
13133 else
13135 if (POINTER_TYPE_P (args[argno]))
13136 step = fold_convert (ssizetype, step);
13137 if (!tree_fits_shwi_p (step))
13139 warning_at (OMP_CLAUSE_LOCATION (t), 0,
13140 "ignoring large linear step");
13141 args.release ();
13142 return NULL;
13144 else if (integer_zerop (step))
13146 warning_at (OMP_CLAUSE_LOCATION (t), 0,
13147 "ignoring zero linear step");
13148 args.release ();
13149 return NULL;
13151 else
13153 clone_info->args[argno].arg_type
13154 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
13155 clone_info->args[argno].linear_step = tree_to_shwi (step);
13158 break;
13160 case OMP_CLAUSE_UNIFORM:
13162 tree decl = OMP_CLAUSE_DECL (t);
13163 int argno = tree_to_uhwi (decl);
13164 clone_info->args[argno].arg_type
13165 = SIMD_CLONE_ARG_TYPE_UNIFORM;
13166 break;
13168 case OMP_CLAUSE_ALIGNED:
13170 tree decl = OMP_CLAUSE_DECL (t);
13171 int argno = tree_to_uhwi (decl);
13172 clone_info->args[argno].alignment
13173 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
13174 break;
13176 default:
13177 break;
13180 args.release ();
13181 return clone_info;
13184 /* Given a SIMD clone in NODE, calculate the characteristic data
13185 type and return the coresponding type. The characteristic data
13186 type is computed as described in the Intel Vector ABI. */
13188 static tree
13189 simd_clone_compute_base_data_type (struct cgraph_node *node,
13190 struct cgraph_simd_clone *clone_info)
13192 tree type = integer_type_node;
13193 tree fndecl = node->decl;
13195 /* a) For non-void function, the characteristic data type is the
13196 return type. */
13197 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
13198 type = TREE_TYPE (TREE_TYPE (fndecl));
13200 /* b) If the function has any non-uniform, non-linear parameters,
13201 then the characteristic data type is the type of the first
13202 such parameter. */
13203 else
13205 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
13206 for (unsigned int i = 0; i < clone_info->nargs; ++i)
13207 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
13209 type = map[i];
13210 break;
13212 map.release ();
13215 /* c) If the characteristic data type determined by a) or b) above
13216 is struct, union, or class type which is pass-by-value (except
13217 for the type that maps to the built-in complex data type), the
13218 characteristic data type is int. */
13219 if (RECORD_OR_UNION_TYPE_P (type)
13220 && !aggregate_value_p (type, NULL)
13221 && TREE_CODE (type) != COMPLEX_TYPE)
13222 return integer_type_node;
13224 /* d) If none of the above three classes is applicable, the
13225 characteristic data type is int. */
13227 return type;
13229 /* e) For Intel Xeon Phi native and offload compilation, if the
13230 resulting characteristic data type is 8-bit or 16-bit integer
13231 data type, the characteristic data type is int. */
13232 /* Well, we don't handle Xeon Phi yet. */
13235 static tree
13236 simd_clone_mangle (struct cgraph_node *node,
13237 struct cgraph_simd_clone *clone_info)
13239 char vecsize_mangle = clone_info->vecsize_mangle;
13240 char mask = clone_info->inbranch ? 'M' : 'N';
13241 unsigned int simdlen = clone_info->simdlen;
13242 unsigned int n;
13243 pretty_printer pp;
13245 gcc_assert (vecsize_mangle && simdlen);
13247 pp_string (&pp, "_ZGV");
13248 pp_character (&pp, vecsize_mangle);
13249 pp_character (&pp, mask);
13250 pp_decimal_int (&pp, simdlen);
13252 for (n = 0; n < clone_info->nargs; ++n)
13254 struct cgraph_simd_clone_arg arg = clone_info->args[n];
13256 if (arg.arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM)
13257 pp_character (&pp, 'u');
13258 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
13260 gcc_assert (arg.linear_step != 0);
13261 pp_character (&pp, 'l');
13262 if (arg.linear_step > 1)
13263 pp_unsigned_wide_integer (&pp, arg.linear_step);
13264 else if (arg.linear_step < 0)
13266 pp_character (&pp, 'n');
13267 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
13268 arg.linear_step));
13271 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
13273 pp_character (&pp, 's');
13274 pp_unsigned_wide_integer (&pp, arg.linear_step);
13276 else
13277 pp_character (&pp, 'v');
13278 if (arg.alignment)
13280 pp_character (&pp, 'a');
13281 pp_decimal_int (&pp, arg.alignment);
13285 pp_underscore (&pp);
13286 pp_string (&pp,
13287 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)));
13288 const char *str = pp_formatted_text (&pp);
13290 /* If there already is a SIMD clone with the same mangled name, don't
13291 add another one. This can happen e.g. for
13292 #pragma omp declare simd
13293 #pragma omp declare simd simdlen(8)
13294 int foo (int, int);
13295 if the simdlen is assumed to be 8 for the first one, etc. */
13296 for (struct cgraph_node *clone = node->simd_clones; clone;
13297 clone = clone->simdclone->next_clone)
13298 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)),
13299 str) == 0)
13300 return NULL_TREE;
13302 return get_identifier (str);
13305 /* Create a simd clone of OLD_NODE and return it. */
13307 static struct cgraph_node *
13308 simd_clone_create (struct cgraph_node *old_node)
13310 struct cgraph_node *new_node;
13311 if (old_node->definition)
13313 if (!old_node->has_gimple_body_p ())
13314 return NULL;
13315 old_node->get_body ();
13316 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
13317 false, NULL, NULL,
13318 "simdclone");
13320 else
13322 tree old_decl = old_node->decl;
13323 tree new_decl = copy_node (old_node->decl);
13324 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
13325 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
13326 SET_DECL_RTL (new_decl, NULL);
13327 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
13328 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
13329 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
13330 symtab->call_cgraph_insertion_hooks (new_node);
13332 if (new_node == NULL)
13333 return new_node;
13335 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
13337 /* The function cgraph_function_versioning () will force the new
13338 symbol local. Undo this, and inherit external visability from
13339 the old node. */
13340 new_node->local.local = old_node->local.local;
13341 new_node->externally_visible = old_node->externally_visible;
13343 return new_node;
13346 /* Adjust the return type of the given function to its appropriate
13347 vector counterpart. Returns a simd array to be used throughout the
13348 function as a return value. */
13350 static tree
13351 simd_clone_adjust_return_type (struct cgraph_node *node)
13353 tree fndecl = node->decl;
13354 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
13355 unsigned int veclen;
13356 tree t;
13358 /* Adjust the function return type. */
13359 if (orig_rettype == void_type_node)
13360 return NULL_TREE;
13361 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
13362 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl)))
13363 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl))))
13364 veclen = node->simdclone->vecsize_int;
13365 else
13366 veclen = node->simdclone->vecsize_float;
13367 veclen /= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl))));
13368 if (veclen > node->simdclone->simdlen)
13369 veclen = node->simdclone->simdlen;
13370 if (veclen == node->simdclone->simdlen)
13371 TREE_TYPE (TREE_TYPE (fndecl))
13372 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl)),
13373 node->simdclone->simdlen);
13374 else
13376 t = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl)), veclen);
13377 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
13378 TREE_TYPE (TREE_TYPE (fndecl)) = t;
13380 if (!node->definition)
13381 return NULL_TREE;
13383 t = DECL_RESULT (fndecl);
13384 /* Adjust the DECL_RESULT. */
13385 gcc_assert (TREE_TYPE (t) != void_type_node);
13386 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
13387 relayout_decl (t);
13389 tree atype = build_array_type_nelts (orig_rettype,
13390 node->simdclone->simdlen);
13391 if (veclen != node->simdclone->simdlen)
13392 return build1 (VIEW_CONVERT_EXPR, atype, t);
13394 /* Set up a SIMD array to use as the return value. */
13395 tree retval = create_tmp_var_raw (atype, "retval");
13396 gimple_add_tmp_var (retval);
13397 return retval;
13400 /* Each vector argument has a corresponding array to be used locally
13401 as part of the eventual loop. Create such temporary array and
13402 return it.
13404 PREFIX is the prefix to be used for the temporary.
13406 TYPE is the inner element type.
13408 SIMDLEN is the number of elements. */
13410 static tree
13411 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
13413 tree atype = build_array_type_nelts (type, simdlen);
13414 tree avar = create_tmp_var_raw (atype, prefix);
13415 gimple_add_tmp_var (avar);
13416 return avar;
13419 /* Modify the function argument types to their corresponding vector
13420 counterparts if appropriate. Also, create one array for each simd
13421 argument to be used locally when using the function arguments as
13422 part of the loop.
13424 NODE is the function whose arguments are to be adjusted.
13426 Returns an adjustment vector that will be filled describing how the
13427 argument types will be adjusted. */
13429 static ipa_parm_adjustment_vec
13430 simd_clone_adjust_argument_types (struct cgraph_node *node)
13432 vec<tree> args;
13433 ipa_parm_adjustment_vec adjustments;
13435 if (node->definition)
13436 args = ipa_get_vector_of_formal_parms (node->decl);
13437 else
13438 args = simd_clone_vector_of_formal_parm_types (node->decl);
13439 adjustments.create (args.length ());
13440 unsigned i, j, veclen;
13441 struct ipa_parm_adjustment adj;
13442 for (i = 0; i < node->simdclone->nargs; ++i)
13444 memset (&adj, 0, sizeof (adj));
13445 tree parm = args[i];
13446 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
13447 adj.base_index = i;
13448 adj.base = parm;
13450 node->simdclone->args[i].orig_arg = node->definition ? parm : NULL_TREE;
13451 node->simdclone->args[i].orig_type = parm_type;
13453 if (node->simdclone->args[i].arg_type != SIMD_CLONE_ARG_TYPE_VECTOR)
13455 /* No adjustment necessary for scalar arguments. */
13456 adj.op = IPA_PARM_OP_COPY;
13458 else
13460 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
13461 veclen = node->simdclone->vecsize_int;
13462 else
13463 veclen = node->simdclone->vecsize_float;
13464 veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type));
13465 if (veclen > node->simdclone->simdlen)
13466 veclen = node->simdclone->simdlen;
13467 adj.arg_prefix = "simd";
13468 adj.type = build_vector_type (parm_type, veclen);
13469 node->simdclone->args[i].vector_type = adj.type;
13470 for (j = veclen; j < node->simdclone->simdlen; j += veclen)
13472 adjustments.safe_push (adj);
13473 if (j == veclen)
13475 memset (&adj, 0, sizeof (adj));
13476 adj.op = IPA_PARM_OP_NEW;
13477 adj.arg_prefix = "simd";
13478 adj.base_index = i;
13479 adj.type = node->simdclone->args[i].vector_type;
13483 if (node->definition)
13484 node->simdclone->args[i].simd_array
13485 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
13486 parm_type, node->simdclone->simdlen);
13488 adjustments.safe_push (adj);
13491 if (node->simdclone->inbranch)
13493 tree base_type
13494 = simd_clone_compute_base_data_type (node->simdclone->origin,
13495 node->simdclone);
13497 memset (&adj, 0, sizeof (adj));
13498 adj.op = IPA_PARM_OP_NEW;
13499 adj.arg_prefix = "mask";
13501 adj.base_index = i;
13502 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
13503 veclen = node->simdclone->vecsize_int;
13504 else
13505 veclen = node->simdclone->vecsize_float;
13506 veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
13507 if (veclen > node->simdclone->simdlen)
13508 veclen = node->simdclone->simdlen;
13509 adj.type = build_vector_type (base_type, veclen);
13510 adjustments.safe_push (adj);
13512 for (j = veclen; j < node->simdclone->simdlen; j += veclen)
13513 adjustments.safe_push (adj);
13515 /* We have previously allocated one extra entry for the mask. Use
13516 it and fill it. */
13517 struct cgraph_simd_clone *sc = node->simdclone;
13518 sc->nargs++;
13519 if (node->definition)
13521 sc->args[i].orig_arg
13522 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
13523 sc->args[i].simd_array
13524 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
13526 sc->args[i].orig_type = base_type;
13527 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
13530 if (node->definition)
13531 ipa_modify_formal_parameters (node->decl, adjustments);
13532 else
13534 tree new_arg_types = NULL_TREE, new_reversed;
13535 bool last_parm_void = false;
13536 if (args.length () > 0 && args.last () == void_type_node)
13537 last_parm_void = true;
13539 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
13540 j = adjustments.length ();
13541 for (i = 0; i < j; i++)
13543 struct ipa_parm_adjustment *adj = &adjustments[i];
13544 tree ptype;
13545 if (adj->op == IPA_PARM_OP_COPY)
13546 ptype = args[adj->base_index];
13547 else
13548 ptype = adj->type;
13549 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
13551 new_reversed = nreverse (new_arg_types);
13552 if (last_parm_void)
13554 if (new_reversed)
13555 TREE_CHAIN (new_arg_types) = void_list_node;
13556 else
13557 new_reversed = void_list_node;
13560 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
13561 TYPE_ARG_TYPES (new_type) = new_reversed;
13562 TREE_TYPE (node->decl) = new_type;
13564 adjustments.release ();
13566 args.release ();
13567 return adjustments;
13570 /* Initialize and copy the function arguments in NODE to their
13571 corresponding local simd arrays. Returns a fresh gimple_seq with
13572 the instruction sequence generated. */
13574 static gimple_seq
13575 simd_clone_init_simd_arrays (struct cgraph_node *node,
13576 ipa_parm_adjustment_vec adjustments)
13578 gimple_seq seq = NULL;
13579 unsigned i = 0, j = 0, k;
13581 for (tree arg = DECL_ARGUMENTS (node->decl);
13582 arg;
13583 arg = DECL_CHAIN (arg), i++, j++)
13585 if (adjustments[j].op == IPA_PARM_OP_COPY)
13586 continue;
13588 node->simdclone->args[i].vector_arg = arg;
13590 tree array = node->simdclone->args[i].simd_array;
13591 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
13593 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
13594 tree ptr = build_fold_addr_expr (array);
13595 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
13596 build_int_cst (ptype, 0));
13597 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
13598 gimplify_and_add (t, &seq);
13600 else
13602 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
13603 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
13604 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
13606 tree ptr = build_fold_addr_expr (array);
13607 int elemsize;
13608 if (k)
13610 arg = DECL_CHAIN (arg);
13611 j++;
13613 elemsize
13614 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg))));
13615 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
13616 build_int_cst (ptype, k * elemsize));
13617 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
13618 gimplify_and_add (t, &seq);
13622 return seq;
13625 /* Callback info for ipa_simd_modify_stmt_ops below. */
13627 struct modify_stmt_info {
13628 ipa_parm_adjustment_vec adjustments;
13629 gimple stmt;
13630 /* True if the parent statement was modified by
13631 ipa_simd_modify_stmt_ops. */
13632 bool modified;
13635 /* Callback for walk_gimple_op.
13637 Adjust operands from a given statement as specified in the
13638 adjustments vector in the callback data. */
13640 static tree
13641 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
13643 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
13644 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
13645 tree *orig_tp = tp;
13646 if (TREE_CODE (*tp) == ADDR_EXPR)
13647 tp = &TREE_OPERAND (*tp, 0);
13648 struct ipa_parm_adjustment *cand = NULL;
13649 if (TREE_CODE (*tp) == PARM_DECL)
13650 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
13651 else
13653 if (TYPE_P (*tp))
13654 *walk_subtrees = 0;
13657 tree repl = NULL_TREE;
13658 if (cand)
13659 repl = unshare_expr (cand->new_decl);
13660 else
13662 if (tp != orig_tp)
13664 *walk_subtrees = 0;
13665 bool modified = info->modified;
13666 info->modified = false;
13667 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
13668 if (!info->modified)
13670 info->modified = modified;
13671 return NULL_TREE;
13673 info->modified = modified;
13674 repl = *tp;
13676 else
13677 return NULL_TREE;
13680 if (tp != orig_tp)
13682 repl = build_fold_addr_expr (repl);
13683 gimple stmt;
13684 if (is_gimple_debug (info->stmt))
13686 tree vexpr = make_node (DEBUG_EXPR_DECL);
13687 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
13688 DECL_ARTIFICIAL (vexpr) = 1;
13689 TREE_TYPE (vexpr) = TREE_TYPE (repl);
13690 DECL_MODE (vexpr) = TYPE_MODE (TREE_TYPE (repl));
13691 repl = vexpr;
13693 else
13695 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl),
13696 NULL), repl);
13697 repl = gimple_assign_lhs (stmt);
13699 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
13700 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
13701 *orig_tp = repl;
13703 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
13705 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
13706 *tp = vce;
13708 else
13709 *tp = repl;
13711 info->modified = true;
13712 return NULL_TREE;
13715 /* Traverse the function body and perform all modifications as
13716 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13717 modified such that the replacement/reduction value will now be an
13718 offset into the corresponding simd_array.
13720 This function will replace all function argument uses with their
13721 corresponding simd array elements, and ajust the return values
13722 accordingly. */
13724 static void
13725 ipa_simd_modify_function_body (struct cgraph_node *node,
13726 ipa_parm_adjustment_vec adjustments,
13727 tree retval_array, tree iter)
13729 basic_block bb;
13730 unsigned int i, j, l;
13732 /* Re-use the adjustments array, but this time use it to replace
13733 every function argument use to an offset into the corresponding
13734 simd_array. */
13735 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
13737 if (!node->simdclone->args[i].vector_arg)
13738 continue;
13740 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
13741 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
13742 adjustments[j].new_decl
13743 = build4 (ARRAY_REF,
13744 basetype,
13745 node->simdclone->args[i].simd_array,
13746 iter,
13747 NULL_TREE, NULL_TREE);
13748 if (adjustments[j].op == IPA_PARM_OP_NONE
13749 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
13750 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
13753 l = adjustments.length ();
13754 for (i = 1; i < num_ssa_names; i++)
13756 tree name = ssa_name (i);
13757 if (name
13758 && SSA_NAME_VAR (name)
13759 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
13761 for (j = 0; j < l; j++)
13762 if (SSA_NAME_VAR (name) == adjustments[j].base
13763 && adjustments[j].new_decl)
13765 tree base_var;
13766 if (adjustments[j].new_ssa_base == NULL_TREE)
13768 base_var
13769 = copy_var_decl (adjustments[j].base,
13770 DECL_NAME (adjustments[j].base),
13771 TREE_TYPE (adjustments[j].base));
13772 adjustments[j].new_ssa_base = base_var;
13774 else
13775 base_var = adjustments[j].new_ssa_base;
13776 if (SSA_NAME_IS_DEFAULT_DEF (name))
13778 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
13779 gimple_stmt_iterator gsi = gsi_after_labels (bb);
13780 tree new_decl = unshare_expr (adjustments[j].new_decl);
13781 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
13782 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
13783 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
13784 gimple stmt = gimple_build_assign (name, new_decl);
13785 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
13787 else
13788 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
13793 struct modify_stmt_info info;
13794 info.adjustments = adjustments;
13796 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
13798 gimple_stmt_iterator gsi;
13800 gsi = gsi_start_bb (bb);
13801 while (!gsi_end_p (gsi))
13803 gimple stmt = gsi_stmt (gsi);
13804 info.stmt = stmt;
13805 struct walk_stmt_info wi;
13807 memset (&wi, 0, sizeof (wi));
13808 info.modified = false;
13809 wi.info = &info;
13810 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
13812 if (gimple_code (stmt) == GIMPLE_RETURN)
13814 tree retval = gimple_return_retval (stmt);
13815 if (!retval)
13817 gsi_remove (&gsi, true);
13818 continue;
13821 /* Replace `return foo' with `retval_array[iter] = foo'. */
13822 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
13823 retval_array, iter, NULL, NULL);
13824 stmt = gimple_build_assign (ref, retval);
13825 gsi_replace (&gsi, stmt, true);
13826 info.modified = true;
13829 if (info.modified)
13831 update_stmt (stmt);
13832 if (maybe_clean_eh_stmt (stmt))
13833 gimple_purge_dead_eh_edges (gimple_bb (stmt));
13835 gsi_next (&gsi);
13840 /* Adjust the argument types in NODE to their appropriate vector
13841 counterparts. */
13843 static void
13844 simd_clone_adjust (struct cgraph_node *node)
13846 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
13848 targetm.simd_clone.adjust (node);
13850 tree retval = simd_clone_adjust_return_type (node);
13851 ipa_parm_adjustment_vec adjustments
13852 = simd_clone_adjust_argument_types (node);
13854 push_gimplify_context ();
13856 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
13858 /* Adjust all uses of vector arguments accordingly. Adjust all
13859 return values accordingly. */
13860 tree iter = create_tmp_var (unsigned_type_node, "iter");
13861 tree iter1 = make_ssa_name (iter, NULL);
13862 tree iter2 = make_ssa_name (iter, NULL);
13863 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
13865 /* Initialize the iteration variable. */
13866 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
13867 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
13868 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
13869 /* Insert the SIMD array and iv initialization at function
13870 entry. */
13871 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
13873 pop_gimplify_context (NULL);
13875 /* Create a new BB right before the original exit BB, to hold the
13876 iteration increment and the condition/branch. */
13877 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
13878 basic_block incr_bb = create_empty_bb (orig_exit);
13879 add_bb_to_loop (incr_bb, body_bb->loop_father);
13880 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13881 flag. Set it now to be a FALLTHRU_EDGE. */
13882 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
13883 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
13884 for (unsigned i = 0;
13885 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
13887 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
13888 redirect_edge_succ (e, incr_bb);
13890 edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
13891 e->probability = REG_BR_PROB_BASE;
13892 gsi = gsi_last_bb (incr_bb);
13893 gimple g = gimple_build_assign_with_ops (PLUS_EXPR, iter2, iter1,
13894 build_int_cst (unsigned_type_node,
13895 1));
13896 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13898 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13899 struct loop *loop = alloc_loop ();
13900 cfun->has_force_vectorize_loops = true;
13901 loop->safelen = node->simdclone->simdlen;
13902 loop->force_vectorize = true;
13903 loop->header = body_bb;
13905 /* Branch around the body if the mask applies. */
13906 if (node->simdclone->inbranch)
13908 gimple_stmt_iterator gsi = gsi_last_bb (loop->header);
13909 tree mask_array
13910 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
13911 tree mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)), NULL);
13912 tree aref = build4 (ARRAY_REF,
13913 TREE_TYPE (TREE_TYPE (mask_array)),
13914 mask_array, iter1,
13915 NULL, NULL);
13916 g = gimple_build_assign (mask, aref);
13917 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13918 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref)));
13919 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
13921 aref = build1 (VIEW_CONVERT_EXPR,
13922 build_nonstandard_integer_type (bitsize, 0), mask);
13923 mask = make_ssa_name (TREE_TYPE (aref), NULL);
13924 g = gimple_build_assign (mask, aref);
13925 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13928 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
13929 NULL, NULL);
13930 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13931 make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
13932 FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE;
13935 /* Generate the condition. */
13936 g = gimple_build_cond (LT_EXPR,
13937 iter2,
13938 build_int_cst (unsigned_type_node,
13939 node->simdclone->simdlen),
13940 NULL, NULL);
13941 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13942 e = split_block (incr_bb, gsi_stmt (gsi));
13943 basic_block latch_bb = e->dest;
13944 basic_block new_exit_bb;
13945 new_exit_bb = split_block (latch_bb, NULL)->dest;
13946 loop->latch = latch_bb;
13948 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
13950 make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
13951 /* The successor of incr_bb is already pointing to latch_bb; just
13952 change the flags.
13953 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13954 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
13956 gimple phi = create_phi_node (iter1, body_bb);
13957 edge preheader_edge = find_edge (entry_bb, body_bb);
13958 edge latch_edge = single_succ_edge (latch_bb);
13959 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
13960 UNKNOWN_LOCATION);
13961 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
13963 /* Generate the new return. */
13964 gsi = gsi_last_bb (new_exit_bb);
13965 if (retval
13966 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
13967 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
13968 retval = TREE_OPERAND (retval, 0);
13969 else if (retval)
13971 retval = build1 (VIEW_CONVERT_EXPR,
13972 TREE_TYPE (TREE_TYPE (node->decl)),
13973 retval);
13974 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
13975 false, GSI_CONTINUE_LINKING);
13977 g = gimple_build_return (retval);
13978 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
13980 /* Handle aligned clauses by replacing default defs of the aligned
13981 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13982 lhs. Handle linear by adding PHIs. */
13983 for (unsigned i = 0; i < node->simdclone->nargs; i++)
13984 if (node->simdclone->args[i].alignment
13985 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
13986 && (node->simdclone->args[i].alignment
13987 & (node->simdclone->args[i].alignment - 1)) == 0
13988 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
13989 == POINTER_TYPE)
13991 unsigned int alignment = node->simdclone->args[i].alignment;
13992 tree orig_arg = node->simdclone->args[i].orig_arg;
13993 tree def = ssa_default_def (cfun, orig_arg);
13994 if (def && !has_zero_uses (def))
13996 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
13997 gimple_seq seq = NULL;
13998 bool need_cvt = false;
13999 gimple call
14000 = gimple_build_call (fn, 2, def, size_int (alignment));
14001 g = call;
14002 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
14003 ptr_type_node))
14004 need_cvt = true;
14005 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg, NULL);
14006 gimple_call_set_lhs (g, t);
14007 gimple_seq_add_stmt_without_update (&seq, g);
14008 if (need_cvt)
14010 t = make_ssa_name (orig_arg, NULL);
14011 g = gimple_build_assign_with_ops (NOP_EXPR, t,
14012 gimple_call_lhs (g),
14013 NULL_TREE);
14014 gimple_seq_add_stmt_without_update (&seq, g);
14016 gsi_insert_seq_on_edge_immediate
14017 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
14019 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
14020 int freq = compute_call_stmt_bb_frequency (current_function_decl,
14021 entry_bb);
14022 node->create_edge (cgraph_node::get_create (fn),
14023 call, entry_bb->count, freq);
14025 imm_use_iterator iter;
14026 use_operand_p use_p;
14027 gimple use_stmt;
14028 tree repl = gimple_get_lhs (g);
14029 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
14030 if (is_gimple_debug (use_stmt) || use_stmt == call)
14031 continue;
14032 else
14033 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
14034 SET_USE (use_p, repl);
14037 else if (node->simdclone->args[i].arg_type
14038 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
14040 tree orig_arg = node->simdclone->args[i].orig_arg;
14041 tree def = ssa_default_def (cfun, orig_arg);
14042 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
14043 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
14044 if (def && !has_zero_uses (def))
14046 iter1 = make_ssa_name (orig_arg, NULL);
14047 iter2 = make_ssa_name (orig_arg, NULL);
14048 phi = create_phi_node (iter1, body_bb);
14049 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
14050 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
14051 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
14052 ? PLUS_EXPR : POINTER_PLUS_EXPR;
14053 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
14054 ? TREE_TYPE (orig_arg) : sizetype;
14055 tree addcst
14056 = build_int_cst (addtype, node->simdclone->args[i].linear_step);
14057 g = gimple_build_assign_with_ops (code, iter2, iter1, addcst);
14058 gsi = gsi_last_bb (incr_bb);
14059 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
14061 imm_use_iterator iter;
14062 use_operand_p use_p;
14063 gimple use_stmt;
14064 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
14065 if (use_stmt == phi)
14066 continue;
14067 else
14068 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
14069 SET_USE (use_p, iter1);
14073 calculate_dominance_info (CDI_DOMINATORS);
14074 add_loop (loop, loop->header->loop_father);
14075 update_ssa (TODO_update_ssa);
14077 pop_cfun ();
14080 /* If the function in NODE is tagged as an elemental SIMD function,
14081 create the appropriate SIMD clones. */
14083 static void
14084 expand_simd_clones (struct cgraph_node *node)
14086 tree attr = lookup_attribute ("omp declare simd",
14087 DECL_ATTRIBUTES (node->decl));
14088 if (attr == NULL_TREE
14089 || node->global.inlined_to
14090 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
14091 return;
14093 /* Ignore
14094 #pragma omp declare simd
14095 extern int foo ();
14096 in C, there we don't know the argument types at all. */
14097 if (!node->definition
14098 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
14099 return;
14103 /* Start with parsing the "omp declare simd" attribute(s). */
14104 bool inbranch_clause_specified;
14105 struct cgraph_simd_clone *clone_info
14106 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
14107 &inbranch_clause_specified);
14108 if (clone_info == NULL)
14109 continue;
14111 int orig_simdlen = clone_info->simdlen;
14112 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
14113 /* The target can return 0 (no simd clones should be created),
14114 1 (just one ISA of simd clones should be created) or higher
14115 count of ISA variants. In that case, clone_info is initialized
14116 for the first ISA variant. */
14117 int count
14118 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
14119 base_type, 0);
14120 if (count == 0)
14121 continue;
14123 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
14124 also create one inbranch and one !inbranch clone of it. */
14125 for (int i = 0; i < count * 2; i++)
14127 struct cgraph_simd_clone *clone = clone_info;
14128 if (inbranch_clause_specified && (i & 1) != 0)
14129 continue;
14131 if (i != 0)
14133 clone = simd_clone_struct_alloc (clone_info->nargs
14134 + ((i & 1) != 0));
14135 simd_clone_struct_copy (clone, clone_info);
14136 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
14137 and simd_clone_adjust_argument_types did to the first
14138 clone's info. */
14139 clone->nargs -= clone_info->inbranch;
14140 clone->simdlen = orig_simdlen;
14141 /* And call the target hook again to get the right ISA. */
14142 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
14143 base_type,
14144 i / 2);
14145 if ((i & 1) != 0)
14146 clone->inbranch = 1;
14149 /* simd_clone_mangle might fail if such a clone has been created
14150 already. */
14151 tree id = simd_clone_mangle (node, clone);
14152 if (id == NULL_TREE)
14153 continue;
14155 /* Only when we are sure we want to create the clone actually
14156 clone the function (or definitions) or create another
14157 extern FUNCTION_DECL (for prototypes without definitions). */
14158 struct cgraph_node *n = simd_clone_create (node);
14159 if (n == NULL)
14160 continue;
14162 n->simdclone = clone;
14163 clone->origin = node;
14164 clone->next_clone = NULL;
14165 if (node->simd_clones == NULL)
14167 clone->prev_clone = n;
14168 node->simd_clones = n;
14170 else
14172 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
14173 clone->prev_clone->simdclone->next_clone = n;
14174 node->simd_clones->simdclone->prev_clone = n;
14176 symtab->change_decl_assembler_name (n->decl, id);
14177 /* And finally adjust the return type, parameters and for
14178 definitions also function body. */
14179 if (node->definition)
14180 simd_clone_adjust (n);
14181 else
14183 simd_clone_adjust_return_type (n);
14184 simd_clone_adjust_argument_types (n);
14188 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
14191 /* Entry point for IPA simd clone creation pass. */
14193 static unsigned int
14194 ipa_omp_simd_clone (void)
14196 struct cgraph_node *node;
14197 FOR_EACH_FUNCTION (node)
14198 expand_simd_clones (node);
14199 return 0;
14202 namespace {
14204 const pass_data pass_data_omp_simd_clone =
14206 SIMPLE_IPA_PASS, /* type */
14207 "simdclone", /* name */
14208 OPTGROUP_NONE, /* optinfo_flags */
14209 TV_NONE, /* tv_id */
14210 ( PROP_ssa | PROP_cfg ), /* properties_required */
14211 0, /* properties_provided */
14212 0, /* properties_destroyed */
14213 0, /* todo_flags_start */
14214 0, /* todo_flags_finish */
14217 class pass_omp_simd_clone : public simple_ipa_opt_pass
14219 public:
14220 pass_omp_simd_clone(gcc::context *ctxt)
14221 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
14224 /* opt_pass methods: */
14225 virtual bool gate (function *);
14226 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
14229 bool
14230 pass_omp_simd_clone::gate (function *)
14232 return ((flag_openmp || flag_openmp_simd
14233 || flag_cilkplus
14234 || (in_lto_p && !flag_wpa))
14235 && (targetm.simd_clone.compute_vecsize_and_simdlen != NULL));
14238 } // anon namespace
14240 simple_ipa_opt_pass *
14241 make_pass_omp_simd_clone (gcc::context *ctxt)
14243 return new pass_omp_simd_clone (ctxt);
14246 /* Helper function for omp_finish_file routine.
14247 Takes decls from V_DECLS and adds their addresses and sizes to
14248 constructor-vector V_CTOR. It will be later used as DECL_INIT for decl
14249 representing a global symbol for OpenMP descriptor. */
14250 static void
14251 add_decls_addresses_to_decl_constructor (vec<tree, va_gc> *v_decls,
14252 vec<constructor_elt, va_gc> *v_ctor)
14254 unsigned len = vec_safe_length (v_decls);
14255 for (unsigned i = 0; i < len; i++)
14257 tree it = (*v_decls)[i];
14258 bool is_function = TREE_CODE (it) != VAR_DECL;
14260 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, build_fold_addr_expr (it));
14261 if (!is_function)
14262 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE,
14263 fold_convert (const_ptr_type_node,
14264 DECL_SIZE (it)));
14268 /* Create new symbol containing (address, size) pairs for omp-marked
14269 functions and global variables. */
14270 void
14271 omp_finish_file (void)
14273 const char *funcs_section_name = OFFLOAD_FUNC_TABLE_SECTION_NAME;
14274 const char *vars_section_name = OFFLOAD_VAR_TABLE_SECTION_NAME;
14276 unsigned num_funcs = vec_safe_length (offload_funcs);
14277 unsigned num_vars = vec_safe_length (offload_vars);
14279 if (num_funcs == 0 && num_vars == 0)
14280 return;
14282 if (targetm_common.have_named_sections)
14284 vec<constructor_elt, va_gc> *v_f, *v_v;
14285 vec_alloc (v_f, num_funcs);
14286 vec_alloc (v_v, num_vars * 2);
14288 add_decls_addresses_to_decl_constructor (offload_funcs, v_f);
14289 add_decls_addresses_to_decl_constructor (offload_vars, v_v);
14291 tree vars_decl_type = build_array_type_nelts (pointer_sized_int_node,
14292 num_vars * 2);
14293 tree funcs_decl_type = build_array_type_nelts (pointer_sized_int_node,
14294 num_funcs);
14295 TYPE_ALIGN (vars_decl_type) = TYPE_ALIGN (pointer_sized_int_node);
14296 TYPE_ALIGN (funcs_decl_type) = TYPE_ALIGN (pointer_sized_int_node);
14297 tree ctor_v = build_constructor (vars_decl_type, v_v);
14298 tree ctor_f = build_constructor (funcs_decl_type, v_f);
14299 TREE_CONSTANT (ctor_v) = TREE_CONSTANT (ctor_f) = 1;
14300 TREE_STATIC (ctor_v) = TREE_STATIC (ctor_f) = 1;
14301 tree funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
14302 get_identifier (".omp_func_table"),
14303 funcs_decl_type);
14304 tree vars_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL,
14305 get_identifier (".omp_var_table"),
14306 vars_decl_type);
14307 TREE_STATIC (funcs_decl) = TREE_STATIC (vars_decl) = 1;
14308 DECL_INITIAL (funcs_decl) = ctor_f;
14309 DECL_INITIAL (vars_decl) = ctor_v;
14310 set_decl_section_name (funcs_decl, funcs_section_name);
14311 set_decl_section_name (vars_decl, vars_section_name);
14313 varpool_node::finalize_decl (funcs_decl);
14314 varpool_node::finalize_decl (vars_decl);
14316 else
14318 for (unsigned i = 0; i < num_funcs; i++)
14320 tree it = (*offload_funcs)[i];
14321 targetm.record_offload_symbol (it);
14323 for (unsigned i = 0; i < num_vars; i++)
14325 tree it = (*offload_vars)[i];
14326 targetm.record_offload_symbol (it);
14331 #include "gt-omp-low.h"