1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2020 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
32 #include "tree-pass.h"
35 #include "pretty-print.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "stor-layout.h"
39 #include "internal-fn.h"
40 #include "gimple-fold.h"
42 #include "gimple-iterator.h"
43 #include "gimplify-me.h"
44 #include "gimple-walk.h"
45 #include "tree-iterator.h"
46 #include "tree-inline.h"
47 #include "langhooks.h"
50 #include "splay-tree.h"
51 #include "omp-general.h"
54 #include "gimple-low.h"
55 #include "alloc-pool.h"
56 #include "symbol-summary.h"
57 #include "tree-nested.h"
59 #include "gomp-constants.h"
60 #include "gimple-pretty-print.h"
61 #include "hsa-common.h"
62 #include "stringpool.h"
65 /* Lowering of OMP parallel and workshare constructs proceeds in two
66 phases. The first phase scans the function looking for OMP statements
67 and then for variables that must be replaced to satisfy data sharing
68 clauses. The second phase expands code for the constructs, as well as
69 re-gimplifying things when variables have been replaced with complex
72 Final code generation is done by pass_expand_omp. The flowgraph is
73 scanned for regions which are then moved to a new
74 function, to be invoked by the thread library, or offloaded. */
76 /* Context structure. Used to store information about each parallel
77 directive in the code. */
81 /* This field must be at the beginning, as we do "inheritance": Some
82 callback functions for tree-inline.c (e.g., omp_copy_decl)
83 receive a copy_body_data pointer that is up-casted to an
84 omp_context pointer. */
87 /* The tree of contexts corresponding to the encountered constructs. */
88 struct omp_context
*outer
;
91 /* Map variables to fields in a structure that allows communication
92 between sending and receiving threads. */
98 /* These are used just by task contexts, if task firstprivate fn is
99 needed. srecord_type is used to communicate from the thread
100 that encountered the task construct to task firstprivate fn,
101 record_type is allocated by GOMP_task, initialized by task firstprivate
102 fn and passed to the task body fn. */
103 splay_tree sfield_map
;
106 /* A chain of variables to add to the top-level block surrounding the
107 construct. In the case of a parallel, this is in the child function. */
110 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
111 barriers should jump to during omplower pass. */
114 /* The sibling GIMPLE_OMP_FOR simd with _simt_ clause or NULL
118 /* For task reductions registered in this context, a vector containing
119 the length of the private copies block (if constant, otherwise NULL)
120 and then offsets (if constant, otherwise NULL) for each entry. */
121 vec
<tree
> task_reductions
;
123 /* A hash map from the reduction clauses to the registered array
125 hash_map
<tree
, unsigned> *task_reduction_map
;
127 /* And a hash map from the lastprivate(conditional:) variables to their
128 corresponding tracking loop iteration variables. */
129 hash_map
<tree
, tree
> *lastprivate_conditional_map
;
131 /* A tree_list of the reduction clauses in this context. */
132 tree local_reduction_clauses
;
134 /* A tree_list of the reduction clauses in outer contexts. */
135 tree outer_reduction_clauses
;
137 /* Nesting depth of this context. Used to beautify error messages re
138 invalid gotos. The outermost ctx is depth 1, with depth 0 being
139 reserved for the main body of the function. */
142 /* True if this parallel directive is nested within another. */
145 /* True if this construct can be cancelled. */
148 /* True if lower_omp_1 should look up lastprivate conditional in parent
150 bool combined_into_simd_safelen1
;
152 /* True if there is nested scan context with inclusive clause. */
155 /* True if there is nested scan context with exclusive clause. */
158 /* True in the second simd loop of for simd with inscan reductions. */
159 bool for_simd_scan_phase
;
161 /* True if there is order(concurrent) clause on the construct. */
162 bool order_concurrent
;
164 /* True if there is bind clause on the construct (i.e. a loop construct). */
168 static splay_tree all_contexts
;
169 static int taskreg_nesting_level
;
170 static int target_nesting_level
;
171 static bitmap task_shared_vars
;
172 static bitmap global_nonaddressable_vars
;
173 static vec
<omp_context
*> taskreg_contexts
;
175 static void scan_omp (gimple_seq
*, omp_context
*);
176 static tree
scan_omp_1_op (tree
*, int *, void *);
178 #define WALK_SUBSTMTS \
182 case GIMPLE_EH_FILTER: \
183 case GIMPLE_TRANSACTION: \
184 /* The sub-statements for these should be walked. */ \
185 *handled_ops_p = false; \
188 /* Return true if CTX corresponds to an OpenACC 'parallel' or 'serial'
192 is_oacc_parallel_or_serial (omp_context
*ctx
)
194 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
195 return ((outer_type
== GIMPLE_OMP_TARGET
)
196 && ((gimple_omp_target_kind (ctx
->stmt
)
197 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
198 || (gimple_omp_target_kind (ctx
->stmt
)
199 == GF_OMP_TARGET_KIND_OACC_SERIAL
)));
202 /* Return true if CTX corresponds to an oacc kernels region. */
205 is_oacc_kernels (omp_context
*ctx
)
207 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
208 return ((outer_type
== GIMPLE_OMP_TARGET
)
209 && (gimple_omp_target_kind (ctx
->stmt
)
210 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
213 /* If DECL is the artificial dummy VAR_DECL created for non-static
214 data member privatization, return the underlying "this" parameter,
215 otherwise return NULL. */
218 omp_member_access_dummy_var (tree decl
)
221 || !DECL_ARTIFICIAL (decl
)
222 || !DECL_IGNORED_P (decl
)
223 || !DECL_HAS_VALUE_EXPR_P (decl
)
224 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
227 tree v
= DECL_VALUE_EXPR (decl
);
228 if (TREE_CODE (v
) != COMPONENT_REF
)
232 switch (TREE_CODE (v
))
238 case POINTER_PLUS_EXPR
:
239 v
= TREE_OPERAND (v
, 0);
242 if (DECL_CONTEXT (v
) == current_function_decl
243 && DECL_ARTIFICIAL (v
)
244 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
252 /* Helper for unshare_and_remap, called through walk_tree. */
255 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
257 tree
*pair
= (tree
*) data
;
260 *tp
= unshare_expr (pair
[1]);
263 else if (IS_TYPE_OR_DECL_P (*tp
))
268 /* Return unshare_expr (X) with all occurrences of FROM
272 unshare_and_remap (tree x
, tree from
, tree to
)
274 tree pair
[2] = { from
, to
};
275 x
= unshare_expr (x
);
276 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
280 /* Convenience function for calling scan_omp_1_op on tree operands. */
283 scan_omp_op (tree
*tp
, omp_context
*ctx
)
285 struct walk_stmt_info wi
;
287 memset (&wi
, 0, sizeof (wi
));
289 wi
.want_locations
= true;
291 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
294 static void lower_omp (gimple_seq
*, omp_context
*);
295 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
296 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
298 /* Return true if CTX is for an omp parallel. */
301 is_parallel_ctx (omp_context
*ctx
)
303 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
307 /* Return true if CTX is for an omp task. */
310 is_task_ctx (omp_context
*ctx
)
312 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
316 /* Return true if CTX is for an omp taskloop. */
319 is_taskloop_ctx (omp_context
*ctx
)
321 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
322 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
326 /* Return true if CTX is for a host omp teams. */
329 is_host_teams_ctx (omp_context
*ctx
)
331 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
332 && gimple_omp_teams_host (as_a
<gomp_teams
*> (ctx
->stmt
));
335 /* Return true if CTX is for an omp parallel or omp task or host omp teams
336 (the last one is strictly not a task region in OpenMP speak, but we
337 need to treat it similarly). */
340 is_taskreg_ctx (omp_context
*ctx
)
342 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
) || is_host_teams_ctx (ctx
);
345 /* Return true if EXPR is variable sized. */
348 is_variable_sized (const_tree expr
)
350 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
353 /* Lookup variables. The "maybe" form
354 allows for the variable form to not have been entered, otherwise we
355 assert that the variable must have been entered. */
358 lookup_decl (tree var
, omp_context
*ctx
)
360 tree
*n
= ctx
->cb
.decl_map
->get (var
);
365 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
367 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
368 return n
? *n
: NULL_TREE
;
372 lookup_field (tree var
, omp_context
*ctx
)
375 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
376 return (tree
) n
->value
;
380 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
383 n
= splay_tree_lookup (ctx
->sfield_map
384 ? ctx
->sfield_map
: ctx
->field_map
, key
);
385 return (tree
) n
->value
;
389 lookup_sfield (tree var
, omp_context
*ctx
)
391 return lookup_sfield ((splay_tree_key
) var
, ctx
);
395 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
398 n
= splay_tree_lookup (ctx
->field_map
, key
);
399 return n
? (tree
) n
->value
: NULL_TREE
;
403 maybe_lookup_field (tree var
, omp_context
*ctx
)
405 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
408 /* Return true if DECL should be copied by pointer. SHARED_CTX is
409 the parallel context if DECL is to be shared. */
412 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
414 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
))
415 || TYPE_ATOMIC (TREE_TYPE (decl
)))
418 /* We can only use copy-in/copy-out semantics for shared variables
419 when we know the value is not accessible from an outer scope. */
422 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
424 /* ??? Trivially accessible from anywhere. But why would we even
425 be passing an address in this case? Should we simply assert
426 this to be false, or should we have a cleanup pass that removes
427 these from the list of mappings? */
428 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
)))
431 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
432 without analyzing the expression whether or not its location
433 is accessible to anyone else. In the case of nested parallel
434 regions it certainly may be. */
435 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
438 /* Do not use copy-in/copy-out for variables that have their
440 if (is_global_var (decl
))
442 /* For file scope vars, track whether we've seen them as
443 non-addressable initially and in that case, keep the same
444 answer for the duration of the pass, even when they are made
445 addressable later on e.g. through reduction expansion. Global
446 variables which weren't addressable before the pass will not
447 have their privatized copies address taken. See PR91216. */
448 if (!TREE_ADDRESSABLE (decl
))
450 if (!global_nonaddressable_vars
)
451 global_nonaddressable_vars
= BITMAP_ALLOC (NULL
);
452 bitmap_set_bit (global_nonaddressable_vars
, DECL_UID (decl
));
454 else if (!global_nonaddressable_vars
455 || !bitmap_bit_p (global_nonaddressable_vars
,
459 else if (TREE_ADDRESSABLE (decl
))
462 /* lower_send_shared_vars only uses copy-in, but not copy-out
464 if (TREE_READONLY (decl
)
465 || ((TREE_CODE (decl
) == RESULT_DECL
466 || TREE_CODE (decl
) == PARM_DECL
)
467 && DECL_BY_REFERENCE (decl
)))
470 /* Disallow copy-in/out in nested parallel if
471 decl is shared in outer parallel, otherwise
472 each thread could store the shared variable
473 in its own copy-in location, making the
474 variable no longer really shared. */
475 if (shared_ctx
->is_nested
)
479 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
480 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
487 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
488 c
; c
= OMP_CLAUSE_CHAIN (c
))
489 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
490 && OMP_CLAUSE_DECL (c
) == decl
)
494 goto maybe_mark_addressable_and_ret
;
498 /* For tasks avoid using copy-in/out. As tasks can be
499 deferred or executed in different thread, when GOMP_task
500 returns, the task hasn't necessarily terminated. */
501 if (is_task_ctx (shared_ctx
))
504 maybe_mark_addressable_and_ret
:
505 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
506 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
508 /* Taking address of OUTER in lower_send_shared_vars
509 might need regimplification of everything that uses the
511 if (!task_shared_vars
)
512 task_shared_vars
= BITMAP_ALLOC (NULL
);
513 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
514 TREE_ADDRESSABLE (outer
) = 1;
523 /* Construct a new automatic decl similar to VAR. */
526 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
528 tree copy
= copy_var_decl (var
, name
, type
);
530 DECL_CONTEXT (copy
) = current_function_decl
;
531 DECL_CHAIN (copy
) = ctx
->block_vars
;
532 /* If VAR is listed in task_shared_vars, it means it wasn't
533 originally addressable and is just because task needs to take
534 it's address. But we don't need to take address of privatizations
536 if (TREE_ADDRESSABLE (var
)
537 && ((task_shared_vars
538 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
539 || (global_nonaddressable_vars
540 && bitmap_bit_p (global_nonaddressable_vars
, DECL_UID (var
)))))
541 TREE_ADDRESSABLE (copy
) = 0;
542 ctx
->block_vars
= copy
;
548 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
550 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
553 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
556 omp_build_component_ref (tree obj
, tree field
)
558 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
559 if (TREE_THIS_VOLATILE (field
))
560 TREE_THIS_VOLATILE (ret
) |= 1;
561 if (TREE_READONLY (field
))
562 TREE_READONLY (ret
) |= 1;
566 /* Build tree nodes to access the field for VAR on the receiver side. */
569 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
571 tree x
, field
= lookup_field (var
, ctx
);
573 /* If the receiver record type was remapped in the child function,
574 remap the field into the new record type. */
575 x
= maybe_lookup_field (field
, ctx
);
579 x
= build_simple_mem_ref (ctx
->receiver_decl
);
580 TREE_THIS_NOTRAP (x
) = 1;
581 x
= omp_build_component_ref (x
, field
);
584 x
= build_simple_mem_ref (x
);
585 TREE_THIS_NOTRAP (x
) = 1;
591 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
592 of a parallel, this is a component reference; for workshare constructs
593 this is some variable. */
596 build_outer_var_ref (tree var
, omp_context
*ctx
,
597 enum omp_clause_code code
= OMP_CLAUSE_ERROR
)
600 omp_context
*outer
= ctx
->outer
;
601 while (outer
&& gimple_code (outer
->stmt
) == GIMPLE_OMP_TASKGROUP
)
602 outer
= outer
->outer
;
604 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
606 else if (is_variable_sized (var
))
608 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
609 x
= build_outer_var_ref (x
, ctx
, code
);
610 x
= build_simple_mem_ref (x
);
612 else if (is_taskreg_ctx (ctx
))
614 bool by_ref
= use_pointer_for_field (var
, NULL
);
615 x
= build_receiver_ref (var
, by_ref
, ctx
);
617 else if ((gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
618 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
620 || (code
== OMP_CLAUSE_PRIVATE
621 && (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
622 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
623 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_SINGLE
)))
625 /* #pragma omp simd isn't a worksharing construct, and can reference
626 even private vars in its linear etc. clauses.
627 Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer
628 to private vars in all worksharing constructs. */
630 if (outer
&& is_taskreg_ctx (outer
))
631 x
= lookup_decl (var
, outer
);
633 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
637 else if (code
== OMP_CLAUSE_LASTPRIVATE
&& is_taskloop_ctx (ctx
))
641 = splay_tree_lookup (outer
->field_map
,
642 (splay_tree_key
) &DECL_UID (var
));
645 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, outer
)))
648 x
= lookup_decl (var
, outer
);
652 tree field
= (tree
) n
->value
;
653 /* If the receiver record type was remapped in the child function,
654 remap the field into the new record type. */
655 x
= maybe_lookup_field (field
, outer
);
659 x
= build_simple_mem_ref (outer
->receiver_decl
);
660 x
= omp_build_component_ref (x
, field
);
661 if (use_pointer_for_field (var
, outer
))
662 x
= build_simple_mem_ref (x
);
667 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_GRID_BODY
)
669 outer
= outer
->outer
;
671 && gimple_code (outer
->stmt
) != GIMPLE_OMP_GRID_BODY
);
673 x
= lookup_decl (var
, outer
);
675 else if (omp_is_reference (var
))
676 /* This can happen with orphaned constructs. If var is reference, it is
677 possible it is shared and as such valid. */
679 else if (omp_member_access_dummy_var (var
))
686 tree t
= omp_member_access_dummy_var (var
);
689 x
= DECL_VALUE_EXPR (var
);
690 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
692 x
= unshare_and_remap (x
, t
, o
);
694 x
= unshare_expr (x
);
698 if (omp_is_reference (var
))
699 x
= build_simple_mem_ref (x
);
704 /* Build tree nodes to access the field for VAR on the sender side. */
707 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
709 tree field
= lookup_sfield (key
, ctx
);
710 return omp_build_component_ref (ctx
->sender_decl
, field
);
714 build_sender_ref (tree var
, omp_context
*ctx
)
716 return build_sender_ref ((splay_tree_key
) var
, ctx
);
719 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. If
720 BASE_POINTERS_RESTRICT, declare the field with restrict. */
723 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
725 tree field
, type
, sfield
= NULL_TREE
;
726 splay_tree_key key
= (splay_tree_key
) var
;
728 if ((mask
& 16) != 0)
730 key
= (splay_tree_key
) &DECL_NAME (var
);
731 gcc_checking_assert (key
!= (splay_tree_key
) var
);
735 key
= (splay_tree_key
) &DECL_UID (var
);
736 gcc_checking_assert (key
!= (splay_tree_key
) var
);
738 gcc_assert ((mask
& 1) == 0
739 || !splay_tree_lookup (ctx
->field_map
, key
));
740 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
741 || !splay_tree_lookup (ctx
->sfield_map
, key
));
742 gcc_assert ((mask
& 3) == 3
743 || !is_gimple_omp_oacc (ctx
->stmt
));
745 type
= TREE_TYPE (var
);
746 if ((mask
& 16) != 0)
747 type
= lang_hooks
.decls
.omp_array_data (var
, true);
749 /* Prevent redeclaring the var in the split-off function with a restrict
750 pointer type. Note that we only clear type itself, restrict qualifiers in
751 the pointed-to type will be ignored by points-to analysis. */
752 if (POINTER_TYPE_P (type
)
753 && TYPE_RESTRICT (type
))
754 type
= build_qualified_type (type
, TYPE_QUALS (type
) & ~TYPE_QUAL_RESTRICT
);
758 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
759 type
= build_pointer_type (build_pointer_type (type
));
762 type
= build_pointer_type (type
);
763 else if ((mask
& 3) == 1 && omp_is_reference (var
))
764 type
= TREE_TYPE (type
);
766 field
= build_decl (DECL_SOURCE_LOCATION (var
),
767 FIELD_DECL
, DECL_NAME (var
), type
);
769 /* Remember what variable this field was created for. This does have a
770 side effect of making dwarf2out ignore this member, so for helpful
771 debugging we clear it later in delete_omp_context. */
772 DECL_ABSTRACT_ORIGIN (field
) = var
;
773 if ((mask
& 16) == 0 && type
== TREE_TYPE (var
))
775 SET_DECL_ALIGN (field
, DECL_ALIGN (var
));
776 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
777 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
780 SET_DECL_ALIGN (field
, TYPE_ALIGN (type
));
784 insert_field_into_struct (ctx
->record_type
, field
);
785 if (ctx
->srecord_type
)
787 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
788 FIELD_DECL
, DECL_NAME (var
), type
);
789 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
790 SET_DECL_ALIGN (sfield
, DECL_ALIGN (field
));
791 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
792 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
793 insert_field_into_struct (ctx
->srecord_type
, sfield
);
798 if (ctx
->srecord_type
== NULL_TREE
)
802 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
803 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
804 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
806 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
807 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
808 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
809 insert_field_into_struct (ctx
->srecord_type
, sfield
);
810 splay_tree_insert (ctx
->sfield_map
,
811 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
812 (splay_tree_value
) sfield
);
816 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
817 : ctx
->srecord_type
, field
);
821 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
822 if ((mask
& 2) && ctx
->sfield_map
)
823 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
827 install_var_local (tree var
, omp_context
*ctx
)
829 tree new_var
= omp_copy_decl_1 (var
, ctx
);
830 insert_decl_map (&ctx
->cb
, var
, new_var
);
834 /* Adjust the replacement for DECL in CTX for the new context. This means
835 copying the DECL_VALUE_EXPR, and fixing up the type. */
838 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
842 new_decl
= lookup_decl (decl
, ctx
);
844 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
846 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
847 && DECL_HAS_VALUE_EXPR_P (decl
))
849 tree ve
= DECL_VALUE_EXPR (decl
);
850 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
851 SET_DECL_VALUE_EXPR (new_decl
, ve
);
852 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
855 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
857 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
858 if (size
== error_mark_node
)
859 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
860 DECL_SIZE (new_decl
) = size
;
862 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
863 if (size
== error_mark_node
)
864 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
865 DECL_SIZE_UNIT (new_decl
) = size
;
869 /* The callback for remap_decl. Search all containing contexts for a
870 mapping of the variable; this avoids having to duplicate the splay
871 tree ahead of time. We know a mapping doesn't already exist in the
872 given context. Create new mappings to implement default semantics. */
875 omp_copy_decl (tree var
, copy_body_data
*cb
)
877 omp_context
*ctx
= (omp_context
*) cb
;
880 if (TREE_CODE (var
) == LABEL_DECL
)
882 if (FORCED_LABEL (var
) || DECL_NONLOCAL (var
))
884 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
885 DECL_CONTEXT (new_var
) = current_function_decl
;
886 insert_decl_map (&ctx
->cb
, var
, new_var
);
890 while (!is_taskreg_ctx (ctx
))
895 new_var
= maybe_lookup_decl (var
, ctx
);
900 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
903 return error_mark_node
;
906 /* Create a new context, with OUTER_CTX being the surrounding context. */
909 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
911 omp_context
*ctx
= XCNEW (omp_context
);
913 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
914 (splay_tree_value
) ctx
);
919 ctx
->outer
= outer_ctx
;
920 ctx
->cb
= outer_ctx
->cb
;
921 ctx
->cb
.block
= NULL
;
922 ctx
->local_reduction_clauses
= NULL
;
923 ctx
->outer_reduction_clauses
= ctx
->outer_reduction_clauses
;
924 ctx
->depth
= outer_ctx
->depth
+ 1;
928 ctx
->cb
.src_fn
= current_function_decl
;
929 ctx
->cb
.dst_fn
= current_function_decl
;
930 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
931 gcc_checking_assert (ctx
->cb
.src_node
);
932 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
933 ctx
->cb
.src_cfun
= cfun
;
934 ctx
->cb
.copy_decl
= omp_copy_decl
;
935 ctx
->cb
.eh_lp_nr
= 0;
936 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
937 ctx
->cb
.adjust_array_error_bounds
= true;
938 ctx
->cb
.dont_remap_vla_if_no_change
= true;
939 ctx
->local_reduction_clauses
= NULL
;
940 ctx
->outer_reduction_clauses
= NULL
;
944 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
949 static gimple_seq
maybe_catch_exception (gimple_seq
);
951 /* Finalize task copyfn. */
954 finalize_task_copyfn (gomp_task
*task_stmt
)
956 struct function
*child_cfun
;
958 gimple_seq seq
= NULL
, new_seq
;
961 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
962 if (child_fn
== NULL_TREE
)
965 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
966 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
968 push_cfun (child_cfun
);
969 bind
= gimplify_body (child_fn
, false);
970 gimple_seq_add_stmt (&seq
, bind
);
971 new_seq
= maybe_catch_exception (seq
);
974 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
976 gimple_seq_add_stmt (&seq
, bind
);
978 gimple_set_body (child_fn
, seq
);
981 /* Inform the callgraph about the new function. */
982 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
983 node
->parallelized_function
= 1;
984 cgraph_node::add_new_function (child_fn
, false);
987 /* Destroy a omp_context data structures. Called through the splay tree
988 value delete callback. */
991 delete_omp_context (splay_tree_value value
)
993 omp_context
*ctx
= (omp_context
*) value
;
995 delete ctx
->cb
.decl_map
;
998 splay_tree_delete (ctx
->field_map
);
1000 splay_tree_delete (ctx
->sfield_map
);
1002 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1003 it produces corrupt debug information. */
1004 if (ctx
->record_type
)
1007 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1008 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1010 if (ctx
->srecord_type
)
1013 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1014 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1017 if (is_task_ctx (ctx
))
1018 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1020 if (ctx
->task_reduction_map
)
1022 ctx
->task_reductions
.release ();
1023 delete ctx
->task_reduction_map
;
1026 delete ctx
->lastprivate_conditional_map
;
1031 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1035 fixup_child_record_type (omp_context
*ctx
)
1037 tree f
, type
= ctx
->record_type
;
1039 if (!ctx
->receiver_decl
)
1041 /* ??? It isn't sufficient to just call remap_type here, because
1042 variably_modified_type_p doesn't work the way we expect for
1043 record types. Testing each field for whether it needs remapping
1044 and creating a new record by hand works, however. */
1045 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1046 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1050 tree name
, new_fields
= NULL
;
1052 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1053 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1054 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1055 TYPE_DECL
, name
, type
);
1056 TYPE_NAME (type
) = name
;
1058 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1060 tree new_f
= copy_node (f
);
1061 DECL_CONTEXT (new_f
) = type
;
1062 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1063 DECL_CHAIN (new_f
) = new_fields
;
1064 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1065 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1067 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1071 /* Arrange to be able to look up the receiver field
1072 given the sender field. */
1073 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1074 (splay_tree_value
) new_f
);
1076 TYPE_FIELDS (type
) = nreverse (new_fields
);
1080 /* In a target region we never modify any of the pointers in *.omp_data_i,
1081 so attempt to help the optimizers. */
1082 if (is_gimple_omp_offloaded (ctx
->stmt
))
1083 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1085 TREE_TYPE (ctx
->receiver_decl
)
1086 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1089 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1090 specified by CLAUSES. */
1093 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1096 bool scan_array_reductions
= false;
1098 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1102 switch (OMP_CLAUSE_CODE (c
))
1104 case OMP_CLAUSE_PRIVATE
:
1105 decl
= OMP_CLAUSE_DECL (c
);
1106 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1108 else if (!is_variable_sized (decl
))
1109 install_var_local (decl
, ctx
);
1112 case OMP_CLAUSE_SHARED
:
1113 decl
= OMP_CLAUSE_DECL (c
);
1114 /* Ignore shared directives in teams construct inside of
1115 target construct. */
1116 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
1117 && !is_host_teams_ctx (ctx
))
1119 /* Global variables don't need to be copied,
1120 the receiver side will use them directly. */
1121 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1122 if (is_global_var (odecl
))
1124 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1127 gcc_assert (is_taskreg_ctx (ctx
));
1128 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1129 || !is_variable_sized (decl
));
1130 /* Global variables don't need to be copied,
1131 the receiver side will use them directly. */
1132 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1134 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1136 use_pointer_for_field (decl
, ctx
);
1139 by_ref
= use_pointer_for_field (decl
, NULL
);
1140 if ((! TREE_READONLY (decl
) && !OMP_CLAUSE_SHARED_READONLY (c
))
1141 || TREE_ADDRESSABLE (decl
)
1143 || omp_is_reference (decl
))
1145 by_ref
= use_pointer_for_field (decl
, ctx
);
1146 install_var_field (decl
, by_ref
, 3, ctx
);
1147 install_var_local (decl
, ctx
);
1150 /* We don't need to copy const scalar vars back. */
1151 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1154 case OMP_CLAUSE_REDUCTION
:
1155 if (is_oacc_parallel_or_serial (ctx
) || is_oacc_kernels (ctx
))
1156 ctx
->local_reduction_clauses
1157 = tree_cons (NULL
, c
, ctx
->local_reduction_clauses
);
1160 case OMP_CLAUSE_IN_REDUCTION
:
1161 decl
= OMP_CLAUSE_DECL (c
);
1162 if (TREE_CODE (decl
) == MEM_REF
)
1164 tree t
= TREE_OPERAND (decl
, 0);
1165 if (TREE_CODE (t
) == POINTER_PLUS_EXPR
)
1166 t
= TREE_OPERAND (t
, 0);
1167 if (TREE_CODE (t
) == INDIRECT_REF
1168 || TREE_CODE (t
) == ADDR_EXPR
)
1169 t
= TREE_OPERAND (t
, 0);
1170 install_var_local (t
, ctx
);
1171 if (is_taskreg_ctx (ctx
)
1172 && (!is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1173 || (is_task_ctx (ctx
)
1174 && (TREE_CODE (TREE_TYPE (t
)) == POINTER_TYPE
1175 || (TREE_CODE (TREE_TYPE (t
)) == REFERENCE_TYPE
1176 && (TREE_CODE (TREE_TYPE (TREE_TYPE (t
)))
1177 == POINTER_TYPE
)))))
1178 && !is_variable_sized (t
)
1179 && (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
1180 || (!OMP_CLAUSE_REDUCTION_TASK (c
)
1181 && !is_task_ctx (ctx
))))
1183 by_ref
= use_pointer_for_field (t
, NULL
);
1184 if (is_task_ctx (ctx
)
1185 && TREE_CODE (TREE_TYPE (t
)) == REFERENCE_TYPE
1186 && TREE_CODE (TREE_TYPE (TREE_TYPE (t
))) == POINTER_TYPE
)
1188 install_var_field (t
, false, 1, ctx
);
1189 install_var_field (t
, by_ref
, 2, ctx
);
1192 install_var_field (t
, by_ref
, 3, ctx
);
1196 if (is_task_ctx (ctx
)
1197 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1198 && OMP_CLAUSE_REDUCTION_TASK (c
)
1199 && is_parallel_ctx (ctx
)))
1201 /* Global variables don't need to be copied,
1202 the receiver side will use them directly. */
1203 if (!is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1205 by_ref
= use_pointer_for_field (decl
, ctx
);
1206 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IN_REDUCTION
)
1207 install_var_field (decl
, by_ref
, 3, ctx
);
1209 install_var_local (decl
, ctx
);
1212 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1213 && OMP_CLAUSE_REDUCTION_TASK (c
))
1215 install_var_local (decl
, ctx
);
1220 case OMP_CLAUSE_LASTPRIVATE
:
1221 /* Let the corresponding firstprivate clause create
1223 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1227 case OMP_CLAUSE_FIRSTPRIVATE
:
1228 case OMP_CLAUSE_LINEAR
:
1229 decl
= OMP_CLAUSE_DECL (c
);
1231 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1232 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1233 && is_gimple_omp_offloaded (ctx
->stmt
))
1235 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1236 install_var_field (decl
, !omp_is_reference (decl
), 3, ctx
);
1237 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1238 install_var_field (decl
, true, 3, ctx
);
1240 install_var_field (decl
, false, 3, ctx
);
1242 if (is_variable_sized (decl
))
1244 if (is_task_ctx (ctx
))
1245 install_var_field (decl
, false, 1, ctx
);
1248 else if (is_taskreg_ctx (ctx
))
1251 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1252 by_ref
= use_pointer_for_field (decl
, NULL
);
1254 if (is_task_ctx (ctx
)
1255 && (global
|| by_ref
|| omp_is_reference (decl
)))
1257 install_var_field (decl
, false, 1, ctx
);
1259 install_var_field (decl
, by_ref
, 2, ctx
);
1262 install_var_field (decl
, by_ref
, 3, ctx
);
1264 install_var_local (decl
, ctx
);
1267 case OMP_CLAUSE_USE_DEVICE_PTR
:
1268 case OMP_CLAUSE_USE_DEVICE_ADDR
:
1269 decl
= OMP_CLAUSE_DECL (c
);
1271 /* Fortran array descriptors. */
1272 if (lang_hooks
.decls
.omp_array_data (decl
, true))
1273 install_var_field (decl
, false, 19, ctx
);
1274 else if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_ADDR
1275 && !omp_is_reference (decl
)
1276 && !omp_is_allocatable_or_ptr (decl
))
1277 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1278 install_var_field (decl
, true, 11, ctx
);
1280 install_var_field (decl
, false, 11, ctx
);
1281 if (DECL_SIZE (decl
)
1282 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1284 tree decl2
= DECL_VALUE_EXPR (decl
);
1285 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1286 decl2
= TREE_OPERAND (decl2
, 0);
1287 gcc_assert (DECL_P (decl2
));
1288 install_var_local (decl2
, ctx
);
1290 install_var_local (decl
, ctx
);
1293 case OMP_CLAUSE_IS_DEVICE_PTR
:
1294 decl
= OMP_CLAUSE_DECL (c
);
1297 case OMP_CLAUSE__LOOPTEMP_
:
1298 case OMP_CLAUSE__REDUCTEMP_
:
1299 gcc_assert (is_taskreg_ctx (ctx
));
1300 decl
= OMP_CLAUSE_DECL (c
);
1301 install_var_field (decl
, false, 3, ctx
);
1302 install_var_local (decl
, ctx
);
1305 case OMP_CLAUSE_COPYPRIVATE
:
1306 case OMP_CLAUSE_COPYIN
:
1307 decl
= OMP_CLAUSE_DECL (c
);
1308 by_ref
= use_pointer_for_field (decl
, NULL
);
1309 install_var_field (decl
, by_ref
, 3, ctx
);
1312 case OMP_CLAUSE_FINAL
:
1314 case OMP_CLAUSE_NUM_THREADS
:
1315 case OMP_CLAUSE_NUM_TEAMS
:
1316 case OMP_CLAUSE_THREAD_LIMIT
:
1317 case OMP_CLAUSE_DEVICE
:
1318 case OMP_CLAUSE_SCHEDULE
:
1319 case OMP_CLAUSE_DIST_SCHEDULE
:
1320 case OMP_CLAUSE_DEPEND
:
1321 case OMP_CLAUSE_PRIORITY
:
1322 case OMP_CLAUSE_GRAINSIZE
:
1323 case OMP_CLAUSE_NUM_TASKS
:
1324 case OMP_CLAUSE_NUM_GANGS
:
1325 case OMP_CLAUSE_NUM_WORKERS
:
1326 case OMP_CLAUSE_VECTOR_LENGTH
:
1328 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1332 case OMP_CLAUSE_FROM
:
1333 case OMP_CLAUSE_MAP
:
1335 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1336 decl
= OMP_CLAUSE_DECL (c
);
1337 /* Global variables with "omp declare target" attribute
1338 don't need to be copied, the receiver side will use them
1339 directly. However, global variables with "omp declare target link"
1340 attribute need to be copied. Or when ALWAYS modifier is used. */
1341 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1343 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
1344 && (OMP_CLAUSE_MAP_KIND (c
)
1345 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
1346 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1347 && OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_ALWAYS_TO
1348 && OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_ALWAYS_FROM
1349 && OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_ALWAYS_TOFROM
1350 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1351 && varpool_node::get_create (decl
)->offloadable
1352 && !lookup_attribute ("omp declare target link",
1353 DECL_ATTRIBUTES (decl
)))
1355 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1356 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
1358 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1359 not offloaded; there is nothing to map for those. */
1360 if (!is_gimple_omp_offloaded (ctx
->stmt
)
1361 && !POINTER_TYPE_P (TREE_TYPE (decl
))
1362 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
1365 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1366 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
1367 || (OMP_CLAUSE_MAP_KIND (c
)
1368 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
1370 if (TREE_CODE (decl
) == COMPONENT_REF
1371 || (TREE_CODE (decl
) == INDIRECT_REF
1372 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
1373 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
1374 == REFERENCE_TYPE
)))
1376 if (DECL_SIZE (decl
)
1377 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1379 tree decl2
= DECL_VALUE_EXPR (decl
);
1380 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1381 decl2
= TREE_OPERAND (decl2
, 0);
1382 gcc_assert (DECL_P (decl2
));
1383 install_var_local (decl2
, ctx
);
1385 install_var_local (decl
, ctx
);
1390 if (DECL_SIZE (decl
)
1391 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1393 tree decl2
= DECL_VALUE_EXPR (decl
);
1394 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1395 decl2
= TREE_OPERAND (decl2
, 0);
1396 gcc_assert (DECL_P (decl2
));
1397 install_var_field (decl2
, true, 3, ctx
);
1398 install_var_local (decl2
, ctx
);
1399 install_var_local (decl
, ctx
);
1403 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1404 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1405 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1406 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1407 install_var_field (decl
, true, 7, ctx
);
1409 install_var_field (decl
, true, 3, ctx
);
1410 if (is_gimple_omp_offloaded (ctx
->stmt
)
1411 && !OMP_CLAUSE_MAP_IN_REDUCTION (c
))
1412 install_var_local (decl
, ctx
);
1417 tree base
= get_base_address (decl
);
1418 tree nc
= OMP_CLAUSE_CHAIN (c
);
1421 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1422 && OMP_CLAUSE_DECL (nc
) == base
1423 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
1424 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1426 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1427 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1433 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1434 decl
= OMP_CLAUSE_DECL (c
);
1436 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1437 (splay_tree_key
) decl
));
1439 = build_decl (OMP_CLAUSE_LOCATION (c
),
1440 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1441 SET_DECL_ALIGN (field
, TYPE_ALIGN (ptr_type_node
));
1442 insert_field_into_struct (ctx
->record_type
, field
);
1443 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1444 (splay_tree_value
) field
);
1449 case OMP_CLAUSE__GRIDDIM_
:
1452 scan_omp_op (&OMP_CLAUSE__GRIDDIM__SIZE (c
), ctx
->outer
);
1453 scan_omp_op (&OMP_CLAUSE__GRIDDIM__GROUP (c
), ctx
->outer
);
1457 case OMP_CLAUSE_ORDER
:
1458 ctx
->order_concurrent
= true;
1461 case OMP_CLAUSE_BIND
:
1465 case OMP_CLAUSE_NOWAIT
:
1466 case OMP_CLAUSE_ORDERED
:
1467 case OMP_CLAUSE_COLLAPSE
:
1468 case OMP_CLAUSE_UNTIED
:
1469 case OMP_CLAUSE_MERGEABLE
:
1470 case OMP_CLAUSE_PROC_BIND
:
1471 case OMP_CLAUSE_SAFELEN
:
1472 case OMP_CLAUSE_SIMDLEN
:
1473 case OMP_CLAUSE_THREADS
:
1474 case OMP_CLAUSE_SIMD
:
1475 case OMP_CLAUSE_NOGROUP
:
1476 case OMP_CLAUSE_DEFAULTMAP
:
1477 case OMP_CLAUSE_ASYNC
:
1478 case OMP_CLAUSE_WAIT
:
1479 case OMP_CLAUSE_GANG
:
1480 case OMP_CLAUSE_WORKER
:
1481 case OMP_CLAUSE_VECTOR
:
1482 case OMP_CLAUSE_INDEPENDENT
:
1483 case OMP_CLAUSE_AUTO
:
1484 case OMP_CLAUSE_SEQ
:
1485 case OMP_CLAUSE_TILE
:
1486 case OMP_CLAUSE__SIMT_
:
1487 case OMP_CLAUSE_DEFAULT
:
1488 case OMP_CLAUSE_NONTEMPORAL
:
1489 case OMP_CLAUSE_IF_PRESENT
:
1490 case OMP_CLAUSE_FINALIZE
:
1491 case OMP_CLAUSE_TASK_REDUCTION
:
1494 case OMP_CLAUSE_ALIGNED
:
1495 decl
= OMP_CLAUSE_DECL (c
);
1496 if (is_global_var (decl
)
1497 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1498 install_var_local (decl
, ctx
);
1501 case OMP_CLAUSE__CONDTEMP_
:
1502 decl
= OMP_CLAUSE_DECL (c
);
1503 if (is_parallel_ctx (ctx
))
1505 install_var_field (decl
, false, 3, ctx
);
1506 install_var_local (decl
, ctx
);
1508 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1509 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
1510 && !OMP_CLAUSE__CONDTEMP__ITER (c
))
1511 install_var_local (decl
, ctx
);
1514 case OMP_CLAUSE__CACHE_
:
1520 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1522 switch (OMP_CLAUSE_CODE (c
))
1524 case OMP_CLAUSE_LASTPRIVATE
:
1525 /* Let the corresponding firstprivate clause create
1527 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1528 scan_array_reductions
= true;
1529 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1533 case OMP_CLAUSE_FIRSTPRIVATE
:
1534 case OMP_CLAUSE_PRIVATE
:
1535 case OMP_CLAUSE_LINEAR
:
1536 case OMP_CLAUSE_IS_DEVICE_PTR
:
1537 decl
= OMP_CLAUSE_DECL (c
);
1538 if (is_variable_sized (decl
))
1540 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1541 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1542 && is_gimple_omp_offloaded (ctx
->stmt
))
1544 tree decl2
= DECL_VALUE_EXPR (decl
);
1545 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1546 decl2
= TREE_OPERAND (decl2
, 0);
1547 gcc_assert (DECL_P (decl2
));
1548 install_var_local (decl2
, ctx
);
1549 fixup_remapped_decl (decl2
, ctx
, false);
1551 install_var_local (decl
, ctx
);
1553 fixup_remapped_decl (decl
, ctx
,
1554 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1555 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1556 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1557 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1558 scan_array_reductions
= true;
1561 case OMP_CLAUSE_REDUCTION
:
1562 case OMP_CLAUSE_IN_REDUCTION
:
1563 decl
= OMP_CLAUSE_DECL (c
);
1564 if (TREE_CODE (decl
) != MEM_REF
)
1566 if (is_variable_sized (decl
))
1567 install_var_local (decl
, ctx
);
1568 fixup_remapped_decl (decl
, ctx
, false);
1570 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1571 scan_array_reductions
= true;
1574 case OMP_CLAUSE_TASK_REDUCTION
:
1575 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1576 scan_array_reductions
= true;
1579 case OMP_CLAUSE_SHARED
:
1580 /* Ignore shared directives in teams construct inside of
1581 target construct. */
1582 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
1583 && !is_host_teams_ctx (ctx
))
1585 decl
= OMP_CLAUSE_DECL (c
);
1586 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1588 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1590 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
1593 bool by_ref
= use_pointer_for_field (decl
, ctx
);
1594 install_var_field (decl
, by_ref
, 11, ctx
);
1597 fixup_remapped_decl (decl
, ctx
, false);
1600 case OMP_CLAUSE_MAP
:
1601 if (!is_gimple_omp_offloaded (ctx
->stmt
))
1603 decl
= OMP_CLAUSE_DECL (c
);
1605 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
1606 && (OMP_CLAUSE_MAP_KIND (c
)
1607 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
1608 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1609 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1610 && varpool_node::get_create (decl
)->offloadable
)
1614 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1615 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
1616 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1617 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1619 tree new_decl
= lookup_decl (decl
, ctx
);
1620 TREE_TYPE (new_decl
)
1621 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1623 else if (DECL_SIZE (decl
)
1624 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1626 tree decl2
= DECL_VALUE_EXPR (decl
);
1627 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1628 decl2
= TREE_OPERAND (decl2
, 0);
1629 gcc_assert (DECL_P (decl2
));
1630 fixup_remapped_decl (decl2
, ctx
, false);
1631 fixup_remapped_decl (decl
, ctx
, true);
1634 fixup_remapped_decl (decl
, ctx
, false);
1638 case OMP_CLAUSE_COPYPRIVATE
:
1639 case OMP_CLAUSE_COPYIN
:
1640 case OMP_CLAUSE_DEFAULT
:
1642 case OMP_CLAUSE_NUM_THREADS
:
1643 case OMP_CLAUSE_NUM_TEAMS
:
1644 case OMP_CLAUSE_THREAD_LIMIT
:
1645 case OMP_CLAUSE_DEVICE
:
1646 case OMP_CLAUSE_SCHEDULE
:
1647 case OMP_CLAUSE_DIST_SCHEDULE
:
1648 case OMP_CLAUSE_NOWAIT
:
1649 case OMP_CLAUSE_ORDERED
:
1650 case OMP_CLAUSE_COLLAPSE
:
1651 case OMP_CLAUSE_UNTIED
:
1652 case OMP_CLAUSE_FINAL
:
1653 case OMP_CLAUSE_MERGEABLE
:
1654 case OMP_CLAUSE_PROC_BIND
:
1655 case OMP_CLAUSE_SAFELEN
:
1656 case OMP_CLAUSE_SIMDLEN
:
1657 case OMP_CLAUSE_ALIGNED
:
1658 case OMP_CLAUSE_DEPEND
:
1659 case OMP_CLAUSE__LOOPTEMP_
:
1660 case OMP_CLAUSE__REDUCTEMP_
:
1662 case OMP_CLAUSE_FROM
:
1663 case OMP_CLAUSE_PRIORITY
:
1664 case OMP_CLAUSE_GRAINSIZE
:
1665 case OMP_CLAUSE_NUM_TASKS
:
1666 case OMP_CLAUSE_THREADS
:
1667 case OMP_CLAUSE_SIMD
:
1668 case OMP_CLAUSE_NOGROUP
:
1669 case OMP_CLAUSE_DEFAULTMAP
:
1670 case OMP_CLAUSE_ORDER
:
1671 case OMP_CLAUSE_BIND
:
1672 case OMP_CLAUSE_USE_DEVICE_PTR
:
1673 case OMP_CLAUSE_USE_DEVICE_ADDR
:
1674 case OMP_CLAUSE_NONTEMPORAL
:
1675 case OMP_CLAUSE_ASYNC
:
1676 case OMP_CLAUSE_WAIT
:
1677 case OMP_CLAUSE_NUM_GANGS
:
1678 case OMP_CLAUSE_NUM_WORKERS
:
1679 case OMP_CLAUSE_VECTOR_LENGTH
:
1680 case OMP_CLAUSE_GANG
:
1681 case OMP_CLAUSE_WORKER
:
1682 case OMP_CLAUSE_VECTOR
:
1683 case OMP_CLAUSE_INDEPENDENT
:
1684 case OMP_CLAUSE_AUTO
:
1685 case OMP_CLAUSE_SEQ
:
1686 case OMP_CLAUSE_TILE
:
1687 case OMP_CLAUSE__GRIDDIM_
:
1688 case OMP_CLAUSE__SIMT_
:
1689 case OMP_CLAUSE_IF_PRESENT
:
1690 case OMP_CLAUSE_FINALIZE
:
1691 case OMP_CLAUSE__CONDTEMP_
:
1694 case OMP_CLAUSE__CACHE_
:
1700 gcc_checking_assert (!scan_array_reductions
1701 || !is_gimple_omp_oacc (ctx
->stmt
));
1702 if (scan_array_reductions
)
1704 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1705 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1706 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IN_REDUCTION
1707 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_TASK_REDUCTION
)
1708 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1710 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1711 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1713 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1714 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1715 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1716 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1717 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1718 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
1722 /* Create a new name for omp child function. Returns an identifier. */
1725 create_omp_child_function_name (bool task_copy
)
1727 return clone_function_name_numbered (current_function_decl
,
1728 task_copy
? "_omp_cpyfn" : "_omp_fn");
1731 /* Return true if CTX may belong to offloaded code: either if current function
1732 is offloaded, or any enclosing context corresponds to a target region. */
1735 omp_maybe_offloaded_ctx (omp_context
*ctx
)
1737 if (cgraph_node::get (current_function_decl
)->offloadable
)
1739 for (; ctx
; ctx
= ctx
->outer
)
1740 if (is_gimple_omp_offloaded (ctx
->stmt
))
1745 /* Build a decl for the omp child function. It'll not contain a body
1746 yet, just the bare decl. */
1749 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1751 tree decl
, type
, name
, t
;
1753 name
= create_omp_child_function_name (task_copy
);
1755 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1756 ptr_type_node
, NULL_TREE
);
1758 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1760 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
1762 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
1765 ctx
->cb
.dst_fn
= decl
;
1767 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1769 TREE_STATIC (decl
) = 1;
1770 TREE_USED (decl
) = 1;
1771 DECL_ARTIFICIAL (decl
) = 1;
1772 DECL_IGNORED_P (decl
) = 0;
1773 TREE_PUBLIC (decl
) = 0;
1774 DECL_UNINLINABLE (decl
) = 1;
1775 DECL_EXTERNAL (decl
) = 0;
1776 DECL_CONTEXT (decl
) = NULL_TREE
;
1777 DECL_INITIAL (decl
) = make_node (BLOCK
);
1778 BLOCK_SUPERCONTEXT (DECL_INITIAL (decl
)) = decl
;
1779 DECL_ATTRIBUTES (decl
) = DECL_ATTRIBUTES (current_function_decl
);
1780 /* Remove omp declare simd attribute from the new attributes. */
1781 if (tree a
= lookup_attribute ("omp declare simd", DECL_ATTRIBUTES (decl
)))
1783 while (tree a2
= lookup_attribute ("omp declare simd", TREE_CHAIN (a
)))
1786 for (tree
*p
= &DECL_ATTRIBUTES (decl
); *p
!= a
;)
1787 if (is_attribute_p ("omp declare simd", get_attribute_name (*p
)))
1788 *p
= TREE_CHAIN (*p
);
1791 tree chain
= TREE_CHAIN (*p
);
1792 *p
= copy_node (*p
);
1793 p
= &TREE_CHAIN (*p
);
1797 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (decl
)
1798 = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (current_function_decl
);
1799 DECL_FUNCTION_SPECIFIC_TARGET (decl
)
1800 = DECL_FUNCTION_SPECIFIC_TARGET (current_function_decl
);
1801 DECL_FUNCTION_VERSIONED (decl
)
1802 = DECL_FUNCTION_VERSIONED (current_function_decl
);
1804 if (omp_maybe_offloaded_ctx (ctx
))
1806 cgraph_node::get_create (decl
)->offloadable
= 1;
1807 if (ENABLE_OFFLOADING
)
1808 g
->have_offload
= true;
1811 if (cgraph_node::get_create (decl
)->offloadable
1812 && !lookup_attribute ("omp declare target",
1813 DECL_ATTRIBUTES (current_function_decl
)))
1815 const char *target_attr
= (is_gimple_omp_offloaded (ctx
->stmt
)
1816 ? "omp target entrypoint"
1817 : "omp declare target");
1818 DECL_ATTRIBUTES (decl
)
1819 = tree_cons (get_identifier (target_attr
),
1820 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1823 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1824 RESULT_DECL
, NULL_TREE
, void_type_node
);
1825 DECL_ARTIFICIAL (t
) = 1;
1826 DECL_IGNORED_P (t
) = 1;
1827 DECL_CONTEXT (t
) = decl
;
1828 DECL_RESULT (decl
) = t
;
1830 tree data_name
= get_identifier (".omp_data_i");
1831 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
1833 DECL_ARTIFICIAL (t
) = 1;
1834 DECL_NAMELESS (t
) = 1;
1835 DECL_ARG_TYPE (t
) = ptr_type_node
;
1836 DECL_CONTEXT (t
) = current_function_decl
;
1838 TREE_READONLY (t
) = 1;
1839 DECL_ARGUMENTS (decl
) = t
;
1841 ctx
->receiver_decl
= t
;
1844 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1845 PARM_DECL
, get_identifier (".omp_data_o"),
1847 DECL_ARTIFICIAL (t
) = 1;
1848 DECL_NAMELESS (t
) = 1;
1849 DECL_ARG_TYPE (t
) = ptr_type_node
;
1850 DECL_CONTEXT (t
) = current_function_decl
;
1852 TREE_ADDRESSABLE (t
) = 1;
1853 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1854 DECL_ARGUMENTS (decl
) = t
;
1857 /* Allocate memory for the function structure. The call to
1858 allocate_struct_function clobbers CFUN, so we need to restore
1860 push_struct_function (decl
);
1861 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1862 init_tree_ssa (cfun
);
1866 /* Callback for walk_gimple_seq. Check if combined parallel
1867 contains gimple_omp_for_combined_into_p OMP_FOR. */
1870 omp_find_combined_for (gimple_stmt_iterator
*gsi_p
,
1871 bool *handled_ops_p
,
1872 struct walk_stmt_info
*wi
)
1874 gimple
*stmt
= gsi_stmt (*gsi_p
);
1876 *handled_ops_p
= true;
1877 switch (gimple_code (stmt
))
1881 case GIMPLE_OMP_FOR
:
1882 if (gimple_omp_for_combined_into_p (stmt
)
1883 && gimple_omp_for_kind (stmt
)
1884 == *(const enum gf_mask
*) (wi
->info
))
1887 return integer_zero_node
;
1896 /* Add _LOOPTEMP_/_REDUCTEMP_ clauses on OpenMP parallel or task. */
1899 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
1900 omp_context
*outer_ctx
)
1902 struct walk_stmt_info wi
;
1904 memset (&wi
, 0, sizeof (wi
));
1906 wi
.info
= (void *) &msk
;
1907 walk_gimple_seq (gimple_omp_body (stmt
), omp_find_combined_for
, NULL
, &wi
);
1908 if (wi
.info
!= (void *) &msk
)
1910 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
1911 struct omp_for_data fd
;
1912 omp_extract_for_data (for_stmt
, &fd
, NULL
);
1913 /* We need two temporaries with fd.loop.v type (istart/iend)
1914 and then (fd.collapse - 1) temporaries with the same
1915 type for count2 ... countN-1 vars if not constant. */
1916 size_t count
= 2, i
;
1917 tree type
= fd
.iter_type
;
1919 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
1921 count
+= fd
.collapse
- 1;
1922 /* If there are lastprivate clauses on the inner
1923 GIMPLE_OMP_FOR, add one more temporaries for the total number
1924 of iterations (product of count1 ... countN-1). */
1925 if (omp_find_clause (gimple_omp_for_clauses (for_stmt
),
1926 OMP_CLAUSE_LASTPRIVATE
))
1928 else if (msk
== GF_OMP_FOR_KIND_FOR
1929 && omp_find_clause (gimple_omp_parallel_clauses (stmt
),
1930 OMP_CLAUSE_LASTPRIVATE
))
1933 for (i
= 0; i
< count
; i
++)
1935 tree temp
= create_tmp_var (type
);
1936 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
1937 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
1938 OMP_CLAUSE_DECL (c
) = temp
;
1939 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
1940 gimple_omp_taskreg_set_clauses (stmt
, c
);
1943 if (msk
== GF_OMP_FOR_KIND_TASKLOOP
1944 && omp_find_clause (gimple_omp_task_clauses (stmt
),
1945 OMP_CLAUSE_REDUCTION
))
1947 tree type
= build_pointer_type (pointer_sized_int_node
);
1948 tree temp
= create_tmp_var (type
);
1949 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__REDUCTEMP_
);
1950 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
1951 OMP_CLAUSE_DECL (c
) = temp
;
1952 OMP_CLAUSE_CHAIN (c
) = gimple_omp_task_clauses (stmt
);
1953 gimple_omp_task_set_clauses (stmt
, c
);
1957 /* Scan an OpenMP parallel directive. */
1960 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1964 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
1966 /* Ignore parallel directives with empty bodies, unless there
1967 are copyin clauses. */
1969 && empty_body_p (gimple_omp_body (stmt
))
1970 && omp_find_clause (gimple_omp_parallel_clauses (stmt
),
1971 OMP_CLAUSE_COPYIN
) == NULL
)
1973 gsi_replace (gsi
, gimple_build_nop (), false);
1977 if (gimple_omp_parallel_combined_p (stmt
))
1978 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
1979 for (tree c
= omp_find_clause (gimple_omp_parallel_clauses (stmt
),
1980 OMP_CLAUSE_REDUCTION
);
1981 c
; c
= omp_find_clause (OMP_CLAUSE_CHAIN (c
), OMP_CLAUSE_REDUCTION
))
1982 if (OMP_CLAUSE_REDUCTION_TASK (c
))
1984 tree type
= build_pointer_type (pointer_sized_int_node
);
1985 tree temp
= create_tmp_var (type
);
1986 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__REDUCTEMP_
);
1988 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
1989 OMP_CLAUSE_DECL (c
) = temp
;
1990 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
1991 gimple_omp_parallel_set_clauses (stmt
, c
);
1994 else if (OMP_CLAUSE_CHAIN (c
) == NULL_TREE
)
1997 ctx
= new_omp_context (stmt
, outer_ctx
);
1998 taskreg_contexts
.safe_push (ctx
);
1999 if (taskreg_nesting_level
> 1)
2000 ctx
->is_nested
= true;
2001 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2002 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2003 name
= create_tmp_var_name (".omp_data_s");
2004 name
= build_decl (gimple_location (stmt
),
2005 TYPE_DECL
, name
, ctx
->record_type
);
2006 DECL_ARTIFICIAL (name
) = 1;
2007 DECL_NAMELESS (name
) = 1;
2008 TYPE_NAME (ctx
->record_type
) = name
;
2009 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2010 if (!gimple_omp_parallel_grid_phony (stmt
))
2012 create_omp_child_function (ctx
, false);
2013 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2016 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2017 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2019 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2020 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2023 /* Scan an OpenMP task directive. */
2026 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2030 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2032 /* Ignore task directives with empty bodies, unless they have depend
2035 && gimple_omp_body (stmt
)
2036 && empty_body_p (gimple_omp_body (stmt
))
2037 && !omp_find_clause (gimple_omp_task_clauses (stmt
), OMP_CLAUSE_DEPEND
))
2039 gsi_replace (gsi
, gimple_build_nop (), false);
2043 if (gimple_omp_task_taskloop_p (stmt
))
2044 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2046 ctx
= new_omp_context (stmt
, outer_ctx
);
2048 if (gimple_omp_task_taskwait_p (stmt
))
2050 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2054 taskreg_contexts
.safe_push (ctx
);
2055 if (taskreg_nesting_level
> 1)
2056 ctx
->is_nested
= true;
2057 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2058 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2059 name
= create_tmp_var_name (".omp_data_s");
2060 name
= build_decl (gimple_location (stmt
),
2061 TYPE_DECL
, name
, ctx
->record_type
);
2062 DECL_ARTIFICIAL (name
) = 1;
2063 DECL_NAMELESS (name
) = 1;
2064 TYPE_NAME (ctx
->record_type
) = name
;
2065 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2066 create_omp_child_function (ctx
, false);
2067 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2069 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2071 if (ctx
->srecord_type
)
2073 name
= create_tmp_var_name (".omp_data_a");
2074 name
= build_decl (gimple_location (stmt
),
2075 TYPE_DECL
, name
, ctx
->srecord_type
);
2076 DECL_ARTIFICIAL (name
) = 1;
2077 DECL_NAMELESS (name
) = 1;
2078 TYPE_NAME (ctx
->srecord_type
) = name
;
2079 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2080 create_omp_child_function (ctx
, true);
2083 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2085 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2087 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2088 t
= build_int_cst (long_integer_type_node
, 0);
2089 gimple_omp_task_set_arg_size (stmt
, t
);
2090 t
= build_int_cst (long_integer_type_node
, 1);
2091 gimple_omp_task_set_arg_align (stmt
, t
);
2095 /* Helper function for finish_taskreg_scan, called through walk_tree.
2096 If maybe_lookup_decl_in_outer_context returns non-NULL for some
2097 tree, replace it in the expression. */
2100 finish_taskreg_remap (tree
*tp
, int *walk_subtrees
, void *data
)
2104 omp_context
*ctx
= (omp_context
*) data
;
2105 tree t
= maybe_lookup_decl_in_outer_ctx (*tp
, ctx
);
2108 if (DECL_HAS_VALUE_EXPR_P (t
))
2109 t
= unshare_expr (DECL_VALUE_EXPR (t
));
2114 else if (IS_TYPE_OR_DECL_P (*tp
))
2119 /* If any decls have been made addressable during scan_omp,
2120 adjust their fields if needed, and layout record types
2121 of parallel/task constructs. */
2124 finish_taskreg_scan (omp_context
*ctx
)
2126 if (ctx
->record_type
== NULL_TREE
)
2129 /* If any task_shared_vars were needed, verify all
2130 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK,TEAMS}
2131 statements if use_pointer_for_field hasn't changed
2132 because of that. If it did, update field types now. */
2133 if (task_shared_vars
)
2137 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2138 c
; c
= OMP_CLAUSE_CHAIN (c
))
2139 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2140 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2142 tree decl
= OMP_CLAUSE_DECL (c
);
2144 /* Global variables don't need to be copied,
2145 the receiver side will use them directly. */
2146 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2148 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2149 || !use_pointer_for_field (decl
, ctx
))
2151 tree field
= lookup_field (decl
, ctx
);
2152 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2153 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2155 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2156 TREE_THIS_VOLATILE (field
) = 0;
2157 DECL_USER_ALIGN (field
) = 0;
2158 SET_DECL_ALIGN (field
, TYPE_ALIGN (TREE_TYPE (field
)));
2159 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2160 SET_TYPE_ALIGN (ctx
->record_type
, DECL_ALIGN (field
));
2161 if (ctx
->srecord_type
)
2163 tree sfield
= lookup_sfield (decl
, ctx
);
2164 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2165 TREE_THIS_VOLATILE (sfield
) = 0;
2166 DECL_USER_ALIGN (sfield
) = 0;
2167 SET_DECL_ALIGN (sfield
, DECL_ALIGN (field
));
2168 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2169 SET_TYPE_ALIGN (ctx
->srecord_type
, DECL_ALIGN (sfield
));
2174 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2176 tree clauses
= gimple_omp_parallel_clauses (ctx
->stmt
);
2177 tree c
= omp_find_clause (clauses
, OMP_CLAUSE__REDUCTEMP_
);
2180 /* Move the _reductemp_ clause first. GOMP_parallel_reductions
2181 expects to find it at the start of data. */
2182 tree f
= lookup_field (OMP_CLAUSE_DECL (c
), ctx
);
2183 tree
*p
= &TYPE_FIELDS (ctx
->record_type
);
2187 *p
= DECL_CHAIN (*p
);
2191 p
= &DECL_CHAIN (*p
);
2192 DECL_CHAIN (f
) = TYPE_FIELDS (ctx
->record_type
);
2193 TYPE_FIELDS (ctx
->record_type
) = f
;
2195 layout_type (ctx
->record_type
);
2196 fixup_child_record_type (ctx
);
2198 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2200 layout_type (ctx
->record_type
);
2201 fixup_child_record_type (ctx
);
2205 location_t loc
= gimple_location (ctx
->stmt
);
2206 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2207 /* Move VLA fields to the end. */
2208 p
= &TYPE_FIELDS (ctx
->record_type
);
2210 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2211 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2214 *p
= TREE_CHAIN (*p
);
2215 TREE_CHAIN (*q
) = NULL_TREE
;
2216 q
= &TREE_CHAIN (*q
);
2219 p
= &DECL_CHAIN (*p
);
2221 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2223 /* Move fields corresponding to first and second _looptemp_
2224 clause first. There are filled by GOMP_taskloop
2225 and thus need to be in specific positions. */
2226 tree clauses
= gimple_omp_task_clauses (ctx
->stmt
);
2227 tree c1
= omp_find_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
2228 tree c2
= omp_find_clause (OMP_CLAUSE_CHAIN (c1
),
2229 OMP_CLAUSE__LOOPTEMP_
);
2230 tree c3
= omp_find_clause (clauses
, OMP_CLAUSE__REDUCTEMP_
);
2231 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2232 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2233 tree f3
= c3
? lookup_field (OMP_CLAUSE_DECL (c3
), ctx
) : NULL_TREE
;
2234 p
= &TYPE_FIELDS (ctx
->record_type
);
2236 if (*p
== f1
|| *p
== f2
|| *p
== f3
)
2237 *p
= DECL_CHAIN (*p
);
2239 p
= &DECL_CHAIN (*p
);
2240 DECL_CHAIN (f1
) = f2
;
2243 DECL_CHAIN (f2
) = f3
;
2244 DECL_CHAIN (f3
) = TYPE_FIELDS (ctx
->record_type
);
2247 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2248 TYPE_FIELDS (ctx
->record_type
) = f1
;
2249 if (ctx
->srecord_type
)
2251 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2252 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2254 f3
= lookup_sfield (OMP_CLAUSE_DECL (c3
), ctx
);
2255 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2257 if (*p
== f1
|| *p
== f2
|| *p
== f3
)
2258 *p
= DECL_CHAIN (*p
);
2260 p
= &DECL_CHAIN (*p
);
2261 DECL_CHAIN (f1
) = f2
;
2262 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2265 DECL_CHAIN (f2
) = f3
;
2266 DECL_CHAIN (f3
) = TYPE_FIELDS (ctx
->srecord_type
);
2269 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2270 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2273 layout_type (ctx
->record_type
);
2274 fixup_child_record_type (ctx
);
2275 if (ctx
->srecord_type
)
2276 layout_type (ctx
->srecord_type
);
2277 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2278 TYPE_SIZE_UNIT (ctx
->record_type
));
2279 if (TREE_CODE (t
) != INTEGER_CST
)
2281 t
= unshare_expr (t
);
2282 walk_tree (&t
, finish_taskreg_remap
, ctx
, NULL
);
2284 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2285 t
= build_int_cst (long_integer_type_node
,
2286 TYPE_ALIGN_UNIT (ctx
->record_type
));
2287 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2291 /* Find the enclosing offload context. */
2293 static omp_context
*
2294 enclosing_target_ctx (omp_context
*ctx
)
2296 for (; ctx
; ctx
= ctx
->outer
)
2297 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2303 /* Return true if ctx is part of an oacc kernels region. */
2306 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2308 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2310 gimple
*stmt
= ctx
->stmt
;
2311 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2312 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2319 /* Check the parallelism clauses inside a kernels regions.
2320 Until kernels handling moves to use the same loop indirection
2321 scheme as parallel, we need to do this checking early. */
2324 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2326 bool checking
= true;
2327 unsigned outer_mask
= 0;
2328 unsigned this_mask
= 0;
2329 bool has_seq
= false, has_auto
= false;
2332 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2336 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2338 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
2341 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2343 switch (OMP_CLAUSE_CODE (c
))
2345 case OMP_CLAUSE_GANG
:
2346 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
2348 case OMP_CLAUSE_WORKER
:
2349 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
2351 case OMP_CLAUSE_VECTOR
:
2352 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
2354 case OMP_CLAUSE_SEQ
:
2357 case OMP_CLAUSE_AUTO
:
2367 if (has_seq
&& (this_mask
|| has_auto
))
2368 error_at (gimple_location (stmt
), "%<seq%> overrides other"
2369 " OpenACC loop specifiers");
2370 else if (has_auto
&& this_mask
)
2371 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
2372 " OpenACC loop specifiers");
2374 if (this_mask
& outer_mask
)
2375 error_at (gimple_location (stmt
), "inner loop uses same"
2376 " OpenACC parallelism as containing loop");
2379 return outer_mask
| this_mask
;
2382 /* Scan a GIMPLE_OMP_FOR. */
2384 static omp_context
*
2385 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2389 tree clauses
= gimple_omp_for_clauses (stmt
);
2391 ctx
= new_omp_context (stmt
, outer_ctx
);
2393 if (is_gimple_omp_oacc (stmt
))
2395 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
2397 if (!tgt
|| is_oacc_parallel_or_serial (tgt
))
2398 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2400 char const *check
= NULL
;
2402 switch (OMP_CLAUSE_CODE (c
))
2404 case OMP_CLAUSE_GANG
:
2408 case OMP_CLAUSE_WORKER
:
2412 case OMP_CLAUSE_VECTOR
:
2420 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
2421 error_at (gimple_location (stmt
),
2422 "argument not permitted on %qs clause in"
2423 " OpenACC %<parallel%> or %<serial%>", check
);
2426 if (tgt
&& is_oacc_kernels (tgt
))
2428 /* Strip out reductions, as they are not handled yet. */
2429 tree
*prev_ptr
= &clauses
;
2431 while (tree probe
= *prev_ptr
)
2433 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
2435 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
2436 *prev_ptr
= *next_ptr
;
2438 prev_ptr
= next_ptr
;
2441 gimple_omp_for_set_clauses (stmt
, clauses
);
2442 check_oacc_kernel_gwv (stmt
, ctx
);
2445 /* Collect all variables named in reductions on this loop. Ensure
2446 that, if this loop has a reduction on some variable v, and there is
2447 a reduction on v somewhere in an outer context, then there is a
2448 reduction on v on all intervening loops as well. */
2449 tree local_reduction_clauses
= NULL
;
2450 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2452 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
2453 local_reduction_clauses
2454 = tree_cons (NULL
, c
, local_reduction_clauses
);
2456 if (ctx
->outer_reduction_clauses
== NULL
&& ctx
->outer
!= NULL
)
2457 ctx
->outer_reduction_clauses
2458 = chainon (unshare_expr (ctx
->outer
->local_reduction_clauses
),
2459 ctx
->outer
->outer_reduction_clauses
);
2460 tree outer_reduction_clauses
= ctx
->outer_reduction_clauses
;
2461 tree local_iter
= local_reduction_clauses
;
2462 for (; local_iter
; local_iter
= TREE_CHAIN (local_iter
))
2464 tree local_clause
= TREE_VALUE (local_iter
);
2465 tree local_var
= OMP_CLAUSE_DECL (local_clause
);
2466 tree_code local_op
= OMP_CLAUSE_REDUCTION_CODE (local_clause
);
2467 bool have_outer_reduction
= false;
2468 tree ctx_iter
= outer_reduction_clauses
;
2469 for (; ctx_iter
; ctx_iter
= TREE_CHAIN (ctx_iter
))
2471 tree outer_clause
= TREE_VALUE (ctx_iter
);
2472 tree outer_var
= OMP_CLAUSE_DECL (outer_clause
);
2473 tree_code outer_op
= OMP_CLAUSE_REDUCTION_CODE (outer_clause
);
2474 if (outer_var
== local_var
&& outer_op
!= local_op
)
2476 warning_at (OMP_CLAUSE_LOCATION (local_clause
), 0,
2477 "conflicting reduction operations for %qE",
2479 inform (OMP_CLAUSE_LOCATION (outer_clause
),
2480 "location of the previous reduction for %qE",
2483 if (outer_var
== local_var
)
2485 have_outer_reduction
= true;
2489 if (have_outer_reduction
)
2491 /* There is a reduction on outer_var both on this loop and on
2492 some enclosing loop. Walk up the context tree until such a
2493 loop with a reduction on outer_var is found, and complain
2494 about all intervening loops that do not have such a
2496 struct omp_context
*curr_loop
= ctx
->outer
;
2498 while (curr_loop
!= NULL
)
2500 tree curr_iter
= curr_loop
->local_reduction_clauses
;
2501 for (; curr_iter
; curr_iter
= TREE_CHAIN (curr_iter
))
2503 tree curr_clause
= TREE_VALUE (curr_iter
);
2504 tree curr_var
= OMP_CLAUSE_DECL (curr_clause
);
2505 if (curr_var
== local_var
)
2512 warning_at (gimple_location (curr_loop
->stmt
), 0,
2513 "nested loop in reduction needs "
2514 "reduction clause for %qE",
2518 curr_loop
= curr_loop
->outer
;
2522 ctx
->local_reduction_clauses
= local_reduction_clauses
;
2523 ctx
->outer_reduction_clauses
2524 = chainon (unshare_expr (ctx
->local_reduction_clauses
),
2525 ctx
->outer_reduction_clauses
);
2528 scan_sharing_clauses (clauses
, ctx
);
2530 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2531 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2533 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2534 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2535 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2536 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2538 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2542 /* Duplicate #pragma omp simd, one for SIMT, another one for SIMD. */
2545 scan_omp_simd (gimple_stmt_iterator
*gsi
, gomp_for
*stmt
,
2546 omp_context
*outer_ctx
)
2548 gbind
*bind
= gimple_build_bind (NULL
, NULL
, NULL
);
2549 gsi_replace (gsi
, bind
, false);
2550 gimple_seq seq
= NULL
;
2551 gimple
*g
= gimple_build_call_internal (IFN_GOMP_USE_SIMT
, 0);
2552 tree cond
= create_tmp_var_raw (integer_type_node
);
2553 DECL_CONTEXT (cond
) = current_function_decl
;
2554 DECL_SEEN_IN_BIND_EXPR_P (cond
) = 1;
2555 gimple_bind_set_vars (bind
, cond
);
2556 gimple_call_set_lhs (g
, cond
);
2557 gimple_seq_add_stmt (&seq
, g
);
2558 tree lab1
= create_artificial_label (UNKNOWN_LOCATION
);
2559 tree lab2
= create_artificial_label (UNKNOWN_LOCATION
);
2560 tree lab3
= create_artificial_label (UNKNOWN_LOCATION
);
2561 g
= gimple_build_cond (NE_EXPR
, cond
, integer_zero_node
, lab1
, lab2
);
2562 gimple_seq_add_stmt (&seq
, g
);
2563 g
= gimple_build_label (lab1
);
2564 gimple_seq_add_stmt (&seq
, g
);
2565 gimple_seq new_seq
= copy_gimple_seq_and_replace_locals (stmt
);
2566 gomp_for
*new_stmt
= as_a
<gomp_for
*> (new_seq
);
2567 tree clause
= build_omp_clause (gimple_location (stmt
), OMP_CLAUSE__SIMT_
);
2568 OMP_CLAUSE_CHAIN (clause
) = gimple_omp_for_clauses (new_stmt
);
2569 gimple_omp_for_set_clauses (new_stmt
, clause
);
2570 gimple_seq_add_stmt (&seq
, new_stmt
);
2571 g
= gimple_build_goto (lab3
);
2572 gimple_seq_add_stmt (&seq
, g
);
2573 g
= gimple_build_label (lab2
);
2574 gimple_seq_add_stmt (&seq
, g
);
2575 gimple_seq_add_stmt (&seq
, stmt
);
2576 g
= gimple_build_label (lab3
);
2577 gimple_seq_add_stmt (&seq
, g
);
2578 gimple_bind_set_body (bind
, seq
);
2580 scan_omp_for (new_stmt
, outer_ctx
);
2581 scan_omp_for (stmt
, outer_ctx
)->simt_stmt
= new_stmt
;
2584 static tree
omp_find_scan (gimple_stmt_iterator
*, bool *,
2585 struct walk_stmt_info
*);
2586 static omp_context
*maybe_lookup_ctx (gimple
*);
2588 /* Duplicate #pragma omp simd, one for the scan input phase loop and one
2589 for scan phase loop. */
2592 scan_omp_simd_scan (gimple_stmt_iterator
*gsi
, gomp_for
*stmt
,
2593 omp_context
*outer_ctx
)
2595 /* The only change between inclusive and exclusive scan will be
2596 within the first simd loop, so just use inclusive in the
2597 worksharing loop. */
2598 outer_ctx
->scan_inclusive
= true;
2599 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_INCLUSIVE
);
2600 OMP_CLAUSE_DECL (c
) = integer_zero_node
;
2602 gomp_scan
*input_stmt
= gimple_build_omp_scan (NULL
, NULL_TREE
);
2603 gomp_scan
*scan_stmt
= gimple_build_omp_scan (NULL
, c
);
2604 gsi_replace (gsi
, input_stmt
, false);
2605 gimple_seq input_body
= NULL
;
2606 gimple_seq_add_stmt (&input_body
, stmt
);
2607 gsi_insert_after (gsi
, scan_stmt
, GSI_NEW_STMT
);
2609 gimple_stmt_iterator input1_gsi
= gsi_none ();
2610 struct walk_stmt_info wi
;
2611 memset (&wi
, 0, sizeof (wi
));
2613 wi
.info
= (void *) &input1_gsi
;
2614 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), omp_find_scan
, NULL
, &wi
);
2615 gcc_assert (!gsi_end_p (input1_gsi
));
2617 gimple
*input_stmt1
= gsi_stmt (input1_gsi
);
2618 gsi_next (&input1_gsi
);
2619 gimple
*scan_stmt1
= gsi_stmt (input1_gsi
);
2620 gcc_assert (scan_stmt1
&& gimple_code (scan_stmt1
) == GIMPLE_OMP_SCAN
);
2621 c
= gimple_omp_scan_clauses (as_a
<gomp_scan
*> (scan_stmt1
));
2622 if (c
&& OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_EXCLUSIVE
)
2623 std::swap (input_stmt1
, scan_stmt1
);
2625 gimple_seq input_body1
= gimple_omp_body (input_stmt1
);
2626 gimple_omp_set_body (input_stmt1
, NULL
);
2628 gimple_seq scan_body
= copy_gimple_seq_and_replace_locals (stmt
);
2629 gomp_for
*new_stmt
= as_a
<gomp_for
*> (scan_body
);
2631 gimple_omp_set_body (input_stmt1
, input_body1
);
2632 gimple_omp_set_body (scan_stmt1
, NULL
);
2634 gimple_stmt_iterator input2_gsi
= gsi_none ();
2635 memset (&wi
, 0, sizeof (wi
));
2637 wi
.info
= (void *) &input2_gsi
;
2638 walk_gimple_seq_mod (gimple_omp_body_ptr (new_stmt
), omp_find_scan
,
2640 gcc_assert (!gsi_end_p (input2_gsi
));
2642 gimple
*input_stmt2
= gsi_stmt (input2_gsi
);
2643 gsi_next (&input2_gsi
);
2644 gimple
*scan_stmt2
= gsi_stmt (input2_gsi
);
2645 gcc_assert (scan_stmt2
&& gimple_code (scan_stmt2
) == GIMPLE_OMP_SCAN
);
2646 if (c
&& OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_EXCLUSIVE
)
2647 std::swap (input_stmt2
, scan_stmt2
);
2649 gimple_omp_set_body (input_stmt2
, NULL
);
2651 gimple_omp_set_body (input_stmt
, input_body
);
2652 gimple_omp_set_body (scan_stmt
, scan_body
);
2654 omp_context
*ctx
= new_omp_context (input_stmt
, outer_ctx
);
2655 scan_omp (gimple_omp_body_ptr (input_stmt
), ctx
);
2657 ctx
= new_omp_context (scan_stmt
, outer_ctx
);
2658 scan_omp (gimple_omp_body_ptr (scan_stmt
), ctx
);
2660 maybe_lookup_ctx (new_stmt
)->for_simd_scan_phase
= true;
2663 /* Scan an OpenMP sections directive. */
2666 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
2670 ctx
= new_omp_context (stmt
, outer_ctx
);
2671 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2672 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2675 /* Scan an OpenMP single directive. */
2678 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
2683 ctx
= new_omp_context (stmt
, outer_ctx
);
2684 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2685 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2686 name
= create_tmp_var_name (".omp_copy_s");
2687 name
= build_decl (gimple_location (stmt
),
2688 TYPE_DECL
, name
, ctx
->record_type
);
2689 TYPE_NAME (ctx
->record_type
) = name
;
2691 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2692 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2694 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2695 ctx
->record_type
= NULL
;
2697 layout_type (ctx
->record_type
);
2700 /* Scan a GIMPLE_OMP_TARGET. */
2703 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
2707 bool offloaded
= is_gimple_omp_offloaded (stmt
);
2708 tree clauses
= gimple_omp_target_clauses (stmt
);
2710 ctx
= new_omp_context (stmt
, outer_ctx
);
2711 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2712 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2713 name
= create_tmp_var_name (".omp_data_t");
2714 name
= build_decl (gimple_location (stmt
),
2715 TYPE_DECL
, name
, ctx
->record_type
);
2716 DECL_ARTIFICIAL (name
) = 1;
2717 DECL_NAMELESS (name
) = 1;
2718 TYPE_NAME (ctx
->record_type
) = name
;
2719 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2723 create_omp_child_function (ctx
, false);
2724 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2727 scan_sharing_clauses (clauses
, ctx
);
2728 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2730 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2731 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2734 TYPE_FIELDS (ctx
->record_type
)
2735 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2738 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2739 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
2741 field
= DECL_CHAIN (field
))
2742 gcc_assert (DECL_ALIGN (field
) == align
);
2744 layout_type (ctx
->record_type
);
2746 fixup_child_record_type (ctx
);
2750 /* Scan an OpenMP teams directive. */
2753 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
2755 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2757 if (!gimple_omp_teams_host (stmt
))
2759 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2760 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2763 taskreg_contexts
.safe_push (ctx
);
2764 gcc_assert (taskreg_nesting_level
== 1);
2765 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2766 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2767 tree name
= create_tmp_var_name (".omp_data_s");
2768 name
= build_decl (gimple_location (stmt
),
2769 TYPE_DECL
, name
, ctx
->record_type
);
2770 DECL_ARTIFICIAL (name
) = 1;
2771 DECL_NAMELESS (name
) = 1;
2772 TYPE_NAME (ctx
->record_type
) = name
;
2773 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2774 create_omp_child_function (ctx
, false);
2775 gimple_omp_teams_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2777 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2778 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2780 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2781 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2784 /* Check nesting restrictions. */
2786 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
2790 if (ctx
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_GRID_BODY
)
2791 /* GRID_BODY is an artificial construct, nesting rules will be checked in
2792 the original copy of its contents. */
2795 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2796 inside an OpenACC CTX. */
2797 if (!(is_gimple_omp (stmt
)
2798 && is_gimple_omp_oacc (stmt
))
2799 /* Except for atomic codes that we share with OpenMP. */
2800 && !(gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
2801 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
2803 if (oacc_get_fn_attrib (cfun
->decl
) != NULL
)
2805 error_at (gimple_location (stmt
),
2806 "non-OpenACC construct inside of OpenACC routine");
2810 for (omp_context
*octx
= ctx
; octx
!= NULL
; octx
= octx
->outer
)
2811 if (is_gimple_omp (octx
->stmt
)
2812 && is_gimple_omp_oacc (octx
->stmt
))
2814 error_at (gimple_location (stmt
),
2815 "non-OpenACC construct inside of OpenACC region");
2822 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SCAN
2824 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
2826 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2827 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
2831 if (ctx
->order_concurrent
2832 && (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
2833 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
2834 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
2836 error_at (gimple_location (stmt
),
2837 "OpenMP constructs other than %<parallel%>, %<loop%>"
2838 " or %<simd%> may not be nested inside a region with"
2839 " the %<order(concurrent)%> clause");
2842 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
2844 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
2845 if (omp_find_clause (c
, OMP_CLAUSE_SIMD
))
2847 if (omp_find_clause (c
, OMP_CLAUSE_THREADS
)
2848 && (ctx
->outer
== NULL
2849 || !gimple_omp_for_combined_into_p (ctx
->stmt
)
2850 || gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
2851 || (gimple_omp_for_kind (ctx
->outer
->stmt
)
2852 != GF_OMP_FOR_KIND_FOR
)
2853 || !gimple_omp_for_combined_p (ctx
->outer
->stmt
)))
2855 error_at (gimple_location (stmt
),
2856 "%<ordered simd threads%> must be closely "
2857 "nested inside of %<for simd%> region");
2863 else if (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
2864 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
2865 || gimple_code (stmt
) == GIMPLE_OMP_SCAN
)
2867 else if (gimple_code (stmt
) == GIMPLE_OMP_FOR
2868 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
2870 error_at (gimple_location (stmt
),
2871 "OpenMP constructs other than "
2872 "%<ordered simd%>, %<simd%>, %<loop%> or %<atomic%> may "
2873 "not be nested inside %<simd%> region");
2876 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2878 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2879 || (gimple_omp_for_kind (stmt
) != GF_OMP_FOR_KIND_DISTRIBUTE
2880 && gimple_omp_for_kind (stmt
) != GF_OMP_FOR_KIND_GRID_LOOP
2881 && omp_find_clause (gimple_omp_for_clauses (stmt
),
2882 OMP_CLAUSE_BIND
) == NULL_TREE
))
2883 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2885 error_at (gimple_location (stmt
),
2886 "only %<distribute%>, %<parallel%> or %<loop%> "
2887 "regions are allowed to be strictly nested inside "
2888 "%<teams%> region");
2892 else if (ctx
->order_concurrent
2893 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
2894 && (gimple_code (stmt
) != GIMPLE_OMP_FOR
2895 || gimple_omp_for_kind (stmt
) != GF_OMP_FOR_KIND_SIMD
)
2896 && gimple_code (stmt
) != GIMPLE_OMP_SCAN
)
2899 error_at (gimple_location (stmt
),
2900 "OpenMP constructs other than %<parallel%>, %<loop%> or "
2901 "%<simd%> may not be nested inside a %<loop%> region");
2903 error_at (gimple_location (stmt
),
2904 "OpenMP constructs other than %<parallel%>, %<loop%> or "
2905 "%<simd%> may not be nested inside a region with "
2906 "the %<order(concurrent)%> clause");
2910 switch (gimple_code (stmt
))
2912 case GIMPLE_OMP_FOR
:
2913 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_SIMD
)
2915 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2917 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2919 error_at (gimple_location (stmt
),
2920 "%<distribute%> region must be strictly nested "
2921 "inside %<teams%> construct");
2926 /* We split taskloop into task and nested taskloop in it. */
2927 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
2929 /* For now, hope this will change and loop bind(parallel) will not
2930 be allowed in lots of contexts. */
2931 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
2932 && omp_find_clause (gimple_omp_for_clauses (stmt
), OMP_CLAUSE_BIND
))
2934 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
2939 switch (gimple_code (ctx
->stmt
))
2941 case GIMPLE_OMP_FOR
:
2942 ok
= (gimple_omp_for_kind (ctx
->stmt
)
2943 == GF_OMP_FOR_KIND_OACC_LOOP
);
2946 case GIMPLE_OMP_TARGET
:
2947 switch (gimple_omp_target_kind (ctx
->stmt
))
2949 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
2950 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
2951 case GF_OMP_TARGET_KIND_OACC_SERIAL
:
2962 else if (oacc_get_fn_attrib (current_function_decl
))
2966 error_at (gimple_location (stmt
),
2967 "OpenACC loop directive must be associated with"
2968 " an OpenACC compute region");
2974 if (is_gimple_call (stmt
)
2975 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2976 == BUILT_IN_GOMP_CANCEL
2977 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2978 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2980 const char *bad
= NULL
;
2981 const char *kind
= NULL
;
2982 const char *construct
2983 = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2984 == BUILT_IN_GOMP_CANCEL
)
2986 : "cancellation point";
2989 error_at (gimple_location (stmt
), "orphaned %qs construct",
2993 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2994 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2998 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3000 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3001 == BUILT_IN_GOMP_CANCEL
3002 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3003 ctx
->cancellable
= true;
3007 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3008 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3010 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3011 == BUILT_IN_GOMP_CANCEL
3012 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3014 ctx
->cancellable
= true;
3015 if (omp_find_clause (gimple_omp_for_clauses (ctx
->stmt
),
3017 warning_at (gimple_location (stmt
), 0,
3018 "%<cancel for%> inside "
3019 "%<nowait%> for construct");
3020 if (omp_find_clause (gimple_omp_for_clauses (ctx
->stmt
),
3021 OMP_CLAUSE_ORDERED
))
3022 warning_at (gimple_location (stmt
), 0,
3023 "%<cancel for%> inside "
3024 "%<ordered%> for construct");
3029 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3030 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3032 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3033 == BUILT_IN_GOMP_CANCEL
3034 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3036 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3038 ctx
->cancellable
= true;
3039 if (omp_find_clause (gimple_omp_sections_clauses
3042 warning_at (gimple_location (stmt
), 0,
3043 "%<cancel sections%> inside "
3044 "%<nowait%> sections construct");
3048 gcc_assert (ctx
->outer
3049 && gimple_code (ctx
->outer
->stmt
)
3050 == GIMPLE_OMP_SECTIONS
);
3051 ctx
->outer
->cancellable
= true;
3052 if (omp_find_clause (gimple_omp_sections_clauses
3055 warning_at (gimple_location (stmt
), 0,
3056 "%<cancel sections%> inside "
3057 "%<nowait%> sections construct");
3063 if (!is_task_ctx (ctx
)
3064 && (!is_taskloop_ctx (ctx
)
3065 || ctx
->outer
== NULL
3066 || !is_task_ctx (ctx
->outer
)))
3070 for (omp_context
*octx
= ctx
->outer
;
3071 octx
; octx
= octx
->outer
)
3073 switch (gimple_code (octx
->stmt
))
3075 case GIMPLE_OMP_TASKGROUP
:
3077 case GIMPLE_OMP_TARGET
:
3078 if (gimple_omp_target_kind (octx
->stmt
)
3079 != GF_OMP_TARGET_KIND_REGION
)
3082 case GIMPLE_OMP_PARALLEL
:
3083 case GIMPLE_OMP_TEAMS
:
3084 error_at (gimple_location (stmt
),
3085 "%<%s taskgroup%> construct not closely "
3086 "nested inside of %<taskgroup%> region",
3089 case GIMPLE_OMP_TASK
:
3090 if (gimple_omp_task_taskloop_p (octx
->stmt
)
3092 && is_taskloop_ctx (octx
->outer
))
3095 = gimple_omp_for_clauses (octx
->outer
->stmt
);
3096 if (!omp_find_clause (clauses
, OMP_CLAUSE_NOGROUP
))
3105 ctx
->cancellable
= true;
3110 error_at (gimple_location (stmt
), "invalid arguments");
3115 error_at (gimple_location (stmt
),
3116 "%<%s %s%> construct not closely nested inside of %qs",
3117 construct
, kind
, bad
);
3122 case GIMPLE_OMP_SECTIONS
:
3123 case GIMPLE_OMP_SINGLE
:
3124 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3125 switch (gimple_code (ctx
->stmt
))
3127 case GIMPLE_OMP_FOR
:
3128 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3129 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3132 case GIMPLE_OMP_SECTIONS
:
3133 case GIMPLE_OMP_SINGLE
:
3134 case GIMPLE_OMP_ORDERED
:
3135 case GIMPLE_OMP_MASTER
:
3136 case GIMPLE_OMP_TASK
:
3137 case GIMPLE_OMP_CRITICAL
:
3138 if (is_gimple_call (stmt
))
3140 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3141 != BUILT_IN_GOMP_BARRIER
)
3143 error_at (gimple_location (stmt
),
3144 "barrier region may not be closely nested inside "
3145 "of work-sharing, %<loop%>, %<critical%>, "
3146 "%<ordered%>, %<master%>, explicit %<task%> or "
3147 "%<taskloop%> region");
3150 error_at (gimple_location (stmt
),
3151 "work-sharing region may not be closely nested inside "
3152 "of work-sharing, %<loop%>, %<critical%>, %<ordered%>, "
3153 "%<master%>, explicit %<task%> or %<taskloop%> region");
3155 case GIMPLE_OMP_PARALLEL
:
3156 case GIMPLE_OMP_TEAMS
:
3158 case GIMPLE_OMP_TARGET
:
3159 if (gimple_omp_target_kind (ctx
->stmt
)
3160 == GF_OMP_TARGET_KIND_REGION
)
3167 case GIMPLE_OMP_MASTER
:
3168 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3169 switch (gimple_code (ctx
->stmt
))
3171 case GIMPLE_OMP_FOR
:
3172 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3173 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3176 case GIMPLE_OMP_SECTIONS
:
3177 case GIMPLE_OMP_SINGLE
:
3178 case GIMPLE_OMP_TASK
:
3179 error_at (gimple_location (stmt
),
3180 "%<master%> region may not be closely nested inside "
3181 "of work-sharing, %<loop%>, explicit %<task%> or "
3182 "%<taskloop%> region");
3184 case GIMPLE_OMP_PARALLEL
:
3185 case GIMPLE_OMP_TEAMS
:
3187 case GIMPLE_OMP_TARGET
:
3188 if (gimple_omp_target_kind (ctx
->stmt
)
3189 == GF_OMP_TARGET_KIND_REGION
)
3196 case GIMPLE_OMP_TASK
:
3197 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3198 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3199 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3200 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3202 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3203 error_at (OMP_CLAUSE_LOCATION (c
),
3204 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3205 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3209 case GIMPLE_OMP_ORDERED
:
3210 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3211 c
; c
= OMP_CLAUSE_CHAIN (c
))
3213 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3215 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3216 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
);
3219 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3220 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3221 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3224 /* Look for containing ordered(N) loop. */
3226 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3228 = omp_find_clause (gimple_omp_for_clauses (ctx
->stmt
),
3229 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3231 error_at (OMP_CLAUSE_LOCATION (c
),
3232 "%<ordered%> construct with %<depend%> clause "
3233 "must be closely nested inside an %<ordered%> "
3237 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3239 error_at (OMP_CLAUSE_LOCATION (c
),
3240 "%<ordered%> construct with %<depend%> clause "
3241 "must be closely nested inside a loop with "
3242 "%<ordered%> clause with a parameter");
3248 error_at (OMP_CLAUSE_LOCATION (c
),
3249 "invalid depend kind in omp %<ordered%> %<depend%>");
3253 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3254 if (omp_find_clause (c
, OMP_CLAUSE_SIMD
))
3256 /* ordered simd must be closely nested inside of simd region,
3257 and simd region must not encounter constructs other than
3258 ordered simd, therefore ordered simd may be either orphaned,
3259 or ctx->stmt must be simd. The latter case is handled already
3263 error_at (gimple_location (stmt
),
3264 "%<ordered%> %<simd%> must be closely nested inside "
3269 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3270 switch (gimple_code (ctx
->stmt
))
3272 case GIMPLE_OMP_CRITICAL
:
3273 case GIMPLE_OMP_TASK
:
3274 case GIMPLE_OMP_ORDERED
:
3275 ordered_in_taskloop
:
3276 error_at (gimple_location (stmt
),
3277 "%<ordered%> region may not be closely nested inside "
3278 "of %<critical%>, %<ordered%>, explicit %<task%> or "
3279 "%<taskloop%> region");
3281 case GIMPLE_OMP_FOR
:
3282 if (gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3283 goto ordered_in_taskloop
;
3285 o
= omp_find_clause (gimple_omp_for_clauses (ctx
->stmt
),
3286 OMP_CLAUSE_ORDERED
);
3289 error_at (gimple_location (stmt
),
3290 "%<ordered%> region must be closely nested inside "
3291 "a loop region with an %<ordered%> clause");
3294 if (OMP_CLAUSE_ORDERED_EXPR (o
) != NULL_TREE
3295 && omp_find_clause (c
, OMP_CLAUSE_DEPEND
) == NULL_TREE
)
3297 error_at (gimple_location (stmt
),
3298 "%<ordered%> region without %<depend%> clause may "
3299 "not be closely nested inside a loop region with "
3300 "an %<ordered%> clause with a parameter");
3304 case GIMPLE_OMP_TARGET
:
3305 if (gimple_omp_target_kind (ctx
->stmt
)
3306 != GF_OMP_TARGET_KIND_REGION
)
3309 case GIMPLE_OMP_PARALLEL
:
3310 case GIMPLE_OMP_TEAMS
:
3311 error_at (gimple_location (stmt
),
3312 "%<ordered%> region must be closely nested inside "
3313 "a loop region with an %<ordered%> clause");
3319 case GIMPLE_OMP_CRITICAL
:
3322 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3323 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3324 if (gomp_critical
*other_crit
3325 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3326 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3328 error_at (gimple_location (stmt
),
3329 "%<critical%> region may not be nested inside "
3330 "a %<critical%> region with the same name");
3335 case GIMPLE_OMP_TEAMS
:
3338 else if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3339 || (gimple_omp_target_kind (ctx
->stmt
)
3340 != GF_OMP_TARGET_KIND_REGION
))
3342 /* Teams construct can appear either strictly nested inside of
3343 target construct with no intervening stmts, or can be encountered
3344 only by initial task (so must not appear inside any OpenMP
3346 error_at (gimple_location (stmt
),
3347 "%<teams%> construct must be closely nested inside of "
3348 "%<target%> construct or not nested in any OpenMP "
3353 case GIMPLE_OMP_TARGET
:
3354 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3355 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3356 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3357 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3359 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3360 error_at (OMP_CLAUSE_LOCATION (c
),
3361 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3362 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3365 if (is_gimple_omp_offloaded (stmt
)
3366 && oacc_get_fn_attrib (cfun
->decl
) != NULL
)
3368 error_at (gimple_location (stmt
),
3369 "OpenACC region inside of OpenACC routine, nested "
3370 "parallelism not supported yet");
3373 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3375 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3377 if (is_gimple_omp (stmt
)
3378 && is_gimple_omp_oacc (stmt
)
3379 && is_gimple_omp (ctx
->stmt
))
3381 error_at (gimple_location (stmt
),
3382 "OpenACC construct inside of non-OpenACC region");
3388 const char *stmt_name
, *ctx_stmt_name
;
3389 switch (gimple_omp_target_kind (stmt
))
3391 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3392 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3393 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3394 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3395 stmt_name
= "target enter data"; break;
3396 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3397 stmt_name
= "target exit data"; break;
3398 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3399 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3400 case GF_OMP_TARGET_KIND_OACC_SERIAL
: stmt_name
= "serial"; break;
3401 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3402 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3403 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3404 stmt_name
= "enter/exit data"; break;
3405 case GF_OMP_TARGET_KIND_OACC_DECLARE
: stmt_name
= "declare"; break;
3406 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
: stmt_name
= "host_data";
3408 default: gcc_unreachable ();
3410 switch (gimple_omp_target_kind (ctx
->stmt
))
3412 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3413 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3414 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3415 ctx_stmt_name
= "parallel"; break;
3416 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3417 ctx_stmt_name
= "kernels"; break;
3418 case GF_OMP_TARGET_KIND_OACC_SERIAL
:
3419 ctx_stmt_name
= "serial"; break;
3420 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3421 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
3422 ctx_stmt_name
= "host_data"; break;
3423 default: gcc_unreachable ();
3426 /* OpenACC/OpenMP mismatch? */
3427 if (is_gimple_omp_oacc (stmt
)
3428 != is_gimple_omp_oacc (ctx
->stmt
))
3430 error_at (gimple_location (stmt
),
3431 "%s %qs construct inside of %s %qs region",
3432 (is_gimple_omp_oacc (stmt
)
3433 ? "OpenACC" : "OpenMP"), stmt_name
,
3434 (is_gimple_omp_oacc (ctx
->stmt
)
3435 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3438 if (is_gimple_omp_offloaded (ctx
->stmt
))
3440 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3441 if (is_gimple_omp_oacc (ctx
->stmt
))
3443 error_at (gimple_location (stmt
),
3444 "%qs construct inside of %qs region",
3445 stmt_name
, ctx_stmt_name
);
3450 warning_at (gimple_location (stmt
), 0,
3451 "%qs construct inside of %qs region",
3452 stmt_name
, ctx_stmt_name
);
3464 /* Helper function scan_omp.
3466 Callback for walk_tree or operators in walk_gimple_stmt used to
3467 scan for OMP directives in TP. */
3470 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3472 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3473 omp_context
*ctx
= (omp_context
*) wi
->info
;
3476 switch (TREE_CODE (t
))
3484 tree repl
= remap_decl (t
, &ctx
->cb
);
3485 gcc_checking_assert (TREE_CODE (repl
) != ERROR_MARK
);
3491 if (ctx
&& TYPE_P (t
))
3492 *tp
= remap_type (t
, &ctx
->cb
);
3493 else if (!DECL_P (t
))
3498 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3499 if (tem
!= TREE_TYPE (t
))
3501 if (TREE_CODE (t
) == INTEGER_CST
)
3502 *tp
= wide_int_to_tree (tem
, wi::to_wide (t
));
3504 TREE_TYPE (t
) = tem
;
3514 /* Return true if FNDECL is a setjmp or a longjmp. */
3517 setjmp_or_longjmp_p (const_tree fndecl
)
3519 if (fndecl_built_in_p (fndecl
, BUILT_IN_SETJMP
)
3520 || fndecl_built_in_p (fndecl
, BUILT_IN_LONGJMP
))
3523 tree declname
= DECL_NAME (fndecl
);
3525 || (DECL_CONTEXT (fndecl
) != NULL_TREE
3526 && TREE_CODE (DECL_CONTEXT (fndecl
)) != TRANSLATION_UNIT_DECL
)
3527 || !TREE_PUBLIC (fndecl
))
3530 const char *name
= IDENTIFIER_POINTER (declname
);
3531 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3534 /* Return true if FNDECL is an omp_* runtime API call. */
3537 omp_runtime_api_call (const_tree fndecl
)
3539 tree declname
= DECL_NAME (fndecl
);
3541 || (DECL_CONTEXT (fndecl
) != NULL_TREE
3542 && TREE_CODE (DECL_CONTEXT (fndecl
)) != TRANSLATION_UNIT_DECL
)
3543 || !TREE_PUBLIC (fndecl
))
3546 const char *name
= IDENTIFIER_POINTER (declname
);
3547 if (strncmp (name
, "omp_", 4) != 0)
3550 static const char *omp_runtime_apis
[] =
3552 /* This array has 3 sections. First omp_* calls that don't
3553 have any suffixes. */
3555 "target_associate_ptr",
3556 "target_disassociate_ptr",
3558 "target_is_present",
3560 "target_memcpy_rect",
3562 /* Now omp_* calls that are available as omp_* and omp_*_. */
3565 "destroy_nest_lock",
3568 "get_affinity_format",
3570 "get_default_device",
3572 "get_initial_device",
3574 "get_max_active_levels",
3575 "get_max_task_priority",
3583 "get_partition_num_places",
3595 "is_initial_device",
3597 "pause_resource_all",
3598 "set_affinity_format",
3606 /* And finally calls available as omp_*, omp_*_ and omp_*_8_. */
3607 "get_ancestor_thread_num",
3608 "get_partition_place_nums",
3609 "get_place_num_procs",
3610 "get_place_proc_ids",
3613 "set_default_device",
3615 "set_max_active_levels",
3622 for (unsigned i
= 0; i
< ARRAY_SIZE (omp_runtime_apis
); i
++)
3624 if (omp_runtime_apis
[i
] == NULL
)
3629 size_t len
= strlen (omp_runtime_apis
[i
]);
3630 if (strncmp (name
+ 4, omp_runtime_apis
[i
], len
) == 0
3631 && (name
[4 + len
] == '\0'
3633 && name
[4 + len
] == '_'
3634 && (name
[4 + len
+ 1] == '\0'
3636 && strcmp (name
+ 4 + len
+ 1, "8_") == 0)))))
3642 /* Helper function for scan_omp.
3644 Callback for walk_gimple_stmt used to scan for OMP directives in
3645 the current statement in GSI. */
3648 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3649 struct walk_stmt_info
*wi
)
3651 gimple
*stmt
= gsi_stmt (*gsi
);
3652 omp_context
*ctx
= (omp_context
*) wi
->info
;
3654 if (gimple_has_location (stmt
))
3655 input_location
= gimple_location (stmt
);
3657 /* Check the nesting restrictions. */
3658 bool remove
= false;
3659 if (is_gimple_omp (stmt
))
3660 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3661 else if (is_gimple_call (stmt
))
3663 tree fndecl
= gimple_call_fndecl (stmt
);
3667 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3668 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
3669 && setjmp_or_longjmp_p (fndecl
)
3673 error_at (gimple_location (stmt
),
3674 "setjmp/longjmp inside %<simd%> construct");
3676 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3677 switch (DECL_FUNCTION_CODE (fndecl
))
3679 case BUILT_IN_GOMP_BARRIER
:
3680 case BUILT_IN_GOMP_CANCEL
:
3681 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3682 case BUILT_IN_GOMP_TASKYIELD
:
3683 case BUILT_IN_GOMP_TASKWAIT
:
3684 case BUILT_IN_GOMP_TASKGROUP_START
:
3685 case BUILT_IN_GOMP_TASKGROUP_END
:
3686 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3693 omp_context
*octx
= ctx
;
3694 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SCAN
&& ctx
->outer
)
3696 if (octx
->order_concurrent
&& omp_runtime_api_call (fndecl
))
3699 error_at (gimple_location (stmt
),
3700 "OpenMP runtime API call %qD in a region with "
3701 "%<order(concurrent)%> clause", fndecl
);
3708 stmt
= gimple_build_nop ();
3709 gsi_replace (gsi
, stmt
, false);
3712 *handled_ops_p
= true;
3714 switch (gimple_code (stmt
))
3716 case GIMPLE_OMP_PARALLEL
:
3717 taskreg_nesting_level
++;
3718 scan_omp_parallel (gsi
, ctx
);
3719 taskreg_nesting_level
--;
3722 case GIMPLE_OMP_TASK
:
3723 taskreg_nesting_level
++;
3724 scan_omp_task (gsi
, ctx
);
3725 taskreg_nesting_level
--;
3728 case GIMPLE_OMP_FOR
:
3729 if ((gimple_omp_for_kind (as_a
<gomp_for
*> (stmt
))
3730 == GF_OMP_FOR_KIND_SIMD
)
3731 && gimple_omp_for_combined_into_p (stmt
)
3732 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SCAN
)
3734 tree clauses
= gimple_omp_for_clauses (as_a
<gomp_for
*> (stmt
));
3735 tree c
= omp_find_clause (clauses
, OMP_CLAUSE_REDUCTION
);
3736 if (c
&& OMP_CLAUSE_REDUCTION_INSCAN (c
) && !seen_error ())
3738 scan_omp_simd_scan (gsi
, as_a
<gomp_for
*> (stmt
), ctx
);
3742 if ((gimple_omp_for_kind (as_a
<gomp_for
*> (stmt
))
3743 == GF_OMP_FOR_KIND_SIMD
)
3744 && omp_maybe_offloaded_ctx (ctx
)
3745 && omp_max_simt_vf ())
3746 scan_omp_simd (gsi
, as_a
<gomp_for
*> (stmt
), ctx
);
3748 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3751 case GIMPLE_OMP_SECTIONS
:
3752 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3755 case GIMPLE_OMP_SINGLE
:
3756 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3759 case GIMPLE_OMP_SCAN
:
3760 if (tree clauses
= gimple_omp_scan_clauses (as_a
<gomp_scan
*> (stmt
)))
3762 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_INCLUSIVE
)
3763 ctx
->scan_inclusive
= true;
3764 else if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_EXCLUSIVE
)
3765 ctx
->scan_exclusive
= true;
3768 case GIMPLE_OMP_SECTION
:
3769 case GIMPLE_OMP_MASTER
:
3770 case GIMPLE_OMP_ORDERED
:
3771 case GIMPLE_OMP_CRITICAL
:
3772 case GIMPLE_OMP_GRID_BODY
:
3773 ctx
= new_omp_context (stmt
, ctx
);
3774 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3777 case GIMPLE_OMP_TASKGROUP
:
3778 ctx
= new_omp_context (stmt
, ctx
);
3779 scan_sharing_clauses (gimple_omp_taskgroup_clauses (stmt
), ctx
);
3780 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3783 case GIMPLE_OMP_TARGET
:
3784 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3787 case GIMPLE_OMP_TEAMS
:
3788 if (gimple_omp_teams_host (as_a
<gomp_teams
*> (stmt
)))
3790 taskreg_nesting_level
++;
3791 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3792 taskreg_nesting_level
--;
3795 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3802 *handled_ops_p
= false;
3804 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3806 var
= DECL_CHAIN (var
))
3807 insert_decl_map (&ctx
->cb
, var
, var
);
3811 *handled_ops_p
= false;
3819 /* Scan all the statements starting at the current statement. CTX
3820 contains context information about the OMP directives and
3821 clauses found during the scan. */
3824 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3826 location_t saved_location
;
3827 struct walk_stmt_info wi
;
3829 memset (&wi
, 0, sizeof (wi
));
3831 wi
.want_locations
= true;
3833 saved_location
= input_location
;
3834 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3835 input_location
= saved_location
;
3838 /* Re-gimplification and code generation routines. */
3840 /* Remove omp_member_access_dummy_var variables from gimple_bind_vars
3841 of BIND if in a method. */
3844 maybe_remove_omp_member_access_dummy_vars (gbind
*bind
)
3846 if (DECL_ARGUMENTS (current_function_decl
)
3847 && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl
))
3848 && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl
)))
3851 tree vars
= gimple_bind_vars (bind
);
3852 for (tree
*pvar
= &vars
; *pvar
; )
3853 if (omp_member_access_dummy_var (*pvar
))
3854 *pvar
= DECL_CHAIN (*pvar
);
3856 pvar
= &DECL_CHAIN (*pvar
);
3857 gimple_bind_set_vars (bind
, vars
);
3861 /* Remove omp_member_access_dummy_var variables from BLOCK_VARS of
3862 block and its subblocks. */
3865 remove_member_access_dummy_vars (tree block
)
3867 for (tree
*pvar
= &BLOCK_VARS (block
); *pvar
; )
3868 if (omp_member_access_dummy_var (*pvar
))
3869 *pvar
= DECL_CHAIN (*pvar
);
3871 pvar
= &DECL_CHAIN (*pvar
);
3873 for (block
= BLOCK_SUBBLOCKS (block
); block
; block
= BLOCK_CHAIN (block
))
3874 remove_member_access_dummy_vars (block
);
3877 /* If a context was created for STMT when it was scanned, return it. */
3879 static omp_context
*
3880 maybe_lookup_ctx (gimple
*stmt
)
3883 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3884 return n
? (omp_context
*) n
->value
: NULL
;
3888 /* Find the mapping for DECL in CTX or the immediately enclosing
3889 context that has a mapping for DECL.
3891 If CTX is a nested parallel directive, we may have to use the decl
3892 mappings created in CTX's parent context. Suppose that we have the
3893 following parallel nesting (variable UIDs showed for clarity):
3896 #omp parallel shared(iD.1562) -> outer parallel
3897 iD.1562 = iD.1562 + 1;
3899 #omp parallel shared (iD.1562) -> inner parallel
3900 iD.1562 = iD.1562 - 1;
3902 Each parallel structure will create a distinct .omp_data_s structure
3903 for copying iD.1562 in/out of the directive:
3905 outer parallel .omp_data_s.1.i -> iD.1562
3906 inner parallel .omp_data_s.2.i -> iD.1562
3908 A shared variable mapping will produce a copy-out operation before
3909 the parallel directive and a copy-in operation after it. So, in
3910 this case we would have:
3913 .omp_data_o.1.i = iD.1562;
3914 #omp parallel shared(iD.1562) -> outer parallel
3915 .omp_data_i.1 = &.omp_data_o.1
3916 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3918 .omp_data_o.2.i = iD.1562; -> **
3919 #omp parallel shared(iD.1562) -> inner parallel
3920 .omp_data_i.2 = &.omp_data_o.2
3921 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3924 ** This is a problem. The symbol iD.1562 cannot be referenced
3925 inside the body of the outer parallel region. But since we are
3926 emitting this copy operation while expanding the inner parallel
3927 directive, we need to access the CTX structure of the outer
3928 parallel directive to get the correct mapping:
3930 .omp_data_o.2.i = .omp_data_i.1->i
3932 Since there may be other workshare or parallel directives enclosing
3933 the parallel directive, it may be necessary to walk up the context
3934 parent chain. This is not a problem in general because nested
3935 parallelism happens only rarely. */
3938 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3943 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3944 t
= maybe_lookup_decl (decl
, up
);
3946 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3948 return t
? t
: decl
;
3952 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3953 in outer contexts. */
3956 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3961 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3962 t
= maybe_lookup_decl (decl
, up
);
3964 return t
? t
: decl
;
3968 /* Construct the initialization value for reduction operation OP. */
3971 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
3980 case TRUTH_ORIF_EXPR
:
3981 case TRUTH_XOR_EXPR
:
3983 return build_zero_cst (type
);
3986 case TRUTH_AND_EXPR
:
3987 case TRUTH_ANDIF_EXPR
:
3989 return fold_convert_loc (loc
, type
, integer_one_node
);
3992 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3995 if (SCALAR_FLOAT_TYPE_P (type
))
3997 REAL_VALUE_TYPE max
, min
;
3998 if (HONOR_INFINITIES (type
))
4001 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
4004 real_maxval (&min
, 1, TYPE_MODE (type
));
4005 return build_real (type
, min
);
4007 else if (POINTER_TYPE_P (type
))
4010 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4011 return wide_int_to_tree (type
, min
);
4015 gcc_assert (INTEGRAL_TYPE_P (type
));
4016 return TYPE_MIN_VALUE (type
);
4020 if (SCALAR_FLOAT_TYPE_P (type
))
4022 REAL_VALUE_TYPE max
;
4023 if (HONOR_INFINITIES (type
))
4026 real_maxval (&max
, 0, TYPE_MODE (type
));
4027 return build_real (type
, max
);
4029 else if (POINTER_TYPE_P (type
))
4032 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4033 return wide_int_to_tree (type
, max
);
4037 gcc_assert (INTEGRAL_TYPE_P (type
));
4038 return TYPE_MAX_VALUE (type
);
4046 /* Construct the initialization value for reduction CLAUSE. */
4049 omp_reduction_init (tree clause
, tree type
)
4051 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
4052 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
4055 /* Return alignment to be assumed for var in CLAUSE, which should be
4056 OMP_CLAUSE_ALIGNED. */
4059 omp_clause_aligned_alignment (tree clause
)
4061 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
4062 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
4064 /* Otherwise return implementation defined alignment. */
4065 unsigned int al
= 1;
4066 opt_scalar_mode mode_iter
;
4067 auto_vector_modes modes
;
4068 targetm
.vectorize
.autovectorize_vector_modes (&modes
, true);
4069 static enum mode_class classes
[]
4070 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4071 for (int i
= 0; i
< 4; i
+= 2)
4072 /* The for loop above dictates that we only walk through scalar classes. */
4073 FOR_EACH_MODE_IN_CLASS (mode_iter
, classes
[i
])
4075 scalar_mode mode
= mode_iter
.require ();
4076 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4077 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4079 machine_mode alt_vmode
;
4080 for (unsigned int j
= 0; j
< modes
.length (); ++j
)
4081 if (related_vector_mode (modes
[j
], mode
).exists (&alt_vmode
)
4082 && known_ge (GET_MODE_SIZE (alt_vmode
), GET_MODE_SIZE (vmode
)))
4085 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4086 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4088 type
= build_vector_type_for_mode (type
, vmode
);
4089 if (TYPE_MODE (type
) != vmode
)
4091 if (TYPE_ALIGN_UNIT (type
) > al
)
4092 al
= TYPE_ALIGN_UNIT (type
);
4094 return build_int_cst (integer_type_node
, al
);
4098 /* This structure is part of the interface between lower_rec_simd_input_clauses
4099 and lower_rec_input_clauses. */
4101 class omplow_simd_context
{
4103 omplow_simd_context () { memset (this, 0, sizeof (*this)); }
4107 vec
<tree
, va_heap
> simt_eargs
;
4108 gimple_seq simt_dlist
;
4109 poly_uint64_pod max_vf
;
4113 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4117 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
,
4118 omplow_simd_context
*sctx
, tree
&ivar
,
4119 tree
&lvar
, tree
*rvar
= NULL
,
4122 if (known_eq (sctx
->max_vf
, 0U))
4124 sctx
->max_vf
= sctx
->is_simt
? omp_max_simt_vf () : omp_max_vf ();
4125 if (maybe_gt (sctx
->max_vf
, 1U))
4127 tree c
= omp_find_clause (gimple_omp_for_clauses (ctx
->stmt
),
4128 OMP_CLAUSE_SAFELEN
);
4131 poly_uint64 safe_len
;
4132 if (!poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c
), &safe_len
)
4133 || maybe_lt (safe_len
, 1U))
4136 sctx
->max_vf
= lower_bound (sctx
->max_vf
, safe_len
);
4139 if (maybe_gt (sctx
->max_vf
, 1U))
4141 sctx
->idx
= create_tmp_var (unsigned_type_node
);
4142 sctx
->lane
= create_tmp_var (unsigned_type_node
);
4145 if (known_eq (sctx
->max_vf
, 1U))
4150 if (is_gimple_reg (new_var
))
4152 ivar
= lvar
= new_var
;
4155 tree type
= TREE_TYPE (new_var
), ptype
= build_pointer_type (type
);
4156 ivar
= lvar
= create_tmp_var (type
);
4157 TREE_ADDRESSABLE (ivar
) = 1;
4158 DECL_ATTRIBUTES (ivar
) = tree_cons (get_identifier ("omp simt private"),
4159 NULL
, DECL_ATTRIBUTES (ivar
));
4160 sctx
->simt_eargs
.safe_push (build1 (ADDR_EXPR
, ptype
, ivar
));
4161 tree clobber
= build_clobber (type
);
4162 gimple
*g
= gimple_build_assign (ivar
, clobber
);
4163 gimple_seq_add_stmt (&sctx
->simt_dlist
, g
);
4167 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), sctx
->max_vf
);
4168 tree avar
= create_tmp_var_raw (atype
);
4169 if (TREE_ADDRESSABLE (new_var
))
4170 TREE_ADDRESSABLE (avar
) = 1;
4171 DECL_ATTRIBUTES (avar
)
4172 = tree_cons (get_identifier ("omp simd array"), NULL
,
4173 DECL_ATTRIBUTES (avar
));
4174 gimple_add_tmp_var (avar
);
4176 if (rvar
&& !ctx
->for_simd_scan_phase
)
4178 /* For inscan reductions, create another array temporary,
4179 which will hold the reduced value. */
4180 iavar
= create_tmp_var_raw (atype
);
4181 if (TREE_ADDRESSABLE (new_var
))
4182 TREE_ADDRESSABLE (iavar
) = 1;
4183 DECL_ATTRIBUTES (iavar
)
4184 = tree_cons (get_identifier ("omp simd array"), NULL
,
4185 tree_cons (get_identifier ("omp simd inscan"), NULL
,
4186 DECL_ATTRIBUTES (iavar
)));
4187 gimple_add_tmp_var (iavar
);
4188 ctx
->cb
.decl_map
->put (avar
, iavar
);
4189 if (sctx
->lastlane
== NULL_TREE
)
4190 sctx
->lastlane
= create_tmp_var (unsigned_type_node
);
4191 *rvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), iavar
,
4192 sctx
->lastlane
, NULL_TREE
, NULL_TREE
);
4193 TREE_THIS_NOTRAP (*rvar
) = 1;
4195 if (ctx
->scan_exclusive
)
4197 /* And for exclusive scan yet another one, which will
4198 hold the value during the scan phase. */
4199 tree savar
= create_tmp_var_raw (atype
);
4200 if (TREE_ADDRESSABLE (new_var
))
4201 TREE_ADDRESSABLE (savar
) = 1;
4202 DECL_ATTRIBUTES (savar
)
4203 = tree_cons (get_identifier ("omp simd array"), NULL
,
4204 tree_cons (get_identifier ("omp simd inscan "
4206 DECL_ATTRIBUTES (savar
)));
4207 gimple_add_tmp_var (savar
);
4208 ctx
->cb
.decl_map
->put (iavar
, savar
);
4209 *rvar2
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), savar
,
4210 sctx
->idx
, NULL_TREE
, NULL_TREE
);
4211 TREE_THIS_NOTRAP (*rvar2
) = 1;
4214 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), iavar
, sctx
->idx
,
4215 NULL_TREE
, NULL_TREE
);
4216 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, sctx
->lane
,
4217 NULL_TREE
, NULL_TREE
);
4218 TREE_THIS_NOTRAP (ivar
) = 1;
4219 TREE_THIS_NOTRAP (lvar
) = 1;
4221 if (DECL_P (new_var
))
4223 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4224 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4229 /* Helper function of lower_rec_input_clauses. For a reference
4230 in simd reduction, add an underlying variable it will reference. */
4233 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4235 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4236 if (TREE_CONSTANT (z
))
4238 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4239 get_name (new_vard
));
4240 gimple_add_tmp_var (z
);
4241 TREE_ADDRESSABLE (z
) = 1;
4242 z
= build_fold_addr_expr_loc (loc
, z
);
4243 gimplify_assign (new_vard
, z
, ilist
);
4247 /* Helper function for lower_rec_input_clauses. Emit into ilist sequence
4248 code to emit (type) (tskred_temp[idx]). */
4251 task_reduction_read (gimple_seq
*ilist
, tree tskred_temp
, tree type
,
4254 unsigned HOST_WIDE_INT sz
4255 = tree_to_uhwi (TYPE_SIZE_UNIT (pointer_sized_int_node
));
4256 tree r
= build2 (MEM_REF
, pointer_sized_int_node
,
4257 tskred_temp
, build_int_cst (TREE_TYPE (tskred_temp
),
4259 tree v
= create_tmp_var (pointer_sized_int_node
);
4260 gimple
*g
= gimple_build_assign (v
, r
);
4261 gimple_seq_add_stmt (ilist
, g
);
4262 if (!useless_type_conversion_p (type
, pointer_sized_int_node
))
4264 v
= create_tmp_var (type
);
4265 g
= gimple_build_assign (v
, NOP_EXPR
, gimple_assign_lhs (g
));
4266 gimple_seq_add_stmt (ilist
, g
);
4271 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4272 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4273 private variables. Initialization statements go in ILIST, while calls
4274 to destructors go in DLIST. */
4277 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4278 omp_context
*ctx
, struct omp_for_data
*fd
)
4280 tree c
, copyin_seq
, x
, ptr
;
4281 bool copyin_by_ref
= false;
4282 bool lastprivate_firstprivate
= false;
4283 bool reduction_omp_orig_ref
= false;
4285 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4286 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
);
4287 omplow_simd_context sctx
= omplow_simd_context ();
4288 tree simt_lane
= NULL_TREE
, simtrec
= NULL_TREE
;
4289 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
, uid
= NULL_TREE
;
4290 gimple_seq llist
[4] = { };
4291 tree nonconst_simd_if
= NULL_TREE
;
4294 sctx
.is_simt
= is_simd
&& omp_find_clause (clauses
, OMP_CLAUSE__SIMT_
);
4296 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4297 with data sharing clauses referencing variable sized vars. That
4298 is unnecessarily hard to support and very unlikely to result in
4299 vectorized code anyway. */
4301 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4302 switch (OMP_CLAUSE_CODE (c
))
4304 case OMP_CLAUSE_LINEAR
:
4305 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4308 case OMP_CLAUSE_PRIVATE
:
4309 case OMP_CLAUSE_FIRSTPRIVATE
:
4310 case OMP_CLAUSE_LASTPRIVATE
:
4311 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4313 else if (omp_is_reference (OMP_CLAUSE_DECL (c
)))
4315 tree rtype
= TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c
)));
4316 if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype
)))
4320 case OMP_CLAUSE_REDUCTION
:
4321 case OMP_CLAUSE_IN_REDUCTION
:
4322 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4323 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4325 else if (omp_is_reference (OMP_CLAUSE_DECL (c
)))
4327 tree rtype
= TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c
)));
4328 if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype
)))
4333 if (integer_zerop (OMP_CLAUSE_IF_EXPR (c
)))
4335 else if (TREE_CODE (OMP_CLAUSE_IF_EXPR (c
)) != INTEGER_CST
)
4336 nonconst_simd_if
= OMP_CLAUSE_IF_EXPR (c
);
4338 case OMP_CLAUSE_SIMDLEN
:
4339 if (integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (c
)))
4342 case OMP_CLAUSE__CONDTEMP_
:
4343 /* FIXME: lastprivate(conditional:) not handled for SIMT yet. */
4351 /* Add a placeholder for simduid. */
4352 if (sctx
.is_simt
&& maybe_ne (sctx
.max_vf
, 1U))
4353 sctx
.simt_eargs
.safe_push (NULL_TREE
);
4355 unsigned task_reduction_cnt
= 0;
4356 unsigned task_reduction_cntorig
= 0;
4357 unsigned task_reduction_cnt_full
= 0;
4358 unsigned task_reduction_cntorig_full
= 0;
4359 unsigned task_reduction_other_cnt
= 0;
4360 tree tskred_atype
= NULL_TREE
, tskred_avar
= NULL_TREE
;
4361 tree tskred_base
= NULL_TREE
, tskred_temp
= NULL_TREE
;
4362 /* Do all the fixed sized types in the first pass, and the variable sized
4363 types in the second pass. This makes sure that the scalar arguments to
4364 the variable sized types are processed before we use them in the
4365 variable sized operations. For task reductions we use 4 passes, in the
4366 first two we ignore them, in the third one gather arguments for
4367 GOMP_task_reduction_remap call and in the last pass actually handle
4368 the task reductions. */
4369 for (pass
= 0; pass
< ((task_reduction_cnt
|| task_reduction_other_cnt
)
4372 if (pass
== 2 && task_reduction_cnt
)
4375 = build_array_type_nelts (ptr_type_node
, task_reduction_cnt
4376 + task_reduction_cntorig
);
4377 tskred_avar
= create_tmp_var_raw (tskred_atype
);
4378 gimple_add_tmp_var (tskred_avar
);
4379 TREE_ADDRESSABLE (tskred_avar
) = 1;
4380 task_reduction_cnt_full
= task_reduction_cnt
;
4381 task_reduction_cntorig_full
= task_reduction_cntorig
;
4383 else if (pass
== 3 && task_reduction_cnt
)
4385 x
= builtin_decl_explicit (BUILT_IN_GOMP_TASK_REDUCTION_REMAP
);
4387 = gimple_build_call (x
, 3, size_int (task_reduction_cnt
),
4388 size_int (task_reduction_cntorig
),
4389 build_fold_addr_expr (tskred_avar
));
4390 gimple_seq_add_stmt (ilist
, g
);
4392 if (pass
== 3 && task_reduction_other_cnt
)
4394 /* For reduction clauses, build
4395 tskred_base = (void *) tskred_temp[2]
4396 + omp_get_thread_num () * tskred_temp[1]
4397 or if tskred_temp[1] is known to be constant, that constant
4398 directly. This is the start of the private reduction copy block
4399 for the current thread. */
4400 tree v
= create_tmp_var (integer_type_node
);
4401 x
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4402 gimple
*g
= gimple_build_call (x
, 0);
4403 gimple_call_set_lhs (g
, v
);
4404 gimple_seq_add_stmt (ilist
, g
);
4405 c
= omp_find_clause (clauses
, OMP_CLAUSE__REDUCTEMP_
);
4406 tskred_temp
= OMP_CLAUSE_DECL (c
);
4407 if (is_taskreg_ctx (ctx
))
4408 tskred_temp
= lookup_decl (tskred_temp
, ctx
);
4409 tree v2
= create_tmp_var (sizetype
);
4410 g
= gimple_build_assign (v2
, NOP_EXPR
, v
);
4411 gimple_seq_add_stmt (ilist
, g
);
4412 if (ctx
->task_reductions
[0])
4413 v
= fold_convert (sizetype
, ctx
->task_reductions
[0]);
4415 v
= task_reduction_read (ilist
, tskred_temp
, sizetype
, 1);
4416 tree v3
= create_tmp_var (sizetype
);
4417 g
= gimple_build_assign (v3
, MULT_EXPR
, v2
, v
);
4418 gimple_seq_add_stmt (ilist
, g
);
4419 v
= task_reduction_read (ilist
, tskred_temp
, ptr_type_node
, 2);
4420 tskred_base
= create_tmp_var (ptr_type_node
);
4421 g
= gimple_build_assign (tskred_base
, POINTER_PLUS_EXPR
, v
, v3
);
4422 gimple_seq_add_stmt (ilist
, g
);
4424 task_reduction_cnt
= 0;
4425 task_reduction_cntorig
= 0;
4426 task_reduction_other_cnt
= 0;
4427 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4429 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4432 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4433 bool task_reduction_p
= false;
4434 bool task_reduction_needs_orig_p
= false;
4435 tree cond
= NULL_TREE
;
4439 case OMP_CLAUSE_PRIVATE
:
4440 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4443 case OMP_CLAUSE_SHARED
:
4444 /* Ignore shared directives in teams construct inside
4445 of target construct. */
4446 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
4447 && !is_host_teams_ctx (ctx
))
4449 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4451 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4452 || is_global_var (OMP_CLAUSE_DECL (c
)));
4455 case OMP_CLAUSE_FIRSTPRIVATE
:
4456 case OMP_CLAUSE_COPYIN
:
4458 case OMP_CLAUSE_LINEAR
:
4459 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4460 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4461 lastprivate_firstprivate
= true;
4463 case OMP_CLAUSE_REDUCTION
:
4464 case OMP_CLAUSE_IN_REDUCTION
:
4465 if (is_task_ctx (ctx
) || OMP_CLAUSE_REDUCTION_TASK (c
))
4467 task_reduction_p
= true;
4468 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4470 task_reduction_other_cnt
++;
4475 task_reduction_cnt
++;
4476 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4478 var
= OMP_CLAUSE_DECL (c
);
4479 /* If var is a global variable that isn't privatized
4480 in outer contexts, we don't need to look up the
4481 original address, it is always the address of the
4482 global variable itself. */
4484 || omp_is_reference (var
)
4486 (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
4488 task_reduction_needs_orig_p
= true;
4489 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4490 task_reduction_cntorig
++;
4494 else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4495 reduction_omp_orig_ref
= true;
4497 case OMP_CLAUSE__REDUCTEMP_
:
4498 if (!is_taskreg_ctx (ctx
))
4501 case OMP_CLAUSE__LOOPTEMP_
:
4502 /* Handle _looptemp_/_reductemp_ clauses only on
4507 case OMP_CLAUSE_LASTPRIVATE
:
4508 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4510 lastprivate_firstprivate
= true;
4511 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4514 /* Even without corresponding firstprivate, if
4515 decl is Fortran allocatable, it needs outer var
4518 && lang_hooks
.decls
.omp_private_outer_ref
4519 (OMP_CLAUSE_DECL (c
)))
4520 lastprivate_firstprivate
= true;
4522 case OMP_CLAUSE_ALIGNED
:
4525 var
= OMP_CLAUSE_DECL (c
);
4526 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4527 && !is_global_var (var
))
4529 new_var
= maybe_lookup_decl (var
, ctx
);
4530 if (new_var
== NULL_TREE
)
4531 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4532 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4533 tree alarg
= omp_clause_aligned_alignment (c
);
4534 alarg
= fold_convert_loc (clause_loc
, size_type_node
, alarg
);
4535 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
, alarg
);
4536 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4537 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4538 gimplify_and_add (x
, ilist
);
4540 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4541 && is_global_var (var
))
4543 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4544 new_var
= lookup_decl (var
, ctx
);
4545 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4546 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4547 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4548 tree alarg
= omp_clause_aligned_alignment (c
);
4549 alarg
= fold_convert_loc (clause_loc
, size_type_node
, alarg
);
4550 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
, alarg
);
4551 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4552 x
= create_tmp_var (ptype
);
4553 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4554 gimplify_and_add (t
, ilist
);
4555 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4556 SET_DECL_VALUE_EXPR (new_var
, t
);
4557 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4560 case OMP_CLAUSE__CONDTEMP_
:
4561 if (is_parallel_ctx (ctx
)
4562 || (is_simd
&& !OMP_CLAUSE__CONDTEMP__ITER (c
)))
4569 if (task_reduction_p
!= (pass
>= 2))
4572 new_var
= var
= OMP_CLAUSE_DECL (c
);
4573 if ((c_kind
== OMP_CLAUSE_REDUCTION
4574 || c_kind
== OMP_CLAUSE_IN_REDUCTION
)
4575 && TREE_CODE (var
) == MEM_REF
)
4577 var
= TREE_OPERAND (var
, 0);
4578 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
4579 var
= TREE_OPERAND (var
, 0);
4580 if (TREE_CODE (var
) == INDIRECT_REF
4581 || TREE_CODE (var
) == ADDR_EXPR
)
4582 var
= TREE_OPERAND (var
, 0);
4583 if (is_variable_sized (var
))
4585 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4586 var
= DECL_VALUE_EXPR (var
);
4587 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4588 var
= TREE_OPERAND (var
, 0);
4589 gcc_assert (DECL_P (var
));
4593 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4594 new_var
= lookup_decl (var
, ctx
);
4596 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4601 /* C/C++ array section reductions. */
4602 else if ((c_kind
== OMP_CLAUSE_REDUCTION
4603 || c_kind
== OMP_CLAUSE_IN_REDUCTION
)
4604 && var
!= OMP_CLAUSE_DECL (c
))
4609 tree bias
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 1);
4610 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4612 if (TREE_CODE (orig_var
) == POINTER_PLUS_EXPR
)
4614 tree b
= TREE_OPERAND (orig_var
, 1);
4615 b
= maybe_lookup_decl (b
, ctx
);
4618 b
= TREE_OPERAND (orig_var
, 1);
4619 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
4621 if (integer_zerop (bias
))
4625 bias
= fold_convert_loc (clause_loc
,
4626 TREE_TYPE (b
), bias
);
4627 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4628 TREE_TYPE (b
), b
, bias
);
4630 orig_var
= TREE_OPERAND (orig_var
, 0);
4634 tree out
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4635 if (is_global_var (out
)
4636 && TREE_CODE (TREE_TYPE (out
)) != POINTER_TYPE
4637 && (TREE_CODE (TREE_TYPE (out
)) != REFERENCE_TYPE
4638 || (TREE_CODE (TREE_TYPE (TREE_TYPE (out
)))
4643 bool by_ref
= use_pointer_for_field (var
, NULL
);
4644 x
= build_receiver_ref (var
, by_ref
, ctx
);
4645 if (TREE_CODE (TREE_TYPE (var
)) == REFERENCE_TYPE
4646 && (TREE_CODE (TREE_TYPE (TREE_TYPE (var
)))
4648 x
= build_fold_addr_expr (x
);
4650 if (TREE_CODE (orig_var
) == INDIRECT_REF
)
4651 x
= build_simple_mem_ref (x
);
4652 else if (TREE_CODE (orig_var
) == ADDR_EXPR
)
4654 if (var
== TREE_OPERAND (orig_var
, 0))
4655 x
= build_fold_addr_expr (x
);
4657 bias
= fold_convert (sizetype
, bias
);
4658 x
= fold_convert (ptr_type_node
, x
);
4659 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
4660 TREE_TYPE (x
), x
, bias
);
4661 unsigned cnt
= task_reduction_cnt
- 1;
4662 if (!task_reduction_needs_orig_p
)
4663 cnt
+= (task_reduction_cntorig_full
4664 - task_reduction_cntorig
);
4666 cnt
= task_reduction_cntorig
- 1;
4667 tree r
= build4 (ARRAY_REF
, ptr_type_node
, tskred_avar
,
4668 size_int (cnt
), NULL_TREE
, NULL_TREE
);
4669 gimplify_assign (r
, x
, ilist
);
4673 if (TREE_CODE (orig_var
) == INDIRECT_REF
4674 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4675 orig_var
= TREE_OPERAND (orig_var
, 0);
4676 tree d
= OMP_CLAUSE_DECL (c
);
4677 tree type
= TREE_TYPE (d
);
4678 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4679 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4680 const char *name
= get_name (orig_var
);
4683 tree xv
= create_tmp_var (ptr_type_node
);
4684 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4686 unsigned cnt
= task_reduction_cnt
- 1;
4687 if (!task_reduction_needs_orig_p
)
4688 cnt
+= (task_reduction_cntorig_full
4689 - task_reduction_cntorig
);
4691 cnt
= task_reduction_cntorig
- 1;
4692 x
= build4 (ARRAY_REF
, ptr_type_node
, tskred_avar
,
4693 size_int (cnt
), NULL_TREE
, NULL_TREE
);
4695 gimple
*g
= gimple_build_assign (xv
, x
);
4696 gimple_seq_add_stmt (ilist
, g
);
4700 unsigned int idx
= *ctx
->task_reduction_map
->get (c
);
4702 if (ctx
->task_reductions
[1 + idx
])
4703 off
= fold_convert (sizetype
,
4704 ctx
->task_reductions
[1 + idx
]);
4706 off
= task_reduction_read (ilist
, tskred_temp
, sizetype
,
4708 gimple
*g
= gimple_build_assign (xv
, POINTER_PLUS_EXPR
,
4710 gimple_seq_add_stmt (ilist
, g
);
4712 x
= fold_convert (build_pointer_type (boolean_type_node
),
4714 if (TREE_CONSTANT (v
))
4715 x
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (x
), x
,
4716 TYPE_SIZE_UNIT (type
));
4719 tree t
= maybe_lookup_decl (v
, ctx
);
4723 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4724 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
,
4726 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4728 build_int_cst (TREE_TYPE (v
), 1));
4729 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4731 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4732 x
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
4734 cond
= create_tmp_var (TREE_TYPE (x
));
4735 gimplify_assign (cond
, x
, ilist
);
4738 else if (TREE_CONSTANT (v
))
4740 x
= create_tmp_var_raw (type
, name
);
4741 gimple_add_tmp_var (x
);
4742 TREE_ADDRESSABLE (x
) = 1;
4743 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4748 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4749 tree t
= maybe_lookup_decl (v
, ctx
);
4753 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4754 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4755 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4757 build_int_cst (TREE_TYPE (v
), 1));
4758 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4760 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4761 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4762 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4765 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4766 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4767 tree y
= create_tmp_var (ptype
, name
);
4768 gimplify_assign (y
, x
, ilist
);
4772 if (!integer_zerop (bias
))
4774 bias
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4776 yb
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4778 yb
= fold_build2_loc (clause_loc
, MINUS_EXPR
,
4779 pointer_sized_int_node
, yb
, bias
);
4780 x
= fold_convert_loc (clause_loc
, TREE_TYPE (x
), yb
);
4781 yb
= create_tmp_var (ptype
, name
);
4782 gimplify_assign (yb
, x
, ilist
);
4786 d
= TREE_OPERAND (d
, 0);
4787 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
4788 d
= TREE_OPERAND (d
, 0);
4789 if (TREE_CODE (d
) == ADDR_EXPR
)
4791 if (orig_var
!= var
)
4793 gcc_assert (is_variable_sized (orig_var
));
4794 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4796 gimplify_assign (new_var
, x
, ilist
);
4797 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4798 tree t
= build_fold_indirect_ref (new_var
);
4799 DECL_IGNORED_P (new_var
) = 0;
4800 TREE_THIS_NOTRAP (t
) = 1;
4801 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4802 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4806 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4807 build_int_cst (ptype
, 0));
4808 SET_DECL_VALUE_EXPR (new_var
, x
);
4809 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4814 gcc_assert (orig_var
== var
);
4815 if (TREE_CODE (d
) == INDIRECT_REF
)
4817 x
= create_tmp_var (ptype
, name
);
4818 TREE_ADDRESSABLE (x
) = 1;
4819 gimplify_assign (x
, yb
, ilist
);
4820 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4822 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4823 gimplify_assign (new_var
, x
, ilist
);
4825 /* GOMP_taskgroup_reduction_register memsets the whole
4826 array to zero. If the initializer is zero, we don't
4827 need to initialize it again, just mark it as ever
4828 used unconditionally, i.e. cond = true. */
4830 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) == NULL_TREE
4831 && initializer_zerop (omp_reduction_init (c
,
4834 gimple
*g
= gimple_build_assign (build_simple_mem_ref (cond
),
4836 gimple_seq_add_stmt (ilist
, g
);
4839 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4843 if (!is_parallel_ctx (ctx
))
4845 tree condv
= create_tmp_var (boolean_type_node
);
4846 g
= gimple_build_assign (condv
,
4847 build_simple_mem_ref (cond
));
4848 gimple_seq_add_stmt (ilist
, g
);
4849 tree lab1
= create_artificial_label (UNKNOWN_LOCATION
);
4850 g
= gimple_build_cond (NE_EXPR
, condv
,
4851 boolean_false_node
, end
, lab1
);
4852 gimple_seq_add_stmt (ilist
, g
);
4853 gimple_seq_add_stmt (ilist
, gimple_build_label (lab1
));
4855 g
= gimple_build_assign (build_simple_mem_ref (cond
),
4857 gimple_seq_add_stmt (ilist
, g
);
4860 tree y1
= create_tmp_var (ptype
);
4861 gimplify_assign (y1
, y
, ilist
);
4862 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4863 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4864 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4865 if (task_reduction_needs_orig_p
)
4867 y3
= create_tmp_var (ptype
);
4869 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4870 ref
= build4 (ARRAY_REF
, ptr_type_node
, tskred_avar
,
4871 size_int (task_reduction_cnt_full
4872 + task_reduction_cntorig
- 1),
4873 NULL_TREE
, NULL_TREE
);
4876 unsigned int idx
= *ctx
->task_reduction_map
->get (c
);
4877 ref
= task_reduction_read (ilist
, tskred_temp
, ptype
,
4880 gimplify_assign (y3
, ref
, ilist
);
4882 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4886 y2
= create_tmp_var (ptype
);
4887 gimplify_assign (y2
, y
, ilist
);
4889 if (is_simd
|| OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4891 tree ref
= build_outer_var_ref (var
, ctx
);
4892 /* For ref build_outer_var_ref already performs this. */
4893 if (TREE_CODE (d
) == INDIRECT_REF
)
4894 gcc_assert (omp_is_reference (var
));
4895 else if (TREE_CODE (d
) == ADDR_EXPR
)
4896 ref
= build_fold_addr_expr (ref
);
4897 else if (omp_is_reference (var
))
4898 ref
= build_fold_addr_expr (ref
);
4899 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4900 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4901 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4903 y3
= create_tmp_var (ptype
);
4904 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4908 y4
= create_tmp_var (ptype
);
4909 gimplify_assign (y4
, ref
, dlist
);
4913 tree i
= create_tmp_var (TREE_TYPE (v
));
4914 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4915 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4916 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4919 i2
= create_tmp_var (TREE_TYPE (v
));
4920 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4921 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4922 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4923 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4925 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4927 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4928 tree decl_placeholder
4929 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4930 SET_DECL_VALUE_EXPR (decl_placeholder
,
4931 build_simple_mem_ref (y1
));
4932 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4933 SET_DECL_VALUE_EXPR (placeholder
,
4934 y3
? build_simple_mem_ref (y3
)
4936 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4937 x
= lang_hooks
.decls
.omp_clause_default_ctor
4938 (c
, build_simple_mem_ref (y1
),
4939 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4941 gimplify_and_add (x
, ilist
);
4942 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4944 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4945 lower_omp (&tseq
, ctx
);
4946 gimple_seq_add_seq (ilist
, tseq
);
4948 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4951 SET_DECL_VALUE_EXPR (decl_placeholder
,
4952 build_simple_mem_ref (y2
));
4953 SET_DECL_VALUE_EXPR (placeholder
,
4954 build_simple_mem_ref (y4
));
4955 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4956 lower_omp (&tseq
, ctx
);
4957 gimple_seq_add_seq (dlist
, tseq
);
4958 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4960 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4961 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4964 x
= lang_hooks
.decls
.omp_clause_dtor
4965 (c
, build_simple_mem_ref (y2
));
4967 gimplify_and_add (x
, dlist
);
4972 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4973 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4975 /* reduction(-:var) sums up the partial results, so it
4976 acts identically to reduction(+:var). */
4977 if (code
== MINUS_EXPR
)
4980 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4983 x
= build2 (code
, TREE_TYPE (type
),
4984 build_simple_mem_ref (y4
),
4985 build_simple_mem_ref (y2
));
4986 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4990 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4991 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4992 gimple_seq_add_stmt (ilist
, g
);
4995 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4996 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4997 gimple_seq_add_stmt (ilist
, g
);
4999 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5000 build_int_cst (TREE_TYPE (i
), 1));
5001 gimple_seq_add_stmt (ilist
, g
);
5002 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5003 gimple_seq_add_stmt (ilist
, g
);
5004 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
5007 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
5008 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5009 gimple_seq_add_stmt (dlist
, g
);
5012 g
= gimple_build_assign
5013 (y4
, POINTER_PLUS_EXPR
, y4
,
5014 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5015 gimple_seq_add_stmt (dlist
, g
);
5017 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
5018 build_int_cst (TREE_TYPE (i2
), 1));
5019 gimple_seq_add_stmt (dlist
, g
);
5020 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
5021 gimple_seq_add_stmt (dlist
, g
);
5022 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
5028 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
5032 bool by_ref
= use_pointer_for_field (var
, ctx
);
5033 x
= build_receiver_ref (var
, by_ref
, ctx
);
5035 if (!omp_is_reference (var
))
5036 x
= build_fold_addr_expr (x
);
5037 x
= fold_convert (ptr_type_node
, x
);
5038 unsigned cnt
= task_reduction_cnt
- 1;
5039 if (!task_reduction_needs_orig_p
)
5040 cnt
+= task_reduction_cntorig_full
- task_reduction_cntorig
;
5042 cnt
= task_reduction_cntorig
- 1;
5043 tree r
= build4 (ARRAY_REF
, ptr_type_node
, tskred_avar
,
5044 size_int (cnt
), NULL_TREE
, NULL_TREE
);
5045 gimplify_assign (r
, x
, ilist
);
5050 tree type
= TREE_TYPE (new_var
);
5051 if (!omp_is_reference (var
))
5052 type
= build_pointer_type (type
);
5053 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5055 unsigned cnt
= task_reduction_cnt
- 1;
5056 if (!task_reduction_needs_orig_p
)
5057 cnt
+= (task_reduction_cntorig_full
5058 - task_reduction_cntorig
);
5060 cnt
= task_reduction_cntorig
- 1;
5061 x
= build4 (ARRAY_REF
, ptr_type_node
, tskred_avar
,
5062 size_int (cnt
), NULL_TREE
, NULL_TREE
);
5066 unsigned int idx
= *ctx
->task_reduction_map
->get (c
);
5068 if (ctx
->task_reductions
[1 + idx
])
5069 off
= fold_convert (sizetype
,
5070 ctx
->task_reductions
[1 + idx
]);
5072 off
= task_reduction_read (ilist
, tskred_temp
, sizetype
,
5074 x
= fold_build2 (POINTER_PLUS_EXPR
, ptr_type_node
,
5077 x
= fold_convert (type
, x
);
5079 if (omp_is_reference (var
))
5081 gimplify_assign (new_var
, x
, ilist
);
5083 new_var
= build_simple_mem_ref (new_var
);
5087 t
= create_tmp_var (type
);
5088 gimplify_assign (t
, x
, ilist
);
5089 SET_DECL_VALUE_EXPR (new_var
, build_simple_mem_ref (t
));
5090 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
5092 t
= fold_convert (build_pointer_type (boolean_type_node
), t
);
5093 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
,
5094 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5095 cond
= create_tmp_var (TREE_TYPE (t
));
5096 gimplify_assign (cond
, t
, ilist
);
5098 else if (is_variable_sized (var
))
5100 /* For variable sized types, we need to allocate the
5101 actual storage here. Call alloca and store the
5102 result in the pointer decl that we created elsewhere. */
5106 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
5111 ptr
= DECL_VALUE_EXPR (new_var
);
5112 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
5113 ptr
= TREE_OPERAND (ptr
, 0);
5114 gcc_assert (DECL_P (ptr
));
5115 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
5117 /* void *tmp = __builtin_alloca */
5118 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
5119 stmt
= gimple_build_call (atmp
, 2, x
,
5120 size_int (DECL_ALIGN (var
)));
5121 tmp
= create_tmp_var_raw (ptr_type_node
);
5122 gimple_add_tmp_var (tmp
);
5123 gimple_call_set_lhs (stmt
, tmp
);
5125 gimple_seq_add_stmt (ilist
, stmt
);
5127 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
5128 gimplify_assign (ptr
, x
, ilist
);
5131 else if (omp_is_reference (var
)
5132 && (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
5133 || !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c
)))
5135 /* For references that are being privatized for Fortran,
5136 allocate new backing storage for the new pointer
5137 variable. This allows us to avoid changing all the
5138 code that expects a pointer to something that expects
5139 a direct variable. */
5143 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
5144 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
5146 x
= build_receiver_ref (var
, false, ctx
);
5147 x
= build_fold_addr_expr_loc (clause_loc
, x
);
5149 else if (TREE_CONSTANT (x
))
5151 /* For reduction in SIMD loop, defer adding the
5152 initialization of the reference, because if we decide
5153 to use SIMD array for it, the initilization could cause
5154 expansion ICE. Ditto for other privatization clauses. */
5159 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
5161 gimple_add_tmp_var (x
);
5162 TREE_ADDRESSABLE (x
) = 1;
5163 x
= build_fold_addr_expr_loc (clause_loc
, x
);
5169 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
5170 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
5171 tree al
= size_int (TYPE_ALIGN (rtype
));
5172 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
5177 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
5178 gimplify_assign (new_var
, x
, ilist
);
5181 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5183 else if ((c_kind
== OMP_CLAUSE_REDUCTION
5184 || c_kind
== OMP_CLAUSE_IN_REDUCTION
)
5185 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5193 switch (OMP_CLAUSE_CODE (c
))
5195 case OMP_CLAUSE_SHARED
:
5196 /* Ignore shared directives in teams construct inside
5197 target construct. */
5198 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
5199 && !is_host_teams_ctx (ctx
))
5201 /* Shared global vars are just accessed directly. */
5202 if (is_global_var (new_var
))
5204 /* For taskloop firstprivate/lastprivate, represented
5205 as firstprivate and shared clause on the task, new_var
5206 is the firstprivate var. */
5207 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5209 /* Set up the DECL_VALUE_EXPR for shared variables now. This
5210 needs to be delayed until after fixup_child_record_type so
5211 that we get the correct type during the dereference. */
5212 by_ref
= use_pointer_for_field (var
, ctx
);
5213 x
= build_receiver_ref (var
, by_ref
, ctx
);
5214 SET_DECL_VALUE_EXPR (new_var
, x
);
5215 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
5217 /* ??? If VAR is not passed by reference, and the variable
5218 hasn't been initialized yet, then we'll get a warning for
5219 the store into the omp_data_s structure. Ideally, we'd be
5220 able to notice this and not store anything at all, but
5221 we're generating code too early. Suppress the warning. */
5223 TREE_NO_WARNING (var
) = 1;
5226 case OMP_CLAUSE__CONDTEMP_
:
5227 if (is_parallel_ctx (ctx
))
5229 x
= build_receiver_ref (var
, false, ctx
);
5230 SET_DECL_VALUE_EXPR (new_var
, x
);
5231 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
5233 else if (is_simd
&& !OMP_CLAUSE__CONDTEMP__ITER (c
))
5235 x
= build_zero_cst (TREE_TYPE (var
));
5240 case OMP_CLAUSE_LASTPRIVATE
:
5241 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
5245 case OMP_CLAUSE_PRIVATE
:
5246 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
5247 x
= build_outer_var_ref (var
, ctx
);
5248 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
5250 if (is_task_ctx (ctx
))
5251 x
= build_receiver_ref (var
, false, ctx
);
5253 x
= build_outer_var_ref (var
, ctx
, OMP_CLAUSE_PRIVATE
);
5261 nx
= unshare_expr (new_var
);
5263 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5264 && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c
))
5267 nx
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, nx
, x
);
5269 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, nx
, x
);
5272 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
5273 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
5274 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5275 && (gimple_omp_for_collapse (ctx
->stmt
) != 1
5276 || (gimple_omp_for_index (ctx
->stmt
, 0)
5278 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE__CONDTEMP_
5279 || omp_is_reference (var
))
5280 && lower_rec_simd_input_clauses (new_var
, ctx
, &sctx
,
5283 if (omp_is_reference (var
))
5285 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5286 tree new_vard
= TREE_OPERAND (new_var
, 0);
5287 gcc_assert (DECL_P (new_vard
));
5288 SET_DECL_VALUE_EXPR (new_vard
,
5289 build_fold_addr_expr (lvar
));
5290 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5295 tree iv
= unshare_expr (ivar
);
5297 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
,
5300 x
= lang_hooks
.decls
.omp_clause_default_ctor (c
,
5304 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE__CONDTEMP_
)
5306 x
= build2 (MODIFY_EXPR
, TREE_TYPE (ivar
),
5307 unshare_expr (ivar
), x
);
5311 gimplify_and_add (x
, &llist
[0]);
5312 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5313 && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c
))
5318 gcc_assert (TREE_CODE (v
) == MEM_REF
);
5319 v
= TREE_OPERAND (v
, 0);
5320 gcc_assert (DECL_P (v
));
5322 v
= *ctx
->lastprivate_conditional_map
->get (v
);
5323 tree t
= create_tmp_var (TREE_TYPE (v
));
5324 tree z
= build_zero_cst (TREE_TYPE (v
));
5326 = build_outer_var_ref (var
, ctx
,
5327 OMP_CLAUSE_LASTPRIVATE
);
5328 gimple_seq_add_stmt (dlist
,
5329 gimple_build_assign (t
, z
));
5330 gcc_assert (DECL_HAS_VALUE_EXPR_P (v
));
5331 tree civar
= DECL_VALUE_EXPR (v
);
5332 gcc_assert (TREE_CODE (civar
) == ARRAY_REF
);
5333 civar
= unshare_expr (civar
);
5334 TREE_OPERAND (civar
, 1) = sctx
.idx
;
5335 x
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
,
5336 unshare_expr (civar
));
5337 x
= build2 (COMPOUND_EXPR
, TREE_TYPE (orig_v
), x
,
5338 build2 (MODIFY_EXPR
, TREE_TYPE (orig_v
),
5339 orig_v
, unshare_expr (ivar
)));
5340 tree cond
= build2 (LT_EXPR
, boolean_type_node
, t
,
5342 x
= build3 (COND_EXPR
, void_type_node
, cond
, x
,
5344 gimple_seq tseq
= NULL
;
5345 gimplify_and_add (x
, &tseq
);
5347 lower_omp (&tseq
, ctx
->outer
);
5348 gimple_seq_add_seq (&llist
[1], tseq
);
5350 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5351 && ctx
->for_simd_scan_phase
)
5353 x
= unshare_expr (ivar
);
5355 = build_outer_var_ref (var
, ctx
,
5356 OMP_CLAUSE_LASTPRIVATE
);
5357 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
,
5359 gimplify_and_add (x
, &llist
[0]);
5363 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5365 gimplify_and_add (y
, &llist
[1]);
5369 if (omp_is_reference (var
))
5371 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5372 tree new_vard
= TREE_OPERAND (new_var
, 0);
5373 gcc_assert (DECL_P (new_vard
));
5374 tree type
= TREE_TYPE (TREE_TYPE (new_vard
));
5375 x
= TYPE_SIZE_UNIT (type
);
5376 if (TREE_CONSTANT (x
))
5378 x
= create_tmp_var_raw (type
, get_name (var
));
5379 gimple_add_tmp_var (x
);
5380 TREE_ADDRESSABLE (x
) = 1;
5381 x
= build_fold_addr_expr_loc (clause_loc
, x
);
5382 x
= fold_convert_loc (clause_loc
,
5383 TREE_TYPE (new_vard
), x
);
5384 gimplify_assign (new_vard
, x
, ilist
);
5389 gimplify_and_add (nx
, ilist
);
5390 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5392 && ctx
->for_simd_scan_phase
)
5394 tree orig_v
= build_outer_var_ref (var
, ctx
,
5395 OMP_CLAUSE_LASTPRIVATE
);
5396 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
,
5398 gimplify_and_add (x
, ilist
);
5403 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
5405 gimplify_and_add (x
, dlist
);
5408 case OMP_CLAUSE_LINEAR
:
5409 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
5410 goto do_firstprivate
;
5411 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
5414 x
= build_outer_var_ref (var
, ctx
);
5417 case OMP_CLAUSE_FIRSTPRIVATE
:
5418 if (is_task_ctx (ctx
))
5420 if ((omp_is_reference (var
)
5421 && !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c
))
5422 || is_variable_sized (var
))
5424 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
5426 || use_pointer_for_field (var
, NULL
))
5428 x
= build_receiver_ref (var
, false, ctx
);
5429 SET_DECL_VALUE_EXPR (new_var
, x
);
5430 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
5434 if (OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c
)
5435 && omp_is_reference (var
))
5437 x
= build_outer_var_ref (var
, ctx
);
5438 gcc_assert (TREE_CODE (x
) == MEM_REF
5439 && integer_zerop (TREE_OPERAND (x
, 1)));
5440 x
= TREE_OPERAND (x
, 0);
5441 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5442 (c
, unshare_expr (new_var
), x
);
5443 gimplify_and_add (x
, ilist
);
5447 x
= build_outer_var_ref (var
, ctx
);
5450 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5451 && gimple_omp_for_combined_into_p (ctx
->stmt
))
5453 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5454 tree stept
= TREE_TYPE (t
);
5455 tree ct
= omp_find_clause (clauses
,
5456 OMP_CLAUSE__LOOPTEMP_
);
5458 tree l
= OMP_CLAUSE_DECL (ct
);
5459 tree n1
= fd
->loop
.n1
;
5460 tree step
= fd
->loop
.step
;
5461 tree itype
= TREE_TYPE (l
);
5462 if (POINTER_TYPE_P (itype
))
5463 itype
= signed_type_for (itype
);
5464 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
5465 if (TYPE_UNSIGNED (itype
)
5466 && fd
->loop
.cond_code
== GT_EXPR
)
5467 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5468 fold_build1 (NEGATE_EXPR
, itype
, l
),
5469 fold_build1 (NEGATE_EXPR
,
5472 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
5473 t
= fold_build2 (MULT_EXPR
, stept
,
5474 fold_convert (stept
, l
), t
);
5476 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
5478 if (omp_is_reference (var
))
5480 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5481 tree new_vard
= TREE_OPERAND (new_var
, 0);
5482 gcc_assert (DECL_P (new_vard
));
5483 tree type
= TREE_TYPE (TREE_TYPE (new_vard
));
5484 nx
= TYPE_SIZE_UNIT (type
);
5485 if (TREE_CONSTANT (nx
))
5487 nx
= create_tmp_var_raw (type
,
5489 gimple_add_tmp_var (nx
);
5490 TREE_ADDRESSABLE (nx
) = 1;
5491 nx
= build_fold_addr_expr_loc (clause_loc
,
5493 nx
= fold_convert_loc (clause_loc
,
5494 TREE_TYPE (new_vard
),
5496 gimplify_assign (new_vard
, nx
, ilist
);
5500 x
= lang_hooks
.decls
.omp_clause_linear_ctor
5502 gimplify_and_add (x
, ilist
);
5506 if (POINTER_TYPE_P (TREE_TYPE (x
)))
5507 x
= fold_build2 (POINTER_PLUS_EXPR
,
5508 TREE_TYPE (x
), x
, t
);
5510 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
5513 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
5514 || TREE_ADDRESSABLE (new_var
)
5515 || omp_is_reference (var
))
5516 && lower_rec_simd_input_clauses (new_var
, ctx
, &sctx
,
5519 if (omp_is_reference (var
))
5521 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5522 tree new_vard
= TREE_OPERAND (new_var
, 0);
5523 gcc_assert (DECL_P (new_vard
));
5524 SET_DECL_VALUE_EXPR (new_vard
,
5525 build_fold_addr_expr (lvar
));
5526 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5528 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
5530 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
5531 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
5532 gimplify_and_add (x
, ilist
);
5533 gimple_stmt_iterator gsi
5534 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5536 = gimple_build_assign (unshare_expr (lvar
), iv
);
5537 gsi_insert_before_without_update (&gsi
, g
,
5539 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5540 enum tree_code code
= PLUS_EXPR
;
5541 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
5542 code
= POINTER_PLUS_EXPR
;
5543 g
= gimple_build_assign (iv
, code
, iv
, t
);
5544 gsi_insert_before_without_update (&gsi
, g
,
5548 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5549 (c
, unshare_expr (ivar
), x
);
5550 gimplify_and_add (x
, &llist
[0]);
5551 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5553 gimplify_and_add (x
, &llist
[1]);
5556 if (omp_is_reference (var
))
5558 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5559 tree new_vard
= TREE_OPERAND (new_var
, 0);
5560 gcc_assert (DECL_P (new_vard
));
5561 tree type
= TREE_TYPE (TREE_TYPE (new_vard
));
5562 nx
= TYPE_SIZE_UNIT (type
);
5563 if (TREE_CONSTANT (nx
))
5565 nx
= create_tmp_var_raw (type
, get_name (var
));
5566 gimple_add_tmp_var (nx
);
5567 TREE_ADDRESSABLE (nx
) = 1;
5568 nx
= build_fold_addr_expr_loc (clause_loc
, nx
);
5569 nx
= fold_convert_loc (clause_loc
,
5570 TREE_TYPE (new_vard
), nx
);
5571 gimplify_assign (new_vard
, nx
, ilist
);
5575 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5576 (c
, unshare_expr (new_var
), x
);
5577 gimplify_and_add (x
, ilist
);
5580 case OMP_CLAUSE__LOOPTEMP_
:
5581 case OMP_CLAUSE__REDUCTEMP_
:
5582 gcc_assert (is_taskreg_ctx (ctx
));
5583 x
= build_outer_var_ref (var
, ctx
);
5584 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
5585 gimplify_and_add (x
, ilist
);
5588 case OMP_CLAUSE_COPYIN
:
5589 by_ref
= use_pointer_for_field (var
, NULL
);
5590 x
= build_receiver_ref (var
, by_ref
, ctx
);
5591 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
5592 append_to_statement_list (x
, ©in_seq
);
5593 copyin_by_ref
|= by_ref
;
5596 case OMP_CLAUSE_REDUCTION
:
5597 case OMP_CLAUSE_IN_REDUCTION
:
5598 /* OpenACC reductions are initialized using the
5599 GOACC_REDUCTION internal function. */
5600 if (is_gimple_omp_oacc (ctx
->stmt
))
5602 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5604 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5606 tree ptype
= TREE_TYPE (placeholder
);
5609 x
= error_mark_node
;
5610 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
)
5611 && !task_reduction_needs_orig_p
)
5613 else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
5615 tree pptype
= build_pointer_type (ptype
);
5616 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5617 x
= build4 (ARRAY_REF
, ptr_type_node
, tskred_avar
,
5618 size_int (task_reduction_cnt_full
5619 + task_reduction_cntorig
- 1),
5620 NULL_TREE
, NULL_TREE
);
5624 = *ctx
->task_reduction_map
->get (c
);
5625 x
= task_reduction_read (ilist
, tskred_temp
,
5626 pptype
, 7 + 3 * idx
);
5628 x
= fold_convert (pptype
, x
);
5629 x
= build_simple_mem_ref (x
);
5634 x
= build_outer_var_ref (var
, ctx
);
5636 if (omp_is_reference (var
)
5637 && !useless_type_conversion_p (ptype
, TREE_TYPE (x
)))
5638 x
= build_fold_addr_expr_loc (clause_loc
, x
);
5640 SET_DECL_VALUE_EXPR (placeholder
, x
);
5641 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5642 tree new_vard
= new_var
;
5643 if (omp_is_reference (var
))
5645 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5646 new_vard
= TREE_OPERAND (new_var
, 0);
5647 gcc_assert (DECL_P (new_vard
));
5649 tree rvar
= NULL_TREE
, *rvarp
= NULL
, rvar2
= NULL_TREE
;
5651 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
5652 && OMP_CLAUSE_REDUCTION_INSCAN (c
))
5655 && lower_rec_simd_input_clauses (new_var
, ctx
, &sctx
,
5659 if (new_vard
== new_var
)
5661 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
5662 SET_DECL_VALUE_EXPR (new_var
, ivar
);
5666 SET_DECL_VALUE_EXPR (new_vard
,
5667 build_fold_addr_expr (ivar
));
5668 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5670 x
= lang_hooks
.decls
.omp_clause_default_ctor
5671 (c
, unshare_expr (ivar
),
5672 build_outer_var_ref (var
, ctx
));
5673 if (rvarp
&& ctx
->for_simd_scan_phase
)
5676 gimplify_and_add (x
, &llist
[0]);
5677 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5679 gimplify_and_add (x
, &llist
[1]);
5686 gimplify_and_add (x
, &llist
[0]);
5688 tree ivar2
= unshare_expr (lvar
);
5689 TREE_OPERAND (ivar2
, 1) = sctx
.idx
;
5690 x
= lang_hooks
.decls
.omp_clause_default_ctor
5691 (c
, ivar2
, build_outer_var_ref (var
, ctx
));
5692 gimplify_and_add (x
, &llist
[0]);
5696 x
= lang_hooks
.decls
.omp_clause_default_ctor
5697 (c
, unshare_expr (rvar2
),
5698 build_outer_var_ref (var
, ctx
));
5699 gimplify_and_add (x
, &llist
[0]);
5702 /* For types that need construction, add another
5703 private var which will be default constructed
5704 and optionally initialized with
5705 OMP_CLAUSE_REDUCTION_GIMPLE_INIT, as in the
5706 loop we want to assign this value instead of
5707 constructing and destructing it in each
5709 tree nv
= create_tmp_var_raw (TREE_TYPE (ivar
));
5710 gimple_add_tmp_var (nv
);
5711 ctx
->cb
.decl_map
->put (TREE_OPERAND (rvar2
5715 x
= lang_hooks
.decls
.omp_clause_default_ctor
5716 (c
, nv
, build_outer_var_ref (var
, ctx
));
5717 gimplify_and_add (x
, ilist
);
5719 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5721 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5722 x
= DECL_VALUE_EXPR (new_vard
);
5724 if (new_vard
!= new_var
)
5725 vexpr
= build_fold_addr_expr (nv
);
5726 SET_DECL_VALUE_EXPR (new_vard
, vexpr
);
5727 lower_omp (&tseq
, ctx
);
5728 SET_DECL_VALUE_EXPR (new_vard
, x
);
5729 gimple_seq_add_seq (ilist
, tseq
);
5730 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5733 x
= lang_hooks
.decls
.omp_clause_dtor (c
, nv
);
5735 gimplify_and_add (x
, dlist
);
5738 tree ref
= build_outer_var_ref (var
, ctx
);
5739 x
= unshare_expr (ivar
);
5740 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
,
5742 gimplify_and_add (x
, &llist
[0]);
5744 ref
= build_outer_var_ref (var
, ctx
);
5745 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, ref
,
5747 gimplify_and_add (x
, &llist
[3]);
5749 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5750 if (new_vard
== new_var
)
5751 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5753 SET_DECL_VALUE_EXPR (new_vard
,
5754 build_fold_addr_expr (lvar
));
5756 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5758 gimplify_and_add (x
, &llist
[1]);
5760 tree ivar2
= unshare_expr (lvar
);
5761 TREE_OPERAND (ivar2
, 1) = sctx
.idx
;
5762 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar2
);
5764 gimplify_and_add (x
, &llist
[1]);
5768 x
= lang_hooks
.decls
.omp_clause_dtor (c
, rvar2
);
5770 gimplify_and_add (x
, &llist
[1]);
5775 gimplify_and_add (x
, &llist
[0]);
5776 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5778 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5779 lower_omp (&tseq
, ctx
);
5780 gimple_seq_add_seq (&llist
[0], tseq
);
5782 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5783 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5784 lower_omp (&tseq
, ctx
);
5785 gimple_seq_add_seq (&llist
[1], tseq
);
5786 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5787 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5788 if (new_vard
== new_var
)
5789 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5791 SET_DECL_VALUE_EXPR (new_vard
,
5792 build_fold_addr_expr (lvar
));
5793 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5795 gimplify_and_add (x
, &llist
[1]);
5798 /* If this is a reference to constant size reduction var
5799 with placeholder, we haven't emitted the initializer
5800 for it because it is undesirable if SIMD arrays are used.
5801 But if they aren't used, we need to emit the deferred
5802 initialization now. */
5803 else if (omp_is_reference (var
) && is_simd
)
5804 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5806 tree lab2
= NULL_TREE
;
5810 if (!is_parallel_ctx (ctx
))
5812 tree condv
= create_tmp_var (boolean_type_node
);
5813 tree m
= build_simple_mem_ref (cond
);
5814 g
= gimple_build_assign (condv
, m
);
5815 gimple_seq_add_stmt (ilist
, g
);
5817 = create_artificial_label (UNKNOWN_LOCATION
);
5818 lab2
= create_artificial_label (UNKNOWN_LOCATION
);
5819 g
= gimple_build_cond (NE_EXPR
, condv
,
5822 gimple_seq_add_stmt (ilist
, g
);
5823 gimple_seq_add_stmt (ilist
,
5824 gimple_build_label (lab1
));
5826 g
= gimple_build_assign (build_simple_mem_ref (cond
),
5828 gimple_seq_add_stmt (ilist
, g
);
5830 x
= lang_hooks
.decls
.omp_clause_default_ctor
5831 (c
, unshare_expr (new_var
),
5833 : build_outer_var_ref (var
, ctx
));
5835 gimplify_and_add (x
, ilist
);
5837 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
5838 && OMP_CLAUSE_REDUCTION_INSCAN (c
))
5840 if (ctx
->for_simd_scan_phase
)
5843 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
)))
5845 tree nv
= create_tmp_var_raw (TREE_TYPE (new_var
));
5846 gimple_add_tmp_var (nv
);
5847 ctx
->cb
.decl_map
->put (new_vard
, nv
);
5848 x
= lang_hooks
.decls
.omp_clause_default_ctor
5849 (c
, nv
, build_outer_var_ref (var
, ctx
));
5851 gimplify_and_add (x
, ilist
);
5852 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5854 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5856 if (new_vard
!= new_var
)
5857 vexpr
= build_fold_addr_expr (nv
);
5858 SET_DECL_VALUE_EXPR (new_vard
, vexpr
);
5859 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5860 lower_omp (&tseq
, ctx
);
5861 SET_DECL_VALUE_EXPR (new_vard
, NULL_TREE
);
5862 DECL_HAS_VALUE_EXPR_P (new_vard
) = 0;
5863 gimple_seq_add_seq (ilist
, tseq
);
5865 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5866 if (is_simd
&& ctx
->scan_exclusive
)
5869 = create_tmp_var_raw (TREE_TYPE (new_var
));
5870 gimple_add_tmp_var (nv2
);
5871 ctx
->cb
.decl_map
->put (nv
, nv2
);
5872 x
= lang_hooks
.decls
.omp_clause_default_ctor
5873 (c
, nv2
, build_outer_var_ref (var
, ctx
));
5874 gimplify_and_add (x
, ilist
);
5875 x
= lang_hooks
.decls
.omp_clause_dtor (c
, nv2
);
5877 gimplify_and_add (x
, dlist
);
5879 x
= lang_hooks
.decls
.omp_clause_dtor (c
, nv
);
5881 gimplify_and_add (x
, dlist
);
5884 && ctx
->scan_exclusive
5885 && TREE_ADDRESSABLE (TREE_TYPE (new_var
)))
5887 tree nv2
= create_tmp_var_raw (TREE_TYPE (new_var
));
5888 gimple_add_tmp_var (nv2
);
5889 ctx
->cb
.decl_map
->put (new_vard
, nv2
);
5890 x
= lang_hooks
.decls
.omp_clause_dtor (c
, nv2
);
5892 gimplify_and_add (x
, dlist
);
5894 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5898 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5900 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5901 lower_omp (&tseq
, ctx
);
5902 gimple_seq_add_seq (ilist
, tseq
);
5904 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5907 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5908 lower_omp (&tseq
, ctx
);
5909 gimple_seq_add_seq (dlist
, tseq
);
5910 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5912 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5916 gimple_seq_add_stmt (ilist
, gimple_build_label (lab2
));
5923 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
5924 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
5925 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5930 tree lab2
= NULL_TREE
;
5931 /* GOMP_taskgroup_reduction_register memsets the whole
5932 array to zero. If the initializer is zero, we don't
5933 need to initialize it again, just mark it as ever
5934 used unconditionally, i.e. cond = true. */
5935 if (initializer_zerop (x
))
5937 g
= gimple_build_assign (build_simple_mem_ref (cond
),
5939 gimple_seq_add_stmt (ilist
, g
);
5944 if (!cond) { cond = true; new_var = x; } */
5945 if (!is_parallel_ctx (ctx
))
5947 tree condv
= create_tmp_var (boolean_type_node
);
5948 tree m
= build_simple_mem_ref (cond
);
5949 g
= gimple_build_assign (condv
, m
);
5950 gimple_seq_add_stmt (ilist
, g
);
5952 = create_artificial_label (UNKNOWN_LOCATION
);
5953 lab2
= create_artificial_label (UNKNOWN_LOCATION
);
5954 g
= gimple_build_cond (NE_EXPR
, condv
,
5957 gimple_seq_add_stmt (ilist
, g
);
5958 gimple_seq_add_stmt (ilist
,
5959 gimple_build_label (lab1
));
5961 g
= gimple_build_assign (build_simple_mem_ref (cond
),
5963 gimple_seq_add_stmt (ilist
, g
);
5964 gimplify_assign (new_var
, x
, ilist
);
5966 gimple_seq_add_stmt (ilist
, gimple_build_label (lab2
));
5970 /* reduction(-:var) sums up the partial results, so it
5971 acts identically to reduction(+:var). */
5972 if (code
== MINUS_EXPR
)
5975 tree new_vard
= new_var
;
5976 if (is_simd
&& omp_is_reference (var
))
5978 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5979 new_vard
= TREE_OPERAND (new_var
, 0);
5980 gcc_assert (DECL_P (new_vard
));
5982 tree rvar
= NULL_TREE
, *rvarp
= NULL
, rvar2
= NULL_TREE
;
5984 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
5985 && OMP_CLAUSE_REDUCTION_INSCAN (c
))
5988 && lower_rec_simd_input_clauses (new_var
, ctx
, &sctx
,
5992 if (new_vard
!= new_var
)
5994 SET_DECL_VALUE_EXPR (new_vard
,
5995 build_fold_addr_expr (lvar
));
5996 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5999 tree ref
= build_outer_var_ref (var
, ctx
);
6003 if (ctx
->for_simd_scan_phase
)
6005 gimplify_assign (ivar
, ref
, &llist
[0]);
6006 ref
= build_outer_var_ref (var
, ctx
);
6007 gimplify_assign (ref
, rvar
, &llist
[3]);
6011 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
6016 simt_lane
= create_tmp_var (unsigned_type_node
);
6017 x
= build_call_expr_internal_loc
6018 (UNKNOWN_LOCATION
, IFN_GOMP_SIMT_XCHG_BFLY
,
6019 TREE_TYPE (ivar
), 2, ivar
, simt_lane
);
6020 x
= build2 (code
, TREE_TYPE (ivar
), ivar
, x
);
6021 gimplify_assign (ivar
, x
, &llist
[2]);
6023 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
6024 ref
= build_outer_var_ref (var
, ctx
);
6025 gimplify_assign (ref
, x
, &llist
[1]);
6030 if (omp_is_reference (var
) && is_simd
)
6031 handle_simd_reference (clause_loc
, new_vard
, ilist
);
6032 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
6033 && OMP_CLAUSE_REDUCTION_INSCAN (c
))
6035 gimplify_assign (new_var
, x
, ilist
);
6038 tree ref
= build_outer_var_ref (var
, ctx
);
6040 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
6041 ref
= build_outer_var_ref (var
, ctx
);
6042 gimplify_assign (ref
, x
, dlist
);
6055 tree clobber
= build_clobber (TREE_TYPE (tskred_avar
));
6056 gimple_seq_add_stmt (ilist
, gimple_build_assign (tskred_avar
, clobber
));
6059 if (known_eq (sctx
.max_vf
, 1U))
6061 sctx
.is_simt
= false;
6062 if (ctx
->lastprivate_conditional_map
)
6064 if (gimple_omp_for_combined_into_p (ctx
->stmt
))
6066 /* Signal to lower_omp_1 that it should use parent context. */
6067 ctx
->combined_into_simd_safelen1
= true;
6068 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6069 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
6070 && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c
))
6072 tree o
= lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
6073 omp_context
*outer
= ctx
->outer
;
6074 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_SCAN
)
6075 outer
= outer
->outer
;
6076 tree
*v
= ctx
->lastprivate_conditional_map
->get (o
);
6077 tree po
= lookup_decl (OMP_CLAUSE_DECL (c
), outer
);
6078 tree
*pv
= outer
->lastprivate_conditional_map
->get (po
);
6084 /* When not vectorized, treat lastprivate(conditional:) like
6085 normal lastprivate, as there will be just one simd lane
6086 writing the privatized variable. */
6087 delete ctx
->lastprivate_conditional_map
;
6088 ctx
->lastprivate_conditional_map
= NULL
;
6093 if (nonconst_simd_if
)
6095 if (sctx
.lane
== NULL_TREE
)
6097 sctx
.idx
= create_tmp_var (unsigned_type_node
);
6098 sctx
.lane
= create_tmp_var (unsigned_type_node
);
6100 /* FIXME: For now. */
6101 sctx
.is_simt
= false;
6104 if (sctx
.lane
|| sctx
.is_simt
)
6106 uid
= create_tmp_var (ptr_type_node
, "simduid");
6107 /* Don't want uninit warnings on simduid, it is always uninitialized,
6108 but we use it not for the value, but for the DECL_UID only. */
6109 TREE_NO_WARNING (uid
) = 1;
6110 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
6111 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
6112 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
6113 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
6115 /* Emit calls denoting privatized variables and initializing a pointer to
6116 structure that holds private variables as fields after ompdevlow pass. */
6119 sctx
.simt_eargs
[0] = uid
;
6121 = gimple_build_call_internal_vec (IFN_GOMP_SIMT_ENTER
, sctx
.simt_eargs
);
6122 gimple_call_set_lhs (g
, uid
);
6123 gimple_seq_add_stmt (ilist
, g
);
6124 sctx
.simt_eargs
.release ();
6126 simtrec
= create_tmp_var (ptr_type_node
, ".omp_simt");
6127 g
= gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC
, 1, uid
);
6128 gimple_call_set_lhs (g
, simtrec
);
6129 gimple_seq_add_stmt (ilist
, g
);
6133 gimple
*g
= gimple_build_call_internal (IFN_GOMP_SIMD_LANE
,
6134 2 + (nonconst_simd_if
!= NULL
),
6135 uid
, integer_zero_node
,
6137 gimple_call_set_lhs (g
, sctx
.lane
);
6138 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
6139 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
6140 g
= gimple_build_assign (sctx
.lane
, INTEGER_CST
,
6141 build_int_cst (unsigned_type_node
, 0));
6142 gimple_seq_add_stmt (ilist
, g
);
6145 g
= gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
6147 gimple_call_set_lhs (g
, sctx
.lastlane
);
6148 gimple_seq_add_stmt (dlist
, g
);
6149 gimple_seq_add_seq (dlist
, llist
[3]);
6151 /* Emit reductions across SIMT lanes in log_2(simt_vf) steps. */
6154 tree simt_vf
= create_tmp_var (unsigned_type_node
);
6155 g
= gimple_build_call_internal (IFN_GOMP_SIMT_VF
, 0);
6156 gimple_call_set_lhs (g
, simt_vf
);
6157 gimple_seq_add_stmt (dlist
, g
);
6159 tree t
= build_int_cst (unsigned_type_node
, 1);
6160 g
= gimple_build_assign (simt_lane
, INTEGER_CST
, t
);
6161 gimple_seq_add_stmt (dlist
, g
);
6163 t
= build_int_cst (unsigned_type_node
, 0);
6164 g
= gimple_build_assign (sctx
.idx
, INTEGER_CST
, t
);
6165 gimple_seq_add_stmt (dlist
, g
);
6167 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
6168 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
6169 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
6170 gimple_seq_add_stmt (dlist
, gimple_build_goto (header
));
6171 gimple_seq_add_stmt (dlist
, gimple_build_label (body
));
6173 gimple_seq_add_seq (dlist
, llist
[2]);
6175 g
= gimple_build_assign (simt_lane
, LSHIFT_EXPR
, simt_lane
, integer_one_node
);
6176 gimple_seq_add_stmt (dlist
, g
);
6178 gimple_seq_add_stmt (dlist
, gimple_build_label (header
));
6179 g
= gimple_build_cond (LT_EXPR
, simt_lane
, simt_vf
, body
, end
);
6180 gimple_seq_add_stmt (dlist
, g
);
6182 gimple_seq_add_stmt (dlist
, gimple_build_label (end
));
6184 for (int i
= 0; i
< 2; i
++)
6187 tree vf
= create_tmp_var (unsigned_type_node
);
6188 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
6189 gimple_call_set_lhs (g
, vf
);
6190 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
6191 gimple_seq_add_stmt (seq
, g
);
6192 tree t
= build_int_cst (unsigned_type_node
, 0);
6193 g
= gimple_build_assign (sctx
.idx
, INTEGER_CST
, t
);
6194 gimple_seq_add_stmt (seq
, g
);
6195 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
6196 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
6197 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
6198 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
6199 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
6200 gimple_seq_add_seq (seq
, llist
[i
]);
6201 t
= build_int_cst (unsigned_type_node
, 1);
6202 g
= gimple_build_assign (sctx
.idx
, PLUS_EXPR
, sctx
.idx
, t
);
6203 gimple_seq_add_stmt (seq
, g
);
6204 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
6205 g
= gimple_build_cond (LT_EXPR
, sctx
.idx
, vf
, body
, end
);
6206 gimple_seq_add_stmt (seq
, g
);
6207 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
6212 gimple_seq_add_seq (dlist
, sctx
.simt_dlist
);
6214 = gimple_build_call_internal (IFN_GOMP_SIMT_EXIT
, 1, simtrec
);
6215 gimple_seq_add_stmt (dlist
, g
);
6218 /* The copyin sequence is not to be executed by the main thread, since
6219 that would result in self-copies. Perhaps not visible to scalars,
6220 but it certainly is to C++ operator=. */
6223 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
6225 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
6226 build_int_cst (TREE_TYPE (x
), 0));
6227 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
6228 gimplify_and_add (x
, ilist
);
6231 /* If any copyin variable is passed by reference, we must ensure the
6232 master thread doesn't modify it before it is copied over in all
6233 threads. Similarly for variables in both firstprivate and
6234 lastprivate clauses we need to ensure the lastprivate copying
6235 happens after firstprivate copying in all threads. And similarly
6236 for UDRs if initializer expression refers to omp_orig. */
6237 if (copyin_by_ref
|| lastprivate_firstprivate
6238 || (reduction_omp_orig_ref
6239 && !ctx
->scan_inclusive
6240 && !ctx
->scan_exclusive
))
6242 /* Don't add any barrier for #pragma omp simd or
6243 #pragma omp distribute. */
6244 if (!is_task_ctx (ctx
)
6245 && (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
6246 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
))
6247 gimple_seq_add_stmt (ilist
, omp_build_barrier (NULL_TREE
));
6250 /* If max_vf is non-zero, then we can use only a vectorization factor
6251 up to the max_vf we chose. So stick it into the safelen clause. */
6252 if (maybe_ne (sctx
.max_vf
, 0U))
6254 tree c
= omp_find_clause (gimple_omp_for_clauses (ctx
->stmt
),
6255 OMP_CLAUSE_SAFELEN
);
6256 poly_uint64 safe_len
;
6258 || (poly_int_tree_p (OMP_CLAUSE_SAFELEN_EXPR (c
), &safe_len
)
6259 && maybe_gt (safe_len
, sctx
.max_vf
)))
6261 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
6262 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
6264 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
6265 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
6270 /* Create temporary variables for lastprivate(conditional:) implementation
6271 in context CTX with CLAUSES. */
6274 lower_lastprivate_conditional_clauses (tree
*clauses
, omp_context
*ctx
)
6276 tree iter_type
= NULL_TREE
;
6277 tree cond_ptr
= NULL_TREE
;
6278 tree iter_var
= NULL_TREE
;
6279 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
6280 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
);
6281 tree next
= *clauses
;
6282 for (tree c
= *clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6283 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
6284 && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c
))
6288 tree cc
= omp_find_clause (next
, OMP_CLAUSE__CONDTEMP_
);
6290 if (iter_type
== NULL_TREE
)
6292 iter_type
= TREE_TYPE (OMP_CLAUSE_DECL (cc
));
6293 iter_var
= create_tmp_var_raw (iter_type
);
6294 DECL_CONTEXT (iter_var
) = current_function_decl
;
6295 DECL_SEEN_IN_BIND_EXPR_P (iter_var
) = 1;
6296 DECL_CHAIN (iter_var
) = ctx
->block_vars
;
6297 ctx
->block_vars
= iter_var
;
6299 = build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__CONDTEMP_
);
6300 OMP_CLAUSE__CONDTEMP__ITER (c3
) = 1;
6301 OMP_CLAUSE_DECL (c3
) = iter_var
;
6302 OMP_CLAUSE_CHAIN (c3
) = *clauses
;
6304 ctx
->lastprivate_conditional_map
= new hash_map
<tree
, tree
>;
6306 next
= OMP_CLAUSE_CHAIN (cc
);
6307 tree o
= lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
6308 tree v
= lookup_decl (OMP_CLAUSE_DECL (cc
), ctx
);
6309 ctx
->lastprivate_conditional_map
->put (o
, v
);
6312 if (iter_type
== NULL
)
6314 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
)
6316 struct omp_for_data fd
;
6317 omp_extract_for_data (as_a
<gomp_for
*> (ctx
->stmt
), &fd
,
6319 iter_type
= unsigned_type_for (fd
.iter_type
);
6321 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
6322 iter_type
= unsigned_type_node
;
6323 tree c2
= omp_find_clause (*clauses
, OMP_CLAUSE__CONDTEMP_
);
6327 = lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c2
), ctx
);
6328 OMP_CLAUSE_DECL (c2
) = cond_ptr
;
6332 cond_ptr
= create_tmp_var_raw (build_pointer_type (iter_type
));
6333 DECL_CONTEXT (cond_ptr
) = current_function_decl
;
6334 DECL_SEEN_IN_BIND_EXPR_P (cond_ptr
) = 1;
6335 DECL_CHAIN (cond_ptr
) = ctx
->block_vars
;
6336 ctx
->block_vars
= cond_ptr
;
6337 c2
= build_omp_clause (UNKNOWN_LOCATION
,
6338 OMP_CLAUSE__CONDTEMP_
);
6339 OMP_CLAUSE_DECL (c2
) = cond_ptr
;
6340 OMP_CLAUSE_CHAIN (c2
) = *clauses
;
6343 iter_var
= create_tmp_var_raw (iter_type
);
6344 DECL_CONTEXT (iter_var
) = current_function_decl
;
6345 DECL_SEEN_IN_BIND_EXPR_P (iter_var
) = 1;
6346 DECL_CHAIN (iter_var
) = ctx
->block_vars
;
6347 ctx
->block_vars
= iter_var
;
6349 = build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__CONDTEMP_
);
6350 OMP_CLAUSE__CONDTEMP__ITER (c3
) = 1;
6351 OMP_CLAUSE_DECL (c3
) = iter_var
;
6352 OMP_CLAUSE_CHAIN (c3
) = OMP_CLAUSE_CHAIN (c2
);
6353 OMP_CLAUSE_CHAIN (c2
) = c3
;
6354 ctx
->lastprivate_conditional_map
= new hash_map
<tree
, tree
>;
6356 tree v
= create_tmp_var_raw (iter_type
);
6357 DECL_CONTEXT (v
) = current_function_decl
;
6358 DECL_SEEN_IN_BIND_EXPR_P (v
) = 1;
6359 DECL_CHAIN (v
) = ctx
->block_vars
;
6360 ctx
->block_vars
= v
;
6361 tree o
= lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
6362 ctx
->lastprivate_conditional_map
->put (o
, v
);
6367 /* Generate code to implement the LASTPRIVATE clauses. This is used for
6368 both parallel and workshare constructs. PREDICATE may be NULL if it's
6369 always true. BODY_P is the sequence to insert early initialization
6370 if needed, STMT_LIST is where the non-conditional lastprivate handling
6371 goes into and CSTMT_LIST is a sequence that needs to be run in a critical
6375 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*body_p
,
6376 gimple_seq
*stmt_list
, gimple_seq
*cstmt_list
,
6379 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
6380 bool par_clauses
= false;
6381 tree simduid
= NULL
, lastlane
= NULL
, simtcond
= NULL
, simtlast
= NULL
;
6382 unsigned HOST_WIDE_INT conditional_off
= 0;
6383 gimple_seq post_stmt_list
= NULL
;
6385 /* Early exit if there are no lastprivate or linear clauses. */
6386 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
6387 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
6388 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
6389 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
6391 if (clauses
== NULL
)
6393 /* If this was a workshare clause, see if it had been combined
6394 with its parallel. In that case, look for the clauses on the
6395 parallel statement itself. */
6396 if (is_parallel_ctx (ctx
))
6400 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
6403 clauses
= omp_find_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
6404 OMP_CLAUSE_LASTPRIVATE
);
6405 if (clauses
== NULL
)
6410 bool maybe_simt
= false;
6411 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
6412 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
6414 maybe_simt
= omp_find_clause (orig_clauses
, OMP_CLAUSE__SIMT_
);
6415 simduid
= omp_find_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
6417 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
6423 tree label_true
, arm1
, arm2
;
6424 enum tree_code pred_code
= TREE_CODE (predicate
);
6426 label
= create_artificial_label (UNKNOWN_LOCATION
);
6427 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
6428 if (TREE_CODE_CLASS (pred_code
) == tcc_comparison
)
6430 arm1
= TREE_OPERAND (predicate
, 0);
6431 arm2
= TREE_OPERAND (predicate
, 1);
6432 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
6433 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
6438 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
6439 arm2
= boolean_false_node
;
6440 pred_code
= NE_EXPR
;
6444 c
= build2 (pred_code
, boolean_type_node
, arm1
, arm2
);
6445 c
= fold_convert (integer_type_node
, c
);
6446 simtcond
= create_tmp_var (integer_type_node
);
6447 gimplify_assign (simtcond
, c
, stmt_list
);
6448 gcall
*g
= gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY
,
6450 c
= create_tmp_var (integer_type_node
);
6451 gimple_call_set_lhs (g
, c
);
6452 gimple_seq_add_stmt (stmt_list
, g
);
6453 stmt
= gimple_build_cond (NE_EXPR
, c
, integer_zero_node
,
6457 stmt
= gimple_build_cond (pred_code
, arm1
, arm2
, label_true
, label
);
6458 gimple_seq_add_stmt (stmt_list
, stmt
);
6459 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
6462 tree cond_ptr
= NULL_TREE
;
6463 for (c
= clauses
; c
;)
6466 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6467 gimple_seq
*this_stmt_list
= stmt_list
;
6468 tree lab2
= NULL_TREE
;
6470 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
6471 && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c
)
6472 && ctx
->lastprivate_conditional_map
6473 && !ctx
->combined_into_simd_safelen1
)
6475 gcc_assert (body_p
);
6478 if (cond_ptr
== NULL_TREE
)
6480 cond_ptr
= omp_find_clause (orig_clauses
, OMP_CLAUSE__CONDTEMP_
);
6481 cond_ptr
= OMP_CLAUSE_DECL (cond_ptr
);
6483 tree type
= TREE_TYPE (TREE_TYPE (cond_ptr
));
6484 tree o
= lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
6485 tree v
= *ctx
->lastprivate_conditional_map
->get (o
);
6486 gimplify_assign (v
, build_zero_cst (type
), body_p
);
6487 this_stmt_list
= cstmt_list
;
6489 if (POINTER_TYPE_P (TREE_TYPE (cond_ptr
)))
6491 mem
= build2 (MEM_REF
, type
, cond_ptr
,
6492 build_int_cst (TREE_TYPE (cond_ptr
),
6494 conditional_off
+= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
6497 mem
= build4 (ARRAY_REF
, type
, cond_ptr
,
6498 size_int (conditional_off
++), NULL_TREE
, NULL_TREE
);
6499 tree mem2
= copy_node (mem
);
6500 gimple_seq seq
= NULL
;
6501 mem
= force_gimple_operand (mem
, &seq
, true, NULL_TREE
);
6502 gimple_seq_add_seq (this_stmt_list
, seq
);
6503 tree lab1
= create_artificial_label (UNKNOWN_LOCATION
);
6504 lab2
= create_artificial_label (UNKNOWN_LOCATION
);
6505 gimple
*g
= gimple_build_cond (GT_EXPR
, v
, mem
, lab1
, lab2
);
6506 gimple_seq_add_stmt (this_stmt_list
, g
);
6507 gimple_seq_add_stmt (this_stmt_list
, gimple_build_label (lab1
));
6508 gimplify_assign (mem2
, v
, this_stmt_list
);
6511 && ctx
->combined_into_simd_safelen1
6512 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
6513 && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c
)
6514 && ctx
->lastprivate_conditional_map
)
6515 this_stmt_list
= &post_stmt_list
;
6517 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
6518 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
6519 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
6521 var
= OMP_CLAUSE_DECL (c
);
6522 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
6523 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
6524 && is_taskloop_ctx (ctx
))
6526 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
6527 new_var
= lookup_decl (var
, ctx
->outer
);
6531 new_var
= lookup_decl (var
, ctx
);
6532 /* Avoid uninitialized warnings for lastprivate and
6533 for linear iterators. */
6535 && (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
6536 || OMP_CLAUSE_LINEAR_NO_COPYIN (c
)))
6537 TREE_NO_WARNING (new_var
) = 1;
6540 if (!maybe_simt
&& simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
6542 tree val
= DECL_VALUE_EXPR (new_var
);
6543 if (TREE_CODE (val
) == ARRAY_REF
6544 && VAR_P (TREE_OPERAND (val
, 0))
6545 && lookup_attribute ("omp simd array",
6546 DECL_ATTRIBUTES (TREE_OPERAND (val
,
6549 if (lastlane
== NULL
)
6551 lastlane
= create_tmp_var (unsigned_type_node
);
6553 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
6555 TREE_OPERAND (val
, 1));
6556 gimple_call_set_lhs (g
, lastlane
);
6557 gimple_seq_add_stmt (this_stmt_list
, g
);
6559 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
6560 TREE_OPERAND (val
, 0), lastlane
,
6561 NULL_TREE
, NULL_TREE
);
6562 TREE_THIS_NOTRAP (new_var
) = 1;
6565 else if (maybe_simt
)
6567 tree val
= (DECL_HAS_VALUE_EXPR_P (new_var
)
6568 ? DECL_VALUE_EXPR (new_var
)
6570 if (simtlast
== NULL
)
6572 simtlast
= create_tmp_var (unsigned_type_node
);
6573 gcall
*g
= gimple_build_call_internal
6574 (IFN_GOMP_SIMT_LAST_LANE
, 1, simtcond
);
6575 gimple_call_set_lhs (g
, simtlast
);
6576 gimple_seq_add_stmt (this_stmt_list
, g
);
6578 x
= build_call_expr_internal_loc
6579 (UNKNOWN_LOCATION
, IFN_GOMP_SIMT_XCHG_IDX
,
6580 TREE_TYPE (val
), 2, val
, simtlast
);
6581 new_var
= unshare_expr (new_var
);
6582 gimplify_assign (new_var
, x
, this_stmt_list
);
6583 new_var
= unshare_expr (new_var
);
6586 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
6587 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
6589 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
6590 gimple_seq_add_seq (this_stmt_list
,
6591 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
6592 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
6594 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
6595 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
6597 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
6598 gimple_seq_add_seq (this_stmt_list
,
6599 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
6600 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
6604 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
6605 && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c
)
6606 && is_taskloop_ctx (ctx
))
6608 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
6610 if (is_global_var (ovar
))
6614 x
= build_outer_var_ref (var
, ctx
, OMP_CLAUSE_LASTPRIVATE
);
6615 if (omp_is_reference (var
))
6616 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
6617 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
6618 gimplify_and_add (x
, this_stmt_list
);
6621 gimple_seq_add_stmt (this_stmt_list
, gimple_build_label (lab2
));
6625 c
= OMP_CLAUSE_CHAIN (c
);
6626 if (c
== NULL
&& !par_clauses
)
6628 /* If this was a workshare clause, see if it had been combined
6629 with its parallel. In that case, continue looking for the
6630 clauses also on the parallel statement itself. */
6631 if (is_parallel_ctx (ctx
))
6635 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
6638 c
= omp_find_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
6639 OMP_CLAUSE_LASTPRIVATE
);
6645 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
6646 gimple_seq_add_seq (stmt_list
, post_stmt_list
);
6649 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
6650 (which might be a placeholder). INNER is true if this is an inner
6651 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
6652 join markers. Generate the before-loop forking sequence in
6653 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
6654 general form of these sequences is
6656 GOACC_REDUCTION_SETUP
6658 GOACC_REDUCTION_INIT
6660 GOACC_REDUCTION_FINI
6662 GOACC_REDUCTION_TEARDOWN. */
6665 lower_oacc_reductions (location_t loc
, tree clauses
, tree level
, bool inner
,
6666 gcall
*fork
, gcall
*join
, gimple_seq
*fork_seq
,
6667 gimple_seq
*join_seq
, omp_context
*ctx
)
6669 gimple_seq before_fork
= NULL
;
6670 gimple_seq after_fork
= NULL
;
6671 gimple_seq before_join
= NULL
;
6672 gimple_seq after_join
= NULL
;
6673 tree init_code
= NULL_TREE
, fini_code
= NULL_TREE
,
6674 setup_code
= NULL_TREE
, teardown_code
= NULL_TREE
;
6675 unsigned offset
= 0;
6677 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6678 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
6680 tree orig
= OMP_CLAUSE_DECL (c
);
6681 tree var
= maybe_lookup_decl (orig
, ctx
);
6682 tree ref_to_res
= NULL_TREE
;
6683 tree incoming
, outgoing
, v1
, v2
, v3
;
6684 bool is_private
= false;
6686 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
6687 if (rcode
== MINUS_EXPR
)
6689 else if (rcode
== TRUTH_ANDIF_EXPR
)
6690 rcode
= BIT_AND_EXPR
;
6691 else if (rcode
== TRUTH_ORIF_EXPR
)
6692 rcode
= BIT_IOR_EXPR
;
6693 tree op
= build_int_cst (unsigned_type_node
, rcode
);
6698 incoming
= outgoing
= var
;
6702 /* See if an outer construct also reduces this variable. */
6703 omp_context
*outer
= ctx
;
6705 while (omp_context
*probe
= outer
->outer
)
6707 enum gimple_code type
= gimple_code (probe
->stmt
);
6712 case GIMPLE_OMP_FOR
:
6713 cls
= gimple_omp_for_clauses (probe
->stmt
);
6716 case GIMPLE_OMP_TARGET
:
6717 if ((gimple_omp_target_kind (probe
->stmt
)
6718 != GF_OMP_TARGET_KIND_OACC_PARALLEL
)
6719 && (gimple_omp_target_kind (probe
->stmt
)
6720 != GF_OMP_TARGET_KIND_OACC_SERIAL
))
6723 cls
= gimple_omp_target_clauses (probe
->stmt
);
6731 for (; cls
; cls
= OMP_CLAUSE_CHAIN (cls
))
6732 if (OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_REDUCTION
6733 && orig
== OMP_CLAUSE_DECL (cls
))
6735 incoming
= outgoing
= lookup_decl (orig
, probe
);
6736 goto has_outer_reduction
;
6738 else if ((OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_FIRSTPRIVATE
6739 || OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_PRIVATE
)
6740 && orig
== OMP_CLAUSE_DECL (cls
))
6748 /* This is the outermost construct with this reduction,
6749 see if there's a mapping for it. */
6750 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_TARGET
6751 && maybe_lookup_field (orig
, outer
) && !is_private
)
6753 ref_to_res
= build_receiver_ref (orig
, false, outer
);
6754 if (omp_is_reference (orig
))
6755 ref_to_res
= build_simple_mem_ref (ref_to_res
);
6757 tree type
= TREE_TYPE (var
);
6758 if (POINTER_TYPE_P (type
))
6759 type
= TREE_TYPE (type
);
6762 incoming
= omp_reduction_init_op (loc
, rcode
, type
);
6766 /* Try to look at enclosing contexts for reduction var,
6767 use original if no mapping found. */
6769 omp_context
*c
= ctx
->outer
;
6772 t
= maybe_lookup_decl (orig
, c
);
6775 incoming
= outgoing
= (t
? t
: orig
);
6778 has_outer_reduction
:;
6782 ref_to_res
= integer_zero_node
;
6784 if (omp_is_reference (orig
))
6786 tree type
= TREE_TYPE (var
);
6787 const char *id
= IDENTIFIER_POINTER (DECL_NAME (var
));
6791 tree x
= create_tmp_var (TREE_TYPE (type
), id
);
6792 gimplify_assign (var
, build_fold_addr_expr (x
), fork_seq
);
6795 v1
= create_tmp_var (type
, id
);
6796 v2
= create_tmp_var (type
, id
);
6797 v3
= create_tmp_var (type
, id
);
6799 gimplify_assign (v1
, var
, fork_seq
);
6800 gimplify_assign (v2
, var
, fork_seq
);
6801 gimplify_assign (v3
, var
, fork_seq
);
6803 var
= build_simple_mem_ref (var
);
6804 v1
= build_simple_mem_ref (v1
);
6805 v2
= build_simple_mem_ref (v2
);
6806 v3
= build_simple_mem_ref (v3
);
6807 outgoing
= build_simple_mem_ref (outgoing
);
6809 if (!TREE_CONSTANT (incoming
))
6810 incoming
= build_simple_mem_ref (incoming
);
6815 /* Determine position in reduction buffer, which may be used
6816 by target. The parser has ensured that this is not a
6817 variable-sized type. */
6818 fixed_size_mode mode
6819 = as_a
<fixed_size_mode
> (TYPE_MODE (TREE_TYPE (var
)));
6820 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
6821 offset
= (offset
+ align
- 1) & ~(align
- 1);
6822 tree off
= build_int_cst (sizetype
, offset
);
6823 offset
+= GET_MODE_SIZE (mode
);
6827 init_code
= build_int_cst (integer_type_node
,
6828 IFN_GOACC_REDUCTION_INIT
);
6829 fini_code
= build_int_cst (integer_type_node
,
6830 IFN_GOACC_REDUCTION_FINI
);
6831 setup_code
= build_int_cst (integer_type_node
,
6832 IFN_GOACC_REDUCTION_SETUP
);
6833 teardown_code
= build_int_cst (integer_type_node
,
6834 IFN_GOACC_REDUCTION_TEARDOWN
);
6838 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
6839 TREE_TYPE (var
), 6, setup_code
,
6840 unshare_expr (ref_to_res
),
6841 incoming
, level
, op
, off
);
6843 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
6844 TREE_TYPE (var
), 6, init_code
,
6845 unshare_expr (ref_to_res
),
6846 v1
, level
, op
, off
);
6848 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
6849 TREE_TYPE (var
), 6, fini_code
,
6850 unshare_expr (ref_to_res
),
6851 v2
, level
, op
, off
);
6853 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
6854 TREE_TYPE (var
), 6, teardown_code
,
6855 ref_to_res
, v3
, level
, op
, off
);
6857 gimplify_assign (v1
, setup_call
, &before_fork
);
6858 gimplify_assign (v2
, init_call
, &after_fork
);
6859 gimplify_assign (v3
, fini_call
, &before_join
);
6860 gimplify_assign (outgoing
, teardown_call
, &after_join
);
6863 /* Now stitch things together. */
6864 gimple_seq_add_seq (fork_seq
, before_fork
);
6866 gimple_seq_add_stmt (fork_seq
, fork
);
6867 gimple_seq_add_seq (fork_seq
, after_fork
);
6869 gimple_seq_add_seq (join_seq
, before_join
);
6871 gimple_seq_add_stmt (join_seq
, join
);
6872 gimple_seq_add_seq (join_seq
, after_join
);
6875 /* Generate code to implement the REDUCTION clauses, append it
6876 to STMT_SEQP. CLIST if non-NULL is a pointer to a sequence
6877 that should be emitted also inside of the critical section,
6878 in that case clear *CLIST afterwards, otherwise leave it as is
6879 and let the caller emit it itself. */
6882 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
,
6883 gimple_seq
*clist
, omp_context
*ctx
)
6885 gimple_seq sub_seq
= NULL
;
6890 /* OpenACC loop reductions are handled elsewhere. */
6891 if (is_gimple_omp_oacc (ctx
->stmt
))
6894 /* SIMD reductions are handled in lower_rec_input_clauses. */
6895 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
6896 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
6899 /* inscan reductions are handled elsewhere. */
6900 if (ctx
->scan_inclusive
|| ctx
->scan_exclusive
)
6903 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
6904 update in that case, otherwise use a lock. */
6905 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
6906 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
6907 && !OMP_CLAUSE_REDUCTION_TASK (c
))
6909 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
6910 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
6912 /* Never use OMP_ATOMIC for array reductions or UDRs. */
6922 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6924 tree var
, ref
, new_var
, orig_var
;
6925 enum tree_code code
;
6926 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6928 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
6929 || OMP_CLAUSE_REDUCTION_TASK (c
))
6932 enum omp_clause_code ccode
= OMP_CLAUSE_REDUCTION
;
6933 orig_var
= var
= OMP_CLAUSE_DECL (c
);
6934 if (TREE_CODE (var
) == MEM_REF
)
6936 var
= TREE_OPERAND (var
, 0);
6937 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
6938 var
= TREE_OPERAND (var
, 0);
6939 if (TREE_CODE (var
) == ADDR_EXPR
)
6940 var
= TREE_OPERAND (var
, 0);
6943 /* If this is a pointer or referenced based array
6944 section, the var could be private in the outer
6945 context e.g. on orphaned loop construct. Pretend this
6946 is private variable's outer reference. */
6947 ccode
= OMP_CLAUSE_PRIVATE
;
6948 if (TREE_CODE (var
) == INDIRECT_REF
)
6949 var
= TREE_OPERAND (var
, 0);
6952 if (is_variable_sized (var
))
6954 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
6955 var
= DECL_VALUE_EXPR (var
);
6956 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
6957 var
= TREE_OPERAND (var
, 0);
6958 gcc_assert (DECL_P (var
));
6961 new_var
= lookup_decl (var
, ctx
);
6962 if (var
== OMP_CLAUSE_DECL (c
) && omp_is_reference (var
))
6963 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
6964 ref
= build_outer_var_ref (var
, ctx
, ccode
);
6965 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
6967 /* reduction(-:var) sums up the partial results, so it acts
6968 identically to reduction(+:var). */
6969 if (code
== MINUS_EXPR
)
6974 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
6976 addr
= save_expr (addr
);
6977 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
6978 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
6979 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
6980 OMP_ATOMIC_MEMORY_ORDER (x
) = OMP_MEMORY_ORDER_RELAXED
;
6981 gimplify_and_add (x
, stmt_seqp
);
6984 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
6986 tree d
= OMP_CLAUSE_DECL (c
);
6987 tree type
= TREE_TYPE (d
);
6988 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
6989 tree i
= create_tmp_var (TREE_TYPE (v
));
6990 tree ptype
= build_pointer_type (TREE_TYPE (type
));
6991 tree bias
= TREE_OPERAND (d
, 1);
6992 d
= TREE_OPERAND (d
, 0);
6993 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
6995 tree b
= TREE_OPERAND (d
, 1);
6996 b
= maybe_lookup_decl (b
, ctx
);
6999 b
= TREE_OPERAND (d
, 1);
7000 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
7002 if (integer_zerop (bias
))
7006 bias
= fold_convert_loc (clause_loc
, TREE_TYPE (b
), bias
);
7007 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
7008 TREE_TYPE (b
), b
, bias
);
7010 d
= TREE_OPERAND (d
, 0);
7012 /* For ref build_outer_var_ref already performs this, so
7013 only new_var needs a dereference. */
7014 if (TREE_CODE (d
) == INDIRECT_REF
)
7016 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
7017 gcc_assert (omp_is_reference (var
) && var
== orig_var
);
7019 else if (TREE_CODE (d
) == ADDR_EXPR
)
7021 if (orig_var
== var
)
7023 new_var
= build_fold_addr_expr (new_var
);
7024 ref
= build_fold_addr_expr (ref
);
7029 gcc_assert (orig_var
== var
);
7030 if (omp_is_reference (var
))
7031 ref
= build_fold_addr_expr (ref
);
7035 tree t
= maybe_lookup_decl (v
, ctx
);
7039 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
7040 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
7042 if (!integer_zerop (bias
))
7044 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
7045 new_var
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
7046 TREE_TYPE (new_var
), new_var
,
7047 unshare_expr (bias
));
7048 ref
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
7049 TREE_TYPE (ref
), ref
, bias
);
7051 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
7052 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
7053 tree m
= create_tmp_var (ptype
);
7054 gimplify_assign (m
, new_var
, stmt_seqp
);
7056 m
= create_tmp_var (ptype
);
7057 gimplify_assign (m
, ref
, stmt_seqp
);
7059 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
7060 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
7061 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
7062 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
7063 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
7064 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
7065 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
7067 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
7068 tree decl_placeholder
7069 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
7070 SET_DECL_VALUE_EXPR (placeholder
, out
);
7071 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
7072 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
7073 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
7074 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
7075 gimple_seq_add_seq (&sub_seq
,
7076 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
7077 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
7078 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
7079 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
7083 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
7084 out
= unshare_expr (out
);
7085 gimplify_assign (out
, x
, &sub_seq
);
7087 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
7088 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
7089 gimple_seq_add_stmt (&sub_seq
, g
);
7090 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
7091 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
7092 gimple_seq_add_stmt (&sub_seq
, g
);
7093 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
7094 build_int_cst (TREE_TYPE (i
), 1));
7095 gimple_seq_add_stmt (&sub_seq
, g
);
7096 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
7097 gimple_seq_add_stmt (&sub_seq
, g
);
7098 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
7100 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
7102 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
7104 if (omp_is_reference (var
)
7105 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
7107 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
7108 SET_DECL_VALUE_EXPR (placeholder
, ref
);
7109 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
7110 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
7111 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
7112 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
7113 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
7117 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
7118 ref
= build_outer_var_ref (var
, ctx
);
7119 gimplify_assign (ref
, x
, &sub_seq
);
7123 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
7125 gimple_seq_add_stmt (stmt_seqp
, stmt
);
7127 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
7131 gimple_seq_add_seq (stmt_seqp
, *clist
);
7135 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
7137 gimple_seq_add_stmt (stmt_seqp
, stmt
);
7141 /* Generate code to implement the COPYPRIVATE clauses. */
7144 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
7149 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
7151 tree var
, new_var
, ref
, x
;
7153 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
7155 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
7158 var
= OMP_CLAUSE_DECL (c
);
7159 by_ref
= use_pointer_for_field (var
, NULL
);
7161 ref
= build_sender_ref (var
, ctx
);
7162 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
7165 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
7166 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
7168 gimplify_assign (ref
, x
, slist
);
7170 ref
= build_receiver_ref (var
, false, ctx
);
7173 ref
= fold_convert_loc (clause_loc
,
7174 build_pointer_type (TREE_TYPE (new_var
)),
7176 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
7178 if (omp_is_reference (var
))
7180 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
7181 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
7182 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
7184 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
7185 gimplify_and_add (x
, rlist
);
7190 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
7191 and REDUCTION from the sender (aka parent) side. */
7194 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
7198 int ignored_looptemp
= 0;
7199 bool is_taskloop
= false;
7201 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
7202 by GOMP_taskloop. */
7203 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
7205 ignored_looptemp
= 2;
7209 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
7211 tree val
, ref
, x
, var
;
7212 bool by_ref
, do_in
= false, do_out
= false;
7213 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
7215 switch (OMP_CLAUSE_CODE (c
))
7217 case OMP_CLAUSE_PRIVATE
:
7218 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
7221 case OMP_CLAUSE_FIRSTPRIVATE
:
7222 case OMP_CLAUSE_COPYIN
:
7223 case OMP_CLAUSE_LASTPRIVATE
:
7224 case OMP_CLAUSE_IN_REDUCTION
:
7225 case OMP_CLAUSE__REDUCTEMP_
:
7227 case OMP_CLAUSE_REDUCTION
:
7228 if (is_task_ctx (ctx
) || OMP_CLAUSE_REDUCTION_TASK (c
))
7231 case OMP_CLAUSE_SHARED
:
7232 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
7235 case OMP_CLAUSE__LOOPTEMP_
:
7236 if (ignored_looptemp
)
7246 val
= OMP_CLAUSE_DECL (c
);
7247 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
7248 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IN_REDUCTION
)
7249 && TREE_CODE (val
) == MEM_REF
)
7251 val
= TREE_OPERAND (val
, 0);
7252 if (TREE_CODE (val
) == POINTER_PLUS_EXPR
)
7253 val
= TREE_OPERAND (val
, 0);
7254 if (TREE_CODE (val
) == INDIRECT_REF
7255 || TREE_CODE (val
) == ADDR_EXPR
)
7256 val
= TREE_OPERAND (val
, 0);
7257 if (is_variable_sized (val
))
7261 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
7262 outer taskloop region. */
7263 omp_context
*ctx_for_o
= ctx
;
7265 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
7266 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
7267 ctx_for_o
= ctx
->outer
;
7269 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
7271 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
7272 && is_global_var (var
)
7273 && (val
== OMP_CLAUSE_DECL (c
)
7274 || !is_task_ctx (ctx
)
7275 || (TREE_CODE (TREE_TYPE (val
)) != POINTER_TYPE
7276 && (TREE_CODE (TREE_TYPE (val
)) != REFERENCE_TYPE
7277 || (TREE_CODE (TREE_TYPE (TREE_TYPE (val
)))
7278 != POINTER_TYPE
)))))
7281 t
= omp_member_access_dummy_var (var
);
7284 var
= DECL_VALUE_EXPR (var
);
7285 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
7287 var
= unshare_and_remap (var
, t
, o
);
7289 var
= unshare_expr (var
);
7292 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
7294 /* Handle taskloop firstprivate/lastprivate, where the
7295 lastprivate on GIMPLE_OMP_TASK is represented as
7296 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
7297 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
7298 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
7299 if (use_pointer_for_field (val
, ctx
))
7300 var
= build_fold_addr_expr (var
);
7301 gimplify_assign (x
, var
, ilist
);
7302 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
7306 if (((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
7307 && OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_IN_REDUCTION
)
7308 || val
== OMP_CLAUSE_DECL (c
))
7309 && is_variable_sized (val
))
7311 by_ref
= use_pointer_for_field (val
, NULL
);
7313 switch (OMP_CLAUSE_CODE (c
))
7315 case OMP_CLAUSE_FIRSTPRIVATE
:
7316 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
)
7318 && is_task_ctx (ctx
))
7319 TREE_NO_WARNING (var
) = 1;
7323 case OMP_CLAUSE_PRIVATE
:
7324 case OMP_CLAUSE_COPYIN
:
7325 case OMP_CLAUSE__LOOPTEMP_
:
7326 case OMP_CLAUSE__REDUCTEMP_
:
7330 case OMP_CLAUSE_LASTPRIVATE
:
7331 if (by_ref
|| omp_is_reference (val
))
7333 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
7340 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
7345 case OMP_CLAUSE_REDUCTION
:
7346 case OMP_CLAUSE_IN_REDUCTION
:
7348 if (val
== OMP_CLAUSE_DECL (c
))
7350 if (is_task_ctx (ctx
))
7351 by_ref
= use_pointer_for_field (val
, ctx
);
7353 do_out
= !(by_ref
|| omp_is_reference (val
));
7356 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
7365 ref
= build_sender_ref (val
, ctx
);
7366 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
7367 gimplify_assign (ref
, x
, ilist
);
7368 if (is_task_ctx (ctx
))
7369 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
7374 ref
= build_sender_ref (val
, ctx
);
7375 gimplify_assign (var
, ref
, olist
);
7380 /* Generate code to implement SHARED from the sender (aka parent)
7381 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
7382 list things that got automatically shared. */
7385 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
7387 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
7389 if (ctx
->record_type
== NULL
)
7392 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
7393 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
7395 ovar
= DECL_ABSTRACT_ORIGIN (f
);
7396 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
7399 nvar
= maybe_lookup_decl (ovar
, ctx
);
7400 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
7403 /* If CTX is a nested parallel directive. Find the immediately
7404 enclosing parallel or workshare construct that contains a
7405 mapping for OVAR. */
7406 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
7408 t
= omp_member_access_dummy_var (var
);
7411 var
= DECL_VALUE_EXPR (var
);
7412 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
7414 var
= unshare_and_remap (var
, t
, o
);
7416 var
= unshare_expr (var
);
7419 if (use_pointer_for_field (ovar
, ctx
))
7421 x
= build_sender_ref (ovar
, ctx
);
7422 if (TREE_CODE (TREE_TYPE (f
)) == ARRAY_TYPE
7423 && TREE_TYPE (f
) == TREE_TYPE (ovar
))
7425 gcc_assert (is_parallel_ctx (ctx
)
7426 && DECL_ARTIFICIAL (ovar
));
7427 /* _condtemp_ clause. */
7428 var
= build_constructor (TREE_TYPE (x
), NULL
);
7431 var
= build_fold_addr_expr (var
);
7432 gimplify_assign (x
, var
, ilist
);
7436 x
= build_sender_ref (ovar
, ctx
);
7437 gimplify_assign (x
, var
, ilist
);
7439 if (!TREE_READONLY (var
)
7440 /* We don't need to receive a new reference to a result
7441 or parm decl. In fact we may not store to it as we will
7442 invalidate any pending RSO and generate wrong gimple
7444 && !((TREE_CODE (var
) == RESULT_DECL
7445 || TREE_CODE (var
) == PARM_DECL
)
7446 && DECL_BY_REFERENCE (var
)))
7448 x
= build_sender_ref (ovar
, ctx
);
7449 gimplify_assign (var
, x
, olist
);
7455 /* Emit an OpenACC head marker call, encapulating the partitioning and
7456 other information that must be processed by the target compiler.
7457 Return the maximum number of dimensions the associated loop might
7458 be partitioned over. */
7461 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
7462 gimple_seq
*seq
, omp_context
*ctx
)
7464 unsigned levels
= 0;
7466 tree gang_static
= NULL_TREE
;
7467 auto_vec
<tree
, 5> args
;
7469 args
.quick_push (build_int_cst
7470 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
7471 args
.quick_push (ddvar
);
7472 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
7474 switch (OMP_CLAUSE_CODE (c
))
7476 case OMP_CLAUSE_GANG
:
7477 tag
|= OLF_DIM_GANG
;
7478 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
7479 /* static:* is represented by -1, and we can ignore it, as
7480 scheduling is always static. */
7481 if (gang_static
&& integer_minus_onep (gang_static
))
7482 gang_static
= NULL_TREE
;
7486 case OMP_CLAUSE_WORKER
:
7487 tag
|= OLF_DIM_WORKER
;
7491 case OMP_CLAUSE_VECTOR
:
7492 tag
|= OLF_DIM_VECTOR
;
7496 case OMP_CLAUSE_SEQ
:
7500 case OMP_CLAUSE_AUTO
:
7504 case OMP_CLAUSE_INDEPENDENT
:
7505 tag
|= OLF_INDEPENDENT
;
7508 case OMP_CLAUSE_TILE
:
7519 if (DECL_P (gang_static
))
7520 gang_static
= build_outer_var_ref (gang_static
, ctx
);
7521 tag
|= OLF_GANG_STATIC
;
7524 /* In a parallel region, loops are implicitly INDEPENDENT. */
7525 omp_context
*tgt
= enclosing_target_ctx (ctx
);
7526 if (!tgt
|| is_oacc_parallel_or_serial (tgt
))
7527 tag
|= OLF_INDEPENDENT
;
7530 /* Tiling could use all 3 levels. */
7534 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR could be AUTO.
7535 Ensure at least one level, or 2 for possible auto
7537 bool maybe_auto
= !(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
7538 << OLF_DIM_BASE
) | OLF_SEQ
));
7540 if (levels
< 1u + maybe_auto
)
7541 levels
= 1u + maybe_auto
;
7544 args
.quick_push (build_int_cst (integer_type_node
, levels
));
7545 args
.quick_push (build_int_cst (integer_type_node
, tag
));
7547 args
.quick_push (gang_static
);
7549 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
7550 gimple_set_location (call
, loc
);
7551 gimple_set_lhs (call
, ddvar
);
7552 gimple_seq_add_stmt (seq
, call
);
7557 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
7558 partitioning level of the enclosed region. */
7561 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
7562 tree tofollow
, gimple_seq
*seq
)
7564 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
7565 : IFN_UNIQUE_OACC_TAIL_MARK
);
7566 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
7567 int nargs
= 2 + (tofollow
!= NULL_TREE
);
7568 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
7569 marker
, ddvar
, tofollow
);
7570 gimple_set_location (call
, loc
);
7571 gimple_set_lhs (call
, ddvar
);
7572 gimple_seq_add_stmt (seq
, call
);
7575 /* Generate the before and after OpenACC loop sequences. CLAUSES are
7576 the loop clauses, from which we extract reductions. Initialize
7580 lower_oacc_head_tail (location_t loc
, tree clauses
,
7581 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
7584 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
7585 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
7587 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
7588 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
7589 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
7592 for (unsigned done
= 1; count
; count
--, done
++)
7594 gimple_seq fork_seq
= NULL
;
7595 gimple_seq join_seq
= NULL
;
7597 tree place
= build_int_cst (integer_type_node
, -1);
7598 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
7599 fork_kind
, ddvar
, place
);
7600 gimple_set_location (fork
, loc
);
7601 gimple_set_lhs (fork
, ddvar
);
7603 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
7604 join_kind
, ddvar
, place
);
7605 gimple_set_location (join
, loc
);
7606 gimple_set_lhs (join
, ddvar
);
7608 /* Mark the beginning of this level sequence. */
7610 lower_oacc_loop_marker (loc
, ddvar
, true,
7611 build_int_cst (integer_type_node
, count
),
7613 lower_oacc_loop_marker (loc
, ddvar
, false,
7614 build_int_cst (integer_type_node
, done
),
7617 lower_oacc_reductions (loc
, clauses
, place
, inner
,
7618 fork
, join
, &fork_seq
, &join_seq
, ctx
);
7620 /* Append this level to head. */
7621 gimple_seq_add_seq (head
, fork_seq
);
7622 /* Prepend it to tail. */
7623 gimple_seq_add_seq (&join_seq
, *tail
);
7629 /* Mark the end of the sequence. */
7630 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
7631 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
7634 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
7635 catch handler and return it. This prevents programs from violating the
7636 structured block semantics with throws. */
7639 maybe_catch_exception (gimple_seq body
)
7644 if (!flag_exceptions
)
7647 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
7648 decl
= lang_hooks
.eh_protect_cleanup_actions ();
7650 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
7652 g
= gimple_build_eh_must_not_throw (decl
);
7653 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
7656 return gimple_seq_alloc_with_stmt (g
);
7660 /* Routines to lower OMP directives into OMP-GIMPLE. */
7662 /* If ctx is a worksharing context inside of a cancellable parallel
7663 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
7664 and conditional branch to parallel's cancel_label to handle
7665 cancellation in the implicit barrier. */
7668 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple
*omp_return
,
7671 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
7672 if (gimple_omp_return_nowait_p (omp_return
))
7674 for (omp_context
*outer
= ctx
->outer
; outer
; outer
= outer
->outer
)
7675 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_PARALLEL
7676 && outer
->cancellable
)
7678 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
7679 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
7680 tree lhs
= create_tmp_var (c_bool_type
);
7681 gimple_omp_return_set_lhs (omp_return
, lhs
);
7682 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
7683 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
7684 fold_convert (c_bool_type
,
7685 boolean_false_node
),
7686 outer
->cancel_label
, fallthru_label
);
7687 gimple_seq_add_stmt (body
, g
);
7688 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
7690 else if (gimple_code (outer
->stmt
) != GIMPLE_OMP_TASKGROUP
)
7694 /* Find the first task_reduction or reduction clause or return NULL
7695 if there are none. */
7698 omp_task_reductions_find_first (tree clauses
, enum tree_code code
,
7699 enum omp_clause_code ccode
)
7703 clauses
= omp_find_clause (clauses
, ccode
);
7704 if (clauses
== NULL_TREE
)
7706 if (ccode
!= OMP_CLAUSE_REDUCTION
7707 || code
== OMP_TASKLOOP
7708 || OMP_CLAUSE_REDUCTION_TASK (clauses
))
7710 clauses
= OMP_CLAUSE_CHAIN (clauses
);
7714 static void lower_omp_task_reductions (omp_context
*, enum tree_code
, tree
,
7715 gimple_seq
*, gimple_seq
*);
7717 /* Lower the OpenMP sections directive in the current statement in GSI_P.
7718 CTX is the enclosing OMP context for the current statement. */
7721 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
7723 tree block
, control
;
7724 gimple_stmt_iterator tgsi
;
7725 gomp_sections
*stmt
;
7727 gbind
*new_stmt
, *bind
;
7728 gimple_seq ilist
, dlist
, olist
, tred_dlist
= NULL
, clist
= NULL
, new_body
;
7730 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
7732 push_gimplify_context ();
7738 = omp_task_reductions_find_first (gimple_omp_sections_clauses (stmt
),
7739 OMP_SECTIONS
, OMP_CLAUSE_REDUCTION
);
7740 tree rtmp
= NULL_TREE
;
7743 tree type
= build_pointer_type (pointer_sized_int_node
);
7744 tree temp
= create_tmp_var (type
);
7745 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__REDUCTEMP_
);
7746 OMP_CLAUSE_DECL (c
) = temp
;
7747 OMP_CLAUSE_CHAIN (c
) = gimple_omp_sections_clauses (stmt
);
7748 gimple_omp_sections_set_clauses (stmt
, c
);
7749 lower_omp_task_reductions (ctx
, OMP_SECTIONS
,
7750 gimple_omp_sections_clauses (stmt
),
7751 &ilist
, &tred_dlist
);
7753 rtmp
= make_ssa_name (type
);
7754 gimple_seq_add_stmt (&ilist
, gimple_build_assign (rtmp
, temp
));
7757 tree
*clauses_ptr
= gimple_omp_sections_clauses_ptr (stmt
);
7758 lower_lastprivate_conditional_clauses (clauses_ptr
, ctx
);
7760 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
7761 &ilist
, &dlist
, ctx
, NULL
);
7763 control
= create_tmp_var (unsigned_type_node
, ".section");
7764 gimple_omp_sections_set_control (stmt
, control
);
7766 new_body
= gimple_omp_body (stmt
);
7767 gimple_omp_set_body (stmt
, NULL
);
7768 tgsi
= gsi_start (new_body
);
7769 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
7774 sec_start
= gsi_stmt (tgsi
);
7775 sctx
= maybe_lookup_ctx (sec_start
);
7778 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
7779 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
7780 GSI_CONTINUE_LINKING
);
7781 gimple_omp_set_body (sec_start
, NULL
);
7783 if (gsi_one_before_end_p (tgsi
))
7785 gimple_seq l
= NULL
;
7786 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
7787 &ilist
, &l
, &clist
, ctx
);
7788 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
7789 gimple_omp_section_set_last (sec_start
);
7792 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
7793 GSI_CONTINUE_LINKING
);
7796 block
= make_node (BLOCK
);
7797 bind
= gimple_build_bind (NULL
, new_body
, block
);
7800 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
,
7804 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7805 gcall
*g
= gimple_build_call (fndecl
, 0);
7806 gimple_seq_add_stmt (&olist
, g
);
7807 gimple_seq_add_seq (&olist
, clist
);
7808 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7809 g
= gimple_build_call (fndecl
, 0);
7810 gimple_seq_add_stmt (&olist
, g
);
7813 block
= make_node (BLOCK
);
7814 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
7815 gsi_replace (gsi_p
, new_stmt
, true);
7817 pop_gimplify_context (new_stmt
);
7818 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
7819 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
7820 if (BLOCK_VARS (block
))
7821 TREE_USED (block
) = 1;
7824 gimple_seq_add_seq (&new_body
, ilist
);
7825 gimple_seq_add_stmt (&new_body
, stmt
);
7826 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
7827 gimple_seq_add_stmt (&new_body
, bind
);
7829 t
= gimple_build_omp_continue (control
, control
);
7830 gimple_seq_add_stmt (&new_body
, t
);
7832 gimple_seq_add_seq (&new_body
, olist
);
7833 if (ctx
->cancellable
)
7834 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
7835 gimple_seq_add_seq (&new_body
, dlist
);
7837 new_body
= maybe_catch_exception (new_body
);
7839 bool nowait
= omp_find_clause (gimple_omp_sections_clauses (stmt
),
7840 OMP_CLAUSE_NOWAIT
) != NULL_TREE
;
7841 t
= gimple_build_omp_return (nowait
);
7842 gimple_seq_add_stmt (&new_body
, t
);
7843 gimple_seq_add_seq (&new_body
, tred_dlist
);
7844 maybe_add_implicit_barrier_cancel (ctx
, t
, &new_body
);
7847 OMP_CLAUSE_DECL (rclauses
) = rtmp
;
7849 gimple_bind_set_body (new_stmt
, new_body
);
7853 /* A subroutine of lower_omp_single. Expand the simple form of
7854 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
7856 if (GOMP_single_start ())
7858 [ GOMP_barrier (); ] -> unless 'nowait' is present.
7860 FIXME. It may be better to delay expanding the logic of this until
7861 pass_expand_omp. The expanded logic may make the job more difficult
7862 to a synchronization analysis pass. */
7865 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
7867 location_t loc
= gimple_location (single_stmt
);
7868 tree tlabel
= create_artificial_label (loc
);
7869 tree flabel
= create_artificial_label (loc
);
7870 gimple
*call
, *cond
;
7873 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
7874 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
7875 call
= gimple_build_call (decl
, 0);
7876 gimple_call_set_lhs (call
, lhs
);
7877 gimple_seq_add_stmt (pre_p
, call
);
7879 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
7880 fold_convert_loc (loc
, TREE_TYPE (lhs
),
7883 gimple_seq_add_stmt (pre_p
, cond
);
7884 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
7885 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
7886 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
7890 /* A subroutine of lower_omp_single. Expand the simple form of
7891 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
7893 #pragma omp single copyprivate (a, b, c)
7895 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
7898 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
7904 GOMP_single_copy_end (©out);
7915 FIXME. It may be better to delay expanding the logic of this until
7916 pass_expand_omp. The expanded logic may make the job more difficult
7917 to a synchronization analysis pass. */
7920 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
7923 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
7924 gimple_seq copyin_seq
;
7925 location_t loc
= gimple_location (single_stmt
);
7927 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
7929 ptr_type
= build_pointer_type (ctx
->record_type
);
7930 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
7932 l0
= create_artificial_label (loc
);
7933 l1
= create_artificial_label (loc
);
7934 l2
= create_artificial_label (loc
);
7936 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
7937 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
7938 t
= fold_convert_loc (loc
, ptr_type
, t
);
7939 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
7941 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
7942 build_int_cst (ptr_type
, 0));
7943 t
= build3 (COND_EXPR
, void_type_node
, t
,
7944 build_and_jump (&l0
), build_and_jump (&l1
));
7945 gimplify_and_add (t
, pre_p
);
7947 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
7949 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
7952 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
7955 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
7956 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
7957 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
7958 gimplify_and_add (t
, pre_p
);
7960 t
= build_and_jump (&l2
);
7961 gimplify_and_add (t
, pre_p
);
7963 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
7965 gimple_seq_add_seq (pre_p
, copyin_seq
);
7967 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
7971 /* Expand code for an OpenMP single directive. */
7974 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
7977 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
7979 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
7981 push_gimplify_context ();
7983 block
= make_node (BLOCK
);
7984 bind
= gimple_build_bind (NULL
, NULL
, block
);
7985 gsi_replace (gsi_p
, bind
, true);
7988 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
7989 &bind_body
, &dlist
, ctx
, NULL
);
7990 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
7992 gimple_seq_add_stmt (&bind_body
, single_stmt
);
7994 if (ctx
->record_type
)
7995 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
7997 lower_omp_single_simple (single_stmt
, &bind_body
);
7999 gimple_omp_set_body (single_stmt
, NULL
);
8001 gimple_seq_add_seq (&bind_body
, dlist
);
8003 bind_body
= maybe_catch_exception (bind_body
);
8005 bool nowait
= omp_find_clause (gimple_omp_single_clauses (single_stmt
),
8006 OMP_CLAUSE_NOWAIT
) != NULL_TREE
;
8007 gimple
*g
= gimple_build_omp_return (nowait
);
8008 gimple_seq_add_stmt (&bind_body_tail
, g
);
8009 maybe_add_implicit_barrier_cancel (ctx
, g
, &bind_body_tail
);
8010 if (ctx
->record_type
)
8012 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8013 tree clobber
= build_clobber (ctx
->record_type
);
8014 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8015 clobber
), GSI_SAME_STMT
);
8017 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8018 gimple_bind_set_body (bind
, bind_body
);
8020 pop_gimplify_context (bind
);
8022 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8023 BLOCK_VARS (block
) = ctx
->block_vars
;
8024 if (BLOCK_VARS (block
))
8025 TREE_USED (block
) = 1;
8029 /* Expand code for an OpenMP master directive. */
8032 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8034 tree block
, lab
= NULL
, x
, bfn_decl
;
8035 gimple
*stmt
= gsi_stmt (*gsi_p
);
8037 location_t loc
= gimple_location (stmt
);
8040 push_gimplify_context ();
8042 block
= make_node (BLOCK
);
8043 bind
= gimple_build_bind (NULL
, NULL
, block
);
8044 gsi_replace (gsi_p
, bind
, true);
8045 gimple_bind_add_stmt (bind
, stmt
);
8047 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8048 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8049 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8050 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8052 gimplify_and_add (x
, &tseq
);
8053 gimple_bind_add_seq (bind
, tseq
);
8055 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8056 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8057 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8058 gimple_omp_set_body (stmt
, NULL
);
8060 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8062 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8064 pop_gimplify_context (bind
);
8066 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8067 BLOCK_VARS (block
) = ctx
->block_vars
;
8070 /* Helper function for lower_omp_task_reductions. For a specific PASS
8071 find out the current clause it should be processed, or return false
8072 if all have been processed already. */
8075 omp_task_reduction_iterate (int pass
, enum tree_code code
,
8076 enum omp_clause_code ccode
, tree
*c
, tree
*decl
,
8077 tree
*type
, tree
*next
)
8079 for (; *c
; *c
= omp_find_clause (OMP_CLAUSE_CHAIN (*c
), ccode
))
8081 if (ccode
== OMP_CLAUSE_REDUCTION
8082 && code
!= OMP_TASKLOOP
8083 && !OMP_CLAUSE_REDUCTION_TASK (*c
))
8085 *decl
= OMP_CLAUSE_DECL (*c
);
8086 *type
= TREE_TYPE (*decl
);
8087 if (TREE_CODE (*decl
) == MEM_REF
)
8094 if (omp_is_reference (*decl
))
8095 *type
= TREE_TYPE (*type
);
8096 if (pass
!= (!TREE_CONSTANT (TYPE_SIZE_UNIT (*type
))))
8099 *next
= omp_find_clause (OMP_CLAUSE_CHAIN (*c
), ccode
);
8108 /* Lower task_reduction and reduction clauses (the latter unless CODE is
8109 OMP_TASKGROUP only with task modifier). Register mapping of those in
8110 START sequence and reducing them and unregister them in the END sequence. */
8113 lower_omp_task_reductions (omp_context
*ctx
, enum tree_code code
, tree clauses
,
8114 gimple_seq
*start
, gimple_seq
*end
)
8116 enum omp_clause_code ccode
8117 = (code
== OMP_TASKGROUP
8118 ? OMP_CLAUSE_TASK_REDUCTION
: OMP_CLAUSE_REDUCTION
);
8119 tree cancellable
= NULL_TREE
;
8120 clauses
= omp_task_reductions_find_first (clauses
, code
, ccode
);
8121 if (clauses
== NULL_TREE
)
8123 if (code
== OMP_FOR
|| code
== OMP_SECTIONS
)
8125 for (omp_context
*outer
= ctx
->outer
; outer
; outer
= outer
->outer
)
8126 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_PARALLEL
8127 && outer
->cancellable
)
8129 cancellable
= error_mark_node
;
8132 else if (gimple_code (outer
->stmt
) != GIMPLE_OMP_TASKGROUP
)
8135 tree record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
8136 tree
*last
= &TYPE_FIELDS (record_type
);
8140 tree field
= build_decl (UNKNOWN_LOCATION
, FIELD_DECL
, NULL_TREE
,
8142 tree ifield
= build_decl (UNKNOWN_LOCATION
, FIELD_DECL
, NULL_TREE
,
8145 DECL_CHAIN (field
) = ifield
;
8146 last
= &DECL_CHAIN (ifield
);
8147 DECL_CONTEXT (field
) = record_type
;
8148 if (TYPE_ALIGN (record_type
) < DECL_ALIGN (field
))
8149 SET_TYPE_ALIGN (record_type
, DECL_ALIGN (field
));
8150 DECL_CONTEXT (ifield
) = record_type
;
8151 if (TYPE_ALIGN (record_type
) < DECL_ALIGN (ifield
))
8152 SET_TYPE_ALIGN (record_type
, DECL_ALIGN (ifield
));
8154 for (int pass
= 0; pass
< 2; pass
++)
8156 tree decl
, type
, next
;
8157 for (tree c
= clauses
;
8158 omp_task_reduction_iterate (pass
, code
, ccode
,
8159 &c
, &decl
, &type
, &next
); c
= next
)
8162 tree new_type
= type
;
8164 new_type
= remap_type (type
, &ctx
->outer
->cb
);
8166 = build_decl (OMP_CLAUSE_LOCATION (c
), FIELD_DECL
,
8167 DECL_P (decl
) ? DECL_NAME (decl
) : NULL_TREE
,
8169 if (DECL_P (decl
) && type
== TREE_TYPE (decl
))
8171 SET_DECL_ALIGN (field
, DECL_ALIGN (decl
));
8172 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (decl
);
8173 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (decl
);
8176 SET_DECL_ALIGN (field
, TYPE_ALIGN (type
));
8177 DECL_CONTEXT (field
) = record_type
;
8178 if (TYPE_ALIGN (record_type
) < DECL_ALIGN (field
))
8179 SET_TYPE_ALIGN (record_type
, DECL_ALIGN (field
));
8181 last
= &DECL_CHAIN (field
);
8183 = build_decl (OMP_CLAUSE_LOCATION (c
), FIELD_DECL
, NULL_TREE
,
8185 DECL_CONTEXT (bfield
) = record_type
;
8186 if (TYPE_ALIGN (record_type
) < DECL_ALIGN (bfield
))
8187 SET_TYPE_ALIGN (record_type
, DECL_ALIGN (bfield
));
8189 last
= &DECL_CHAIN (bfield
);
8193 layout_type (record_type
);
8195 /* Build up an array which registers with the runtime all the reductions
8196 and deregisters them at the end. Format documented in libgomp/task.c. */
8197 tree atype
= build_array_type_nelts (pointer_sized_int_node
, 7 + cnt
* 3);
8198 tree avar
= create_tmp_var_raw (atype
);
8199 gimple_add_tmp_var (avar
);
8200 TREE_ADDRESSABLE (avar
) = 1;
8201 tree r
= build4 (ARRAY_REF
, pointer_sized_int_node
, avar
, size_zero_node
,
8202 NULL_TREE
, NULL_TREE
);
8203 tree t
= build_int_cst (pointer_sized_int_node
, cnt
);
8204 gimple_seq_add_stmt (start
, gimple_build_assign (r
, t
));
8205 gimple_seq seq
= NULL
;
8206 tree sz
= fold_convert (pointer_sized_int_node
,
8207 TYPE_SIZE_UNIT (record_type
));
8209 sz
= fold_build2 (PLUS_EXPR
, pointer_sized_int_node
, sz
,
8210 build_int_cst (pointer_sized_int_node
, cachesz
- 1));
8211 sz
= fold_build2 (BIT_AND_EXPR
, pointer_sized_int_node
, sz
,
8212 build_int_cst (pointer_sized_int_node
, ~(cachesz
- 1)));
8213 ctx
->task_reductions
.create (1 + cnt
);
8214 ctx
->task_reduction_map
= new hash_map
<tree
, unsigned>;
8215 ctx
->task_reductions
.quick_push (TREE_CODE (sz
) == INTEGER_CST
8217 sz
= force_gimple_operand (sz
, &seq
, true, NULL_TREE
);
8218 gimple_seq_add_seq (start
, seq
);
8219 r
= build4 (ARRAY_REF
, pointer_sized_int_node
, avar
, size_one_node
,
8220 NULL_TREE
, NULL_TREE
);
8221 gimple_seq_add_stmt (start
, gimple_build_assign (r
, sz
));
8222 r
= build4 (ARRAY_REF
, pointer_sized_int_node
, avar
, size_int (2),
8223 NULL_TREE
, NULL_TREE
);
8224 t
= build_int_cst (pointer_sized_int_node
,
8225 MAX (TYPE_ALIGN_UNIT (record_type
), (unsigned) cachesz
));
8226 gimple_seq_add_stmt (start
, gimple_build_assign (r
, t
));
8227 r
= build4 (ARRAY_REF
, pointer_sized_int_node
, avar
, size_int (3),
8228 NULL_TREE
, NULL_TREE
);
8229 t
= build_int_cst (pointer_sized_int_node
, -1);
8230 gimple_seq_add_stmt (start
, gimple_build_assign (r
, t
));
8231 r
= build4 (ARRAY_REF
, pointer_sized_int_node
, avar
, size_int (4),
8232 NULL_TREE
, NULL_TREE
);
8233 t
= build_int_cst (pointer_sized_int_node
, 0);
8234 gimple_seq_add_stmt (start
, gimple_build_assign (r
, t
));
8236 /* In end, build a loop that iterates from 0 to < omp_get_num_threads ()
8237 and for each task reduction checks a bool right after the private variable
8238 within that thread's chunk; if the bool is clear, it hasn't been
8239 initialized and thus isn't going to be reduced nor destructed, otherwise
8240 reduce and destruct it. */
8241 tree idx
= create_tmp_var (size_type_node
);
8242 gimple_seq_add_stmt (end
, gimple_build_assign (idx
, size_zero_node
));
8243 tree num_thr_sz
= create_tmp_var (size_type_node
);
8244 tree lab1
= create_artificial_label (UNKNOWN_LOCATION
);
8245 tree lab2
= create_artificial_label (UNKNOWN_LOCATION
);
8246 tree lab3
= NULL_TREE
;
8248 if (code
== OMP_FOR
|| code
== OMP_SECTIONS
)
8250 /* For worksharing constructs, only perform it in the master thread,
8251 with the exception of cancelled implicit barriers - then only handle
8252 the current thread. */
8253 tree lab4
= create_artificial_label (UNKNOWN_LOCATION
);
8254 t
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8255 tree thr_num
= create_tmp_var (integer_type_node
);
8256 g
= gimple_build_call (t
, 0);
8257 gimple_call_set_lhs (g
, thr_num
);
8258 gimple_seq_add_stmt (end
, g
);
8262 tree lab5
= create_artificial_label (UNKNOWN_LOCATION
);
8263 tree lab6
= create_artificial_label (UNKNOWN_LOCATION
);
8264 lab3
= create_artificial_label (UNKNOWN_LOCATION
);
8265 if (code
== OMP_FOR
)
8266 c
= gimple_omp_for_clauses (ctx
->stmt
);
8267 else /* if (code == OMP_SECTIONS) */
8268 c
= gimple_omp_sections_clauses (ctx
->stmt
);
8269 c
= OMP_CLAUSE_DECL (omp_find_clause (c
, OMP_CLAUSE__REDUCTEMP_
));
8271 g
= gimple_build_cond (NE_EXPR
, c
, build_zero_cst (TREE_TYPE (c
)),
8273 gimple_seq_add_stmt (end
, g
);
8274 gimple_seq_add_stmt (end
, gimple_build_label (lab5
));
8275 g
= gimple_build_assign (idx
, NOP_EXPR
, thr_num
);
8276 gimple_seq_add_stmt (end
, g
);
8277 g
= gimple_build_assign (num_thr_sz
, PLUS_EXPR
, idx
,
8278 build_one_cst (TREE_TYPE (idx
)));
8279 gimple_seq_add_stmt (end
, g
);
8280 gimple_seq_add_stmt (end
, gimple_build_goto (lab3
));
8281 gimple_seq_add_stmt (end
, gimple_build_label (lab6
));
8283 g
= gimple_build_cond (NE_EXPR
, thr_num
, integer_zero_node
, lab2
, lab4
);
8284 gimple_seq_add_stmt (end
, g
);
8285 gimple_seq_add_stmt (end
, gimple_build_label (lab4
));
8287 if (code
!= OMP_PARALLEL
)
8289 t
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
8290 tree num_thr
= create_tmp_var (integer_type_node
);
8291 g
= gimple_build_call (t
, 0);
8292 gimple_call_set_lhs (g
, num_thr
);
8293 gimple_seq_add_stmt (end
, g
);
8294 g
= gimple_build_assign (num_thr_sz
, NOP_EXPR
, num_thr
);
8295 gimple_seq_add_stmt (end
, g
);
8297 gimple_seq_add_stmt (end
, gimple_build_label (lab3
));
8301 tree c
= omp_find_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
8302 OMP_CLAUSE__REDUCTEMP_
);
8303 t
= fold_convert (pointer_sized_int_node
, OMP_CLAUSE_DECL (c
));
8304 t
= fold_convert (size_type_node
, t
);
8305 gimplify_assign (num_thr_sz
, t
, end
);
8307 t
= build4 (ARRAY_REF
, pointer_sized_int_node
, avar
, size_int (2),
8308 NULL_TREE
, NULL_TREE
);
8309 tree data
= create_tmp_var (pointer_sized_int_node
);
8310 gimple_seq_add_stmt (end
, gimple_build_assign (data
, t
));
8311 gimple_seq_add_stmt (end
, gimple_build_label (lab1
));
8313 if (TREE_CODE (TYPE_SIZE_UNIT (record_type
)) == INTEGER_CST
)
8314 ptr
= create_tmp_var (build_pointer_type (record_type
));
8316 ptr
= create_tmp_var (ptr_type_node
);
8317 gimple_seq_add_stmt (end
, gimple_build_assign (ptr
, NOP_EXPR
, data
));
8319 tree field
= TYPE_FIELDS (record_type
);
8322 field
= DECL_CHAIN (DECL_CHAIN (field
));
8323 for (int pass
= 0; pass
< 2; pass
++)
8325 tree decl
, type
, next
;
8326 for (tree c
= clauses
;
8327 omp_task_reduction_iterate (pass
, code
, ccode
,
8328 &c
, &decl
, &type
, &next
); c
= next
)
8330 tree var
= decl
, ref
;
8331 if (TREE_CODE (decl
) == MEM_REF
)
8333 var
= TREE_OPERAND (var
, 0);
8334 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
8335 var
= TREE_OPERAND (var
, 0);
8337 if (TREE_CODE (var
) == ADDR_EXPR
)
8338 var
= TREE_OPERAND (var
, 0);
8339 else if (TREE_CODE (var
) == INDIRECT_REF
)
8340 var
= TREE_OPERAND (var
, 0);
8341 tree orig_var
= var
;
8342 if (is_variable_sized (var
))
8344 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
8345 var
= DECL_VALUE_EXPR (var
);
8346 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
8347 var
= TREE_OPERAND (var
, 0);
8348 gcc_assert (DECL_P (var
));
8350 t
= ref
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
8351 if (orig_var
!= var
)
8352 gcc_assert (TREE_CODE (v
) == ADDR_EXPR
);
8353 else if (TREE_CODE (v
) == ADDR_EXPR
)
8354 t
= build_fold_addr_expr (t
);
8355 else if (TREE_CODE (v
) == INDIRECT_REF
)
8356 t
= build_fold_indirect_ref (t
);
8357 if (TREE_CODE (TREE_OPERAND (decl
, 0)) == POINTER_PLUS_EXPR
)
8359 tree b
= TREE_OPERAND (TREE_OPERAND (decl
, 0), 1);
8360 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
8361 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, b
);
8363 if (!integer_zerop (TREE_OPERAND (decl
, 1)))
8364 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
,
8365 fold_convert (size_type_node
,
8366 TREE_OPERAND (decl
, 1)));
8370 t
= ref
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
8371 if (!omp_is_reference (decl
))
8372 t
= build_fold_addr_expr (t
);
8374 t
= fold_convert (pointer_sized_int_node
, t
);
8376 t
= force_gimple_operand (t
, &seq
, true, NULL_TREE
);
8377 gimple_seq_add_seq (start
, seq
);
8378 r
= build4 (ARRAY_REF
, pointer_sized_int_node
, avar
,
8379 size_int (7 + cnt
* 3), NULL_TREE
, NULL_TREE
);
8380 gimple_seq_add_stmt (start
, gimple_build_assign (r
, t
));
8381 t
= unshare_expr (byte_position (field
));
8382 t
= fold_convert (pointer_sized_int_node
, t
);
8383 ctx
->task_reduction_map
->put (c
, cnt
);
8384 ctx
->task_reductions
.quick_push (TREE_CODE (t
) == INTEGER_CST
8387 t
= force_gimple_operand (t
, &seq
, true, NULL_TREE
);
8388 gimple_seq_add_seq (start
, seq
);
8389 r
= build4 (ARRAY_REF
, pointer_sized_int_node
, avar
,
8390 size_int (7 + cnt
* 3 + 1), NULL_TREE
, NULL_TREE
);
8391 gimple_seq_add_stmt (start
, gimple_build_assign (r
, t
));
8393 tree bfield
= DECL_CHAIN (field
);
8395 if (code
== OMP_PARALLEL
|| code
== OMP_FOR
|| code
== OMP_SECTIONS
)
8396 /* In parallel or worksharing all threads unconditionally
8397 initialize all their task reduction private variables. */
8398 cond
= boolean_true_node
;
8399 else if (TREE_TYPE (ptr
) == ptr_type_node
)
8401 cond
= build2 (POINTER_PLUS_EXPR
, ptr_type_node
, ptr
,
8402 unshare_expr (byte_position (bfield
)));
8404 cond
= force_gimple_operand (cond
, &seq
, true, NULL_TREE
);
8405 gimple_seq_add_seq (end
, seq
);
8406 tree pbool
= build_pointer_type (TREE_TYPE (bfield
));
8407 cond
= build2 (MEM_REF
, TREE_TYPE (bfield
), cond
,
8408 build_int_cst (pbool
, 0));
8411 cond
= build3 (COMPONENT_REF
, TREE_TYPE (bfield
),
8412 build_simple_mem_ref (ptr
), bfield
, NULL_TREE
);
8413 tree lab3
= create_artificial_label (UNKNOWN_LOCATION
);
8414 tree lab4
= create_artificial_label (UNKNOWN_LOCATION
);
8415 tree condv
= create_tmp_var (boolean_type_node
);
8416 gimple_seq_add_stmt (end
, gimple_build_assign (condv
, cond
));
8417 g
= gimple_build_cond (NE_EXPR
, condv
, boolean_false_node
,
8419 gimple_seq_add_stmt (end
, g
);
8420 gimple_seq_add_stmt (end
, gimple_build_label (lab3
));
8421 if (cancellable
&& OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) == NULL_TREE
)
8423 /* If this reduction doesn't need destruction and parallel
8424 has been cancelled, there is nothing to do for this
8425 reduction, so jump around the merge operation. */
8426 tree lab5
= create_artificial_label (UNKNOWN_LOCATION
);
8427 g
= gimple_build_cond (NE_EXPR
, cancellable
,
8428 build_zero_cst (TREE_TYPE (cancellable
)),
8430 gimple_seq_add_stmt (end
, g
);
8431 gimple_seq_add_stmt (end
, gimple_build_label (lab5
));
8435 if (TREE_TYPE (ptr
) == ptr_type_node
)
8437 new_var
= build2 (POINTER_PLUS_EXPR
, ptr_type_node
, ptr
,
8438 unshare_expr (byte_position (field
)));
8440 new_var
= force_gimple_operand (new_var
, &seq
, true, NULL_TREE
);
8441 gimple_seq_add_seq (end
, seq
);
8442 tree pbool
= build_pointer_type (TREE_TYPE (field
));
8443 new_var
= build2 (MEM_REF
, TREE_TYPE (field
), new_var
,
8444 build_int_cst (pbool
, 0));
8447 new_var
= build3 (COMPONENT_REF
, TREE_TYPE (field
),
8448 build_simple_mem_ref (ptr
), field
, NULL_TREE
);
8450 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
8451 if (TREE_CODE (decl
) != MEM_REF
&& omp_is_reference (decl
))
8452 ref
= build_simple_mem_ref (ref
);
8453 /* reduction(-:var) sums up the partial results, so it acts
8454 identically to reduction(+:var). */
8455 if (rcode
== MINUS_EXPR
)
8457 if (TREE_CODE (decl
) == MEM_REF
)
8459 tree type
= TREE_TYPE (new_var
);
8460 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
8461 tree i
= create_tmp_var (TREE_TYPE (v
));
8462 tree ptype
= build_pointer_type (TREE_TYPE (type
));
8465 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
8466 tree vv
= create_tmp_var (TREE_TYPE (v
));
8467 gimplify_assign (vv
, v
, start
);
8470 ref
= build4 (ARRAY_REF
, pointer_sized_int_node
, avar
,
8471 size_int (7 + cnt
* 3), NULL_TREE
, NULL_TREE
);
8472 new_var
= build_fold_addr_expr (new_var
);
8473 new_var
= fold_convert (ptype
, new_var
);
8474 ref
= fold_convert (ptype
, ref
);
8475 tree m
= create_tmp_var (ptype
);
8476 gimplify_assign (m
, new_var
, end
);
8478 m
= create_tmp_var (ptype
);
8479 gimplify_assign (m
, ref
, end
);
8481 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), end
);
8482 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
8483 tree endl
= create_artificial_label (UNKNOWN_LOCATION
);
8484 gimple_seq_add_stmt (end
, gimple_build_label (body
));
8485 tree priv
= build_simple_mem_ref (new_var
);
8486 tree out
= build_simple_mem_ref (ref
);
8487 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
8489 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
8490 tree decl_placeholder
8491 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
8492 tree lab6
= NULL_TREE
;
8495 /* If this reduction needs destruction and parallel
8496 has been cancelled, jump around the merge operation
8497 to the destruction. */
8498 tree lab5
= create_artificial_label (UNKNOWN_LOCATION
);
8499 lab6
= create_artificial_label (UNKNOWN_LOCATION
);
8500 tree zero
= build_zero_cst (TREE_TYPE (cancellable
));
8501 g
= gimple_build_cond (NE_EXPR
, cancellable
, zero
,
8503 gimple_seq_add_stmt (end
, g
);
8504 gimple_seq_add_stmt (end
, gimple_build_label (lab5
));
8506 SET_DECL_VALUE_EXPR (placeholder
, out
);
8507 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
8508 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
8509 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
8510 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
8511 gimple_seq_add_seq (end
,
8512 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
8513 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
8514 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_TASK_REDUCTION
)
8516 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
8517 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
8520 gimple_seq_add_stmt (end
, gimple_build_label (lab6
));
8521 tree x
= lang_hooks
.decls
.omp_clause_dtor (c
, priv
);
8524 gimple_seq tseq
= NULL
;
8525 gimplify_stmt (&x
, &tseq
);
8526 gimple_seq_add_seq (end
, tseq
);
8531 tree x
= build2 (rcode
, TREE_TYPE (out
), out
, priv
);
8532 out
= unshare_expr (out
);
8533 gimplify_assign (out
, x
, end
);
8536 = gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
8537 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
8538 gimple_seq_add_stmt (end
, g
);
8539 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
8540 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
8541 gimple_seq_add_stmt (end
, g
);
8542 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
8543 build_int_cst (TREE_TYPE (i
), 1));
8544 gimple_seq_add_stmt (end
, g
);
8545 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, endl
);
8546 gimple_seq_add_stmt (end
, g
);
8547 gimple_seq_add_stmt (end
, gimple_build_label (endl
));
8549 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
8551 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
8552 tree oldv
= NULL_TREE
;
8553 tree lab6
= NULL_TREE
;
8556 /* If this reduction needs destruction and parallel
8557 has been cancelled, jump around the merge operation
8558 to the destruction. */
8559 tree lab5
= create_artificial_label (UNKNOWN_LOCATION
);
8560 lab6
= create_artificial_label (UNKNOWN_LOCATION
);
8561 tree zero
= build_zero_cst (TREE_TYPE (cancellable
));
8562 g
= gimple_build_cond (NE_EXPR
, cancellable
, zero
,
8564 gimple_seq_add_stmt (end
, g
);
8565 gimple_seq_add_stmt (end
, gimple_build_label (lab5
));
8567 if (omp_is_reference (decl
)
8568 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
8570 ref
= build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c
), ref
);
8571 ref
= build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c
), ref
);
8572 tree refv
= create_tmp_var (TREE_TYPE (ref
));
8573 gimplify_assign (refv
, ref
, end
);
8574 ref
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), refv
);
8575 SET_DECL_VALUE_EXPR (placeholder
, ref
);
8576 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
8577 tree d
= maybe_lookup_decl (decl
, ctx
);
8579 if (DECL_HAS_VALUE_EXPR_P (d
))
8580 oldv
= DECL_VALUE_EXPR (d
);
8581 if (omp_is_reference (var
))
8583 tree v
= fold_convert (TREE_TYPE (d
),
8584 build_fold_addr_expr (new_var
));
8585 SET_DECL_VALUE_EXPR (d
, v
);
8588 SET_DECL_VALUE_EXPR (d
, new_var
);
8589 DECL_HAS_VALUE_EXPR_P (d
) = 1;
8590 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
8592 SET_DECL_VALUE_EXPR (d
, oldv
);
8595 SET_DECL_VALUE_EXPR (d
, NULL_TREE
);
8596 DECL_HAS_VALUE_EXPR_P (d
) = 0;
8598 gimple_seq_add_seq (end
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
8599 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
8600 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_TASK_REDUCTION
)
8601 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
8603 gimple_seq_add_stmt (end
, gimple_build_label (lab6
));
8604 tree x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
8607 gimple_seq tseq
= NULL
;
8608 gimplify_stmt (&x
, &tseq
);
8609 gimple_seq_add_seq (end
, tseq
);
8614 tree x
= build2 (rcode
, TREE_TYPE (ref
), ref
, new_var
);
8615 ref
= unshare_expr (ref
);
8616 gimplify_assign (ref
, x
, end
);
8618 gimple_seq_add_stmt (end
, gimple_build_label (lab4
));
8620 field
= DECL_CHAIN (bfield
);
8624 if (code
== OMP_TASKGROUP
)
8626 t
= builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER
);
8627 g
= gimple_build_call (t
, 1, build_fold_addr_expr (avar
));
8628 gimple_seq_add_stmt (start
, g
);
8633 if (code
== OMP_FOR
)
8634 c
= gimple_omp_for_clauses (ctx
->stmt
);
8635 else if (code
== OMP_SECTIONS
)
8636 c
= gimple_omp_sections_clauses (ctx
->stmt
);
8638 c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
8639 c
= omp_find_clause (c
, OMP_CLAUSE__REDUCTEMP_
);
8640 t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (c
)),
8641 build_fold_addr_expr (avar
));
8642 gimplify_assign (OMP_CLAUSE_DECL (c
), t
, start
);
8645 gimple_seq_add_stmt (end
, gimple_build_assign (data
, PLUS_EXPR
, data
, sz
));
8646 gimple_seq_add_stmt (end
, gimple_build_assign (idx
, PLUS_EXPR
, idx
,
8648 g
= gimple_build_cond (NE_EXPR
, idx
, num_thr_sz
, lab1
, lab2
);
8649 gimple_seq_add_stmt (end
, g
);
8650 gimple_seq_add_stmt (end
, gimple_build_label (lab2
));
8651 if (code
== OMP_FOR
|| code
== OMP_SECTIONS
)
8653 enum built_in_function bfn
8654 = BUILT_IN_GOMP_WORKSHARE_TASK_REDUCTION_UNREGISTER
;
8655 t
= builtin_decl_explicit (bfn
);
8656 tree c_bool_type
= TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (t
)));
8660 arg
= create_tmp_var (c_bool_type
);
8661 gimple_seq_add_stmt (end
, gimple_build_assign (arg
, NOP_EXPR
,
8665 arg
= build_int_cst (c_bool_type
, 0);
8666 g
= gimple_build_call (t
, 1, arg
);
8670 t
= builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_UNREGISTER
);
8671 g
= gimple_build_call (t
, 1, build_fold_addr_expr (avar
));
8673 gimple_seq_add_stmt (end
, g
);
8674 t
= build_constructor (atype
, NULL
);
8675 TREE_THIS_VOLATILE (t
) = 1;
8676 gimple_seq_add_stmt (end
, gimple_build_assign (avar
, t
));
8679 /* Expand code for an OpenMP taskgroup directive. */
8682 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8684 gimple
*stmt
= gsi_stmt (*gsi_p
);
8687 gimple_seq dseq
= NULL
;
8688 tree block
= make_node (BLOCK
);
8690 bind
= gimple_build_bind (NULL
, NULL
, block
);
8691 gsi_replace (gsi_p
, bind
, true);
8692 gimple_bind_add_stmt (bind
, stmt
);
8694 push_gimplify_context ();
8696 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8698 gimple_bind_add_stmt (bind
, x
);
8700 lower_omp_task_reductions (ctx
, OMP_TASKGROUP
,
8701 gimple_omp_taskgroup_clauses (stmt
),
8702 gimple_bind_body_ptr (bind
), &dseq
);
8704 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8705 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8706 gimple_omp_set_body (stmt
, NULL
);
8708 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8709 gimple_bind_add_seq (bind
, dseq
);
8711 pop_gimplify_context (bind
);
8713 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8714 BLOCK_VARS (block
) = ctx
->block_vars
;
8718 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
8721 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
8724 struct omp_for_data fd
;
8725 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
8728 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
8729 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
8730 omp_extract_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
8734 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
8735 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
8736 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
8737 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
8739 /* Merge depend clauses from multiple adjacent
8740 #pragma omp ordered depend(sink:...) constructs
8741 into one #pragma omp ordered depend(sink:...), so that
8742 we can optimize them together. */
8743 gimple_stmt_iterator gsi
= *gsi_p
;
8745 while (!gsi_end_p (gsi
))
8747 gimple
*stmt
= gsi_stmt (gsi
);
8748 if (is_gimple_debug (stmt
)
8749 || gimple_code (stmt
) == GIMPLE_NOP
)
8754 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
8756 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
8757 c
= gimple_omp_ordered_clauses (ord_stmt2
);
8759 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
8760 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
8763 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
8765 gsi_remove (&gsi
, true);
8769 /* Canonicalize sink dependence clauses into one folded clause if
8772 The basic algorithm is to create a sink vector whose first
8773 element is the GCD of all the first elements, and whose remaining
8774 elements are the minimum of the subsequent columns.
8776 We ignore dependence vectors whose first element is zero because
8777 such dependencies are known to be executed by the same thread.
8779 We take into account the direction of the loop, so a minimum
8780 becomes a maximum if the loop is iterating forwards. We also
8781 ignore sink clauses where the loop direction is unknown, or where
8782 the offsets are clearly invalid because they are not a multiple
8783 of the loop increment.
8787 #pragma omp for ordered(2)
8788 for (i=0; i < N; ++i)
8789 for (j=0; j < M; ++j)
8791 #pragma omp ordered \
8792 depend(sink:i-8,j-2) \
8793 depend(sink:i,j-1) \ // Completely ignored because i+0.
8794 depend(sink:i-4,j-3) \
8795 depend(sink:i-6,j-4)
8796 #pragma omp ordered depend(source)
8801 depend(sink:-gcd(8,4,6),-min(2,3,4))
8806 /* FIXME: Computing GCD's where the first element is zero is
8807 non-trivial in the presence of collapsed loops. Do this later. */
8808 if (fd
.collapse
> 1)
8811 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
8813 /* wide_int is not a POD so it must be default-constructed. */
8814 for (unsigned i
= 0; i
!= 2 * len
- 1; ++i
)
8815 new (static_cast<void*>(folded_deps
+ i
)) wide_int ();
8817 tree folded_dep
= NULL_TREE
;
8818 /* TRUE if the first dimension's offset is negative. */
8819 bool neg_offset_p
= false;
8821 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
8823 while ((c
= *list_p
) != NULL
)
8825 bool remove
= false;
8827 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
8828 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
8829 goto next_ordered_clause
;
8832 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
8833 vec
&& TREE_CODE (vec
) == TREE_LIST
;
8834 vec
= TREE_CHAIN (vec
), ++i
)
8836 gcc_assert (i
< len
);
8838 /* omp_extract_for_data has canonicalized the condition. */
8839 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
8840 || fd
.loops
[i
].cond_code
== GT_EXPR
);
8841 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
8842 bool maybe_lexically_later
= true;
8844 /* While the committee makes up its mind, bail if we have any
8845 non-constant steps. */
8846 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
8847 goto lower_omp_ordered_ret
;
8849 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
8850 if (POINTER_TYPE_P (itype
))
8852 wide_int offset
= wide_int::from (wi::to_wide (TREE_PURPOSE (vec
)),
8853 TYPE_PRECISION (itype
),
8856 /* Ignore invalid offsets that are not multiples of the step. */
8857 if (!wi::multiple_of_p (wi::abs (offset
),
8858 wi::abs (wi::to_wide (fd
.loops
[i
].step
)),
8861 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
8862 "ignoring sink clause with offset that is not "
8863 "a multiple of the loop step");
8865 goto next_ordered_clause
;
8868 /* Calculate the first dimension. The first dimension of
8869 the folded dependency vector is the GCD of the first
8870 elements, while ignoring any first elements whose offset
8874 /* Ignore dependence vectors whose first dimension is 0. */
8878 goto next_ordered_clause
;
8882 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
8884 error_at (OMP_CLAUSE_LOCATION (c
),
8885 "first offset must be in opposite direction "
8886 "of loop iterations");
8887 goto lower_omp_ordered_ret
;
8891 neg_offset_p
= forward
;
8892 /* Initialize the first time around. */
8893 if (folded_dep
== NULL_TREE
)
8896 folded_deps
[0] = offset
;
8899 folded_deps
[0] = wi::gcd (folded_deps
[0],
8903 /* Calculate minimum for the remaining dimensions. */
8906 folded_deps
[len
+ i
- 1] = offset
;
8907 if (folded_dep
== c
)
8908 folded_deps
[i
] = offset
;
8909 else if (maybe_lexically_later
8910 && !wi::eq_p (folded_deps
[i
], offset
))
8912 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
8916 for (j
= 1; j
<= i
; j
++)
8917 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
8920 maybe_lexically_later
= false;
8924 gcc_assert (i
== len
);
8928 next_ordered_clause
:
8930 *list_p
= OMP_CLAUSE_CHAIN (c
);
8932 list_p
= &OMP_CLAUSE_CHAIN (c
);
8938 folded_deps
[0] = -folded_deps
[0];
8940 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
8941 if (POINTER_TYPE_P (itype
))
8944 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
8945 = wide_int_to_tree (itype
, folded_deps
[0]);
8946 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
8947 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
8950 lower_omp_ordered_ret
:
8952 /* Ordered without clauses is #pragma omp threads, while we want
8953 a nop instead if we remove all clauses. */
8954 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
8955 gsi_replace (gsi_p
, gimple_build_nop (), true);
8959 /* Expand code for an OpenMP ordered directive. */
8962 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8965 gimple
*stmt
= gsi_stmt (*gsi_p
), *g
;
8966 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
8969 bool simd
= omp_find_clause (gimple_omp_ordered_clauses (ord_stmt
),
8971 /* FIXME: this should check presence of OMP_CLAUSE__SIMT_ on the enclosing
8974 = simd
&& omp_maybe_offloaded_ctx (ctx
) && omp_max_simt_vf () > 1;
8975 bool threads
= omp_find_clause (gimple_omp_ordered_clauses (ord_stmt
),
8976 OMP_CLAUSE_THREADS
);
8978 if (omp_find_clause (gimple_omp_ordered_clauses (ord_stmt
),
8981 /* FIXME: This is needs to be moved to the expansion to verify various
8982 conditions only testable on cfg with dominators computed, and also
8983 all the depend clauses to be merged still might need to be available
8984 for the runtime checks. */
8986 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
8990 push_gimplify_context ();
8992 block
= make_node (BLOCK
);
8993 bind
= gimple_build_bind (NULL
, NULL
, block
);
8994 gsi_replace (gsi_p
, bind
, true);
8995 gimple_bind_add_stmt (bind
, stmt
);
8999 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 1,
9000 build_int_cst (NULL_TREE
, threads
));
9001 cfun
->has_simduid_loops
= true;
9004 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
9006 gimple_bind_add_stmt (bind
, x
);
9008 tree counter
= NULL_TREE
, test
= NULL_TREE
, body
= NULL_TREE
;
9011 counter
= create_tmp_var (integer_type_node
);
9012 g
= gimple_build_call_internal (IFN_GOMP_SIMT_LANE
, 0);
9013 gimple_call_set_lhs (g
, counter
);
9014 gimple_bind_add_stmt (bind
, g
);
9016 body
= create_artificial_label (UNKNOWN_LOCATION
);
9017 test
= create_artificial_label (UNKNOWN_LOCATION
);
9018 gimple_bind_add_stmt (bind
, gimple_build_label (body
));
9020 tree simt_pred
= create_tmp_var (integer_type_node
);
9021 g
= gimple_build_call_internal (IFN_GOMP_SIMT_ORDERED_PRED
, 1, counter
);
9022 gimple_call_set_lhs (g
, simt_pred
);
9023 gimple_bind_add_stmt (bind
, g
);
9025 tree t
= create_artificial_label (UNKNOWN_LOCATION
);
9026 g
= gimple_build_cond (EQ_EXPR
, simt_pred
, integer_zero_node
, t
, test
);
9027 gimple_bind_add_stmt (bind
, g
);
9029 gimple_bind_add_stmt (bind
, gimple_build_label (t
));
9031 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9032 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9033 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9034 gimple_omp_set_body (stmt
, NULL
);
9038 gimple_bind_add_stmt (bind
, gimple_build_label (test
));
9039 g
= gimple_build_assign (counter
, MINUS_EXPR
, counter
, integer_one_node
);
9040 gimple_bind_add_stmt (bind
, g
);
9042 tree c
= build2 (GE_EXPR
, boolean_type_node
, counter
, integer_zero_node
);
9043 tree nonneg
= create_tmp_var (integer_type_node
);
9044 gimple_seq tseq
= NULL
;
9045 gimplify_assign (nonneg
, fold_convert (integer_type_node
, c
), &tseq
);
9046 gimple_bind_add_seq (bind
, tseq
);
9048 g
= gimple_build_call_internal (IFN_GOMP_SIMT_VOTE_ANY
, 1, nonneg
);
9049 gimple_call_set_lhs (g
, nonneg
);
9050 gimple_bind_add_stmt (bind
, g
);
9052 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
9053 g
= gimple_build_cond (NE_EXPR
, nonneg
, integer_zero_node
, body
, end
);
9054 gimple_bind_add_stmt (bind
, g
);
9056 gimple_bind_add_stmt (bind
, gimple_build_label (end
));
9059 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 1,
9060 build_int_cst (NULL_TREE
, threads
));
9062 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
9064 gimple_bind_add_stmt (bind
, x
);
9066 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9068 pop_gimplify_context (bind
);
9070 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9071 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9075 /* Expand code for an OpenMP scan directive and the structured block
9076 before the scan directive. */
9079 lower_omp_scan (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9081 gimple
*stmt
= gsi_stmt (*gsi_p
);
9083 = gimple_omp_scan_clauses (as_a
<gomp_scan
*> (stmt
)) != NULL
;
9084 tree lane
= NULL_TREE
;
9085 gimple_seq before
= NULL
;
9086 omp_context
*octx
= ctx
->outer
;
9088 if (octx
->scan_exclusive
&& !has_clauses
)
9090 gimple_stmt_iterator gsi2
= *gsi_p
;
9092 gimple
*stmt2
= gsi_stmt (gsi2
);
9093 /* For exclusive scan, swap GIMPLE_OMP_SCAN without clauses
9094 with following GIMPLE_OMP_SCAN with clauses, so that input_phase,
9095 the one with exclusive clause(s), comes first. */
9097 && gimple_code (stmt2
) == GIMPLE_OMP_SCAN
9098 && gimple_omp_scan_clauses (as_a
<gomp_scan
*> (stmt2
)) != NULL
)
9100 gsi_remove (gsi_p
, false);
9101 gsi_insert_after (gsi_p
, stmt
, GSI_SAME_STMT
);
9102 ctx
= maybe_lookup_ctx (stmt2
);
9104 lower_omp_scan (gsi_p
, ctx
);
9109 bool input_phase
= has_clauses
^ octx
->scan_inclusive
;
9110 bool is_simd
= (gimple_code (octx
->stmt
) == GIMPLE_OMP_FOR
9111 && gimple_omp_for_kind (octx
->stmt
) == GF_OMP_FOR_KIND_SIMD
);
9112 bool is_for
= (gimple_code (octx
->stmt
) == GIMPLE_OMP_FOR
9113 && gimple_omp_for_kind (octx
->stmt
) == GF_OMP_FOR_KIND_FOR
9114 && !gimple_omp_for_combined_p (octx
->stmt
));
9115 bool is_for_simd
= is_simd
&& gimple_omp_for_combined_into_p (octx
->stmt
);
9116 if (is_for_simd
&& octx
->for_simd_scan_phase
)
9119 if (tree c
= omp_find_clause (gimple_omp_for_clauses (octx
->stmt
),
9120 OMP_CLAUSE__SIMDUID_
))
9122 tree uid
= OMP_CLAUSE__SIMDUID__DECL (c
);
9123 lane
= create_tmp_var (unsigned_type_node
);
9124 tree t
= build_int_cst (integer_type_node
,
9126 : octx
->scan_inclusive
? 2 : 3);
9128 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 2, uid
, t
);
9129 gimple_call_set_lhs (g
, lane
);
9130 gimple_seq_add_stmt (&before
, g
);
9133 if (is_simd
|| is_for
)
9135 for (tree c
= gimple_omp_for_clauses (octx
->stmt
);
9136 c
; c
= OMP_CLAUSE_CHAIN (c
))
9137 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
9138 && OMP_CLAUSE_REDUCTION_INSCAN (c
))
9140 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
9141 tree var
= OMP_CLAUSE_DECL (c
);
9142 tree new_var
= lookup_decl (var
, octx
);
9144 tree var2
= NULL_TREE
;
9145 tree var3
= NULL_TREE
;
9146 tree var4
= NULL_TREE
;
9147 tree lane0
= NULL_TREE
;
9148 tree new_vard
= new_var
;
9149 if (omp_is_reference (var
))
9151 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
9154 if (DECL_HAS_VALUE_EXPR_P (new_vard
))
9156 val
= DECL_VALUE_EXPR (new_vard
);
9157 if (new_vard
!= new_var
)
9159 gcc_assert (TREE_CODE (val
) == ADDR_EXPR
);
9160 val
= TREE_OPERAND (val
, 0);
9162 if (TREE_CODE (val
) == ARRAY_REF
9163 && VAR_P (TREE_OPERAND (val
, 0)))
9165 tree v
= TREE_OPERAND (val
, 0);
9166 if (lookup_attribute ("omp simd array",
9167 DECL_ATTRIBUTES (v
)))
9169 val
= unshare_expr (val
);
9170 lane0
= TREE_OPERAND (val
, 1);
9171 TREE_OPERAND (val
, 1) = lane
;
9172 var2
= lookup_decl (v
, octx
);
9173 if (octx
->scan_exclusive
)
9174 var4
= lookup_decl (var2
, octx
);
9176 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
9177 var3
= maybe_lookup_decl (var4
? var4
: var2
, octx
);
9180 var2
= build4 (ARRAY_REF
, TREE_TYPE (val
),
9181 var2
, lane
, NULL_TREE
, NULL_TREE
);
9182 TREE_THIS_NOTRAP (var2
) = 1;
9183 if (octx
->scan_exclusive
)
9185 var4
= build4 (ARRAY_REF
, TREE_TYPE (val
),
9186 var4
, lane
, NULL_TREE
,
9188 TREE_THIS_NOTRAP (var4
) = 1;
9199 var2
= build_outer_var_ref (var
, octx
);
9200 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
9202 var3
= maybe_lookup_decl (new_vard
, octx
);
9203 if (var3
== new_vard
|| var3
== NULL_TREE
)
9205 else if (is_simd
&& octx
->scan_exclusive
&& !input_phase
)
9207 var4
= maybe_lookup_decl (var3
, octx
);
9208 if (var4
== var3
|| var4
== NULL_TREE
)
9210 if (TREE_ADDRESSABLE (TREE_TYPE (new_var
)))
9221 && octx
->scan_exclusive
9223 && var4
== NULL_TREE
)
9224 var4
= create_tmp_var (TREE_TYPE (val
));
9226 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
9228 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
9233 /* If we've added a separate identity element
9234 variable, copy it over into val. */
9235 tree x
= lang_hooks
.decls
.omp_clause_assign_op (c
, val
,
9237 gimplify_and_add (x
, &before
);
9239 else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
9241 /* Otherwise, assign to it the identity element. */
9242 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
9244 tseq
= copy_gimple_seq_and_replace_locals (tseq
);
9245 tree ref
= build_outer_var_ref (var
, octx
);
9246 tree x
= (DECL_HAS_VALUE_EXPR_P (new_vard
)
9247 ? DECL_VALUE_EXPR (new_vard
) : NULL_TREE
);
9250 if (new_vard
!= new_var
)
9251 val
= build_fold_addr_expr_loc (clause_loc
, val
);
9252 SET_DECL_VALUE_EXPR (new_vard
, val
);
9254 SET_DECL_VALUE_EXPR (placeholder
, ref
);
9255 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
9256 lower_omp (&tseq
, octx
);
9258 SET_DECL_VALUE_EXPR (new_vard
, x
);
9259 SET_DECL_VALUE_EXPR (placeholder
, NULL_TREE
);
9260 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
9261 gimple_seq_add_seq (&before
, tseq
);
9263 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
9269 if (octx
->scan_exclusive
)
9271 tree v4
= unshare_expr (var4
);
9272 tree v2
= unshare_expr (var2
);
9273 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, v4
, v2
);
9274 gimplify_and_add (x
, &before
);
9276 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
9277 x
= (DECL_HAS_VALUE_EXPR_P (new_vard
)
9278 ? DECL_VALUE_EXPR (new_vard
) : NULL_TREE
);
9280 if (x
&& new_vard
!= new_var
)
9281 vexpr
= build_fold_addr_expr_loc (clause_loc
, val
);
9283 SET_DECL_VALUE_EXPR (new_vard
, vexpr
);
9284 SET_DECL_VALUE_EXPR (placeholder
, var2
);
9285 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
9286 lower_omp (&tseq
, octx
);
9287 gimple_seq_add_seq (&before
, tseq
);
9288 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
9290 SET_DECL_VALUE_EXPR (new_vard
, x
);
9291 SET_DECL_VALUE_EXPR (placeholder
, NULL_TREE
);
9292 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
9293 if (octx
->scan_inclusive
)
9295 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, val
,
9297 gimplify_and_add (x
, &before
);
9299 else if (lane0
== NULL_TREE
)
9301 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, val
,
9303 gimplify_and_add (x
, &before
);
9311 /* input phase. Set val to initializer before
9313 tree x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
9314 gimplify_assign (val
, x
, &before
);
9319 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
9320 if (code
== MINUS_EXPR
)
9323 tree x
= build2 (code
, TREE_TYPE (var2
),
9324 unshare_expr (var2
), unshare_expr (val
));
9325 if (octx
->scan_inclusive
)
9327 gimplify_assign (unshare_expr (var2
), x
, &before
);
9328 gimplify_assign (val
, var2
, &before
);
9332 gimplify_assign (unshare_expr (var4
),
9333 unshare_expr (var2
), &before
);
9334 gimplify_assign (var2
, x
, &before
);
9335 if (lane0
== NULL_TREE
)
9336 gimplify_assign (val
, var4
, &before
);
9340 if (octx
->scan_exclusive
&& !input_phase
&& lane0
)
9342 tree vexpr
= unshare_expr (var4
);
9343 TREE_OPERAND (vexpr
, 1) = lane0
;
9344 if (new_vard
!= new_var
)
9345 vexpr
= build_fold_addr_expr_loc (clause_loc
, vexpr
);
9346 SET_DECL_VALUE_EXPR (new_vard
, vexpr
);
9350 if (is_simd
&& !is_for_simd
)
9352 gsi_insert_seq_after (gsi_p
, gimple_omp_body (stmt
), GSI_SAME_STMT
);
9353 gsi_insert_seq_after (gsi_p
, before
, GSI_SAME_STMT
);
9354 gsi_replace (gsi_p
, gimple_build_nop (), true);
9357 lower_omp (gimple_omp_body_ptr (stmt
), octx
);
9360 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (stmt
));
9361 gsi_insert_seq_before (&gsi
, before
, GSI_SAME_STMT
);
9366 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
9367 substitution of a couple of function calls. But in the NAMED case,
9368 requires that languages coordinate a symbol name. It is therefore
9369 best put here in common code. */
9371 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
9374 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9377 tree name
, lock
, unlock
;
9378 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
9380 location_t loc
= gimple_location (stmt
);
9383 name
= gimple_omp_critical_name (stmt
);
9388 if (!critical_name_mutexes
)
9389 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
9391 tree
*n
= critical_name_mutexes
->get (name
);
9396 decl
= create_tmp_var_raw (ptr_type_node
);
9398 new_str
= ACONCAT ((".gomp_critical_user_",
9399 IDENTIFIER_POINTER (name
), NULL
));
9400 DECL_NAME (decl
) = get_identifier (new_str
);
9401 TREE_PUBLIC (decl
) = 1;
9402 TREE_STATIC (decl
) = 1;
9403 DECL_COMMON (decl
) = 1;
9404 DECL_ARTIFICIAL (decl
) = 1;
9405 DECL_IGNORED_P (decl
) = 1;
9407 varpool_node::finalize_decl (decl
);
9409 critical_name_mutexes
->put (name
, decl
);
9414 /* If '#pragma omp critical' is inside offloaded region or
9415 inside function marked as offloadable, the symbol must be
9416 marked as offloadable too. */
9418 if (cgraph_node::get (current_function_decl
)->offloadable
)
9419 varpool_node::get_create (decl
)->offloadable
= 1;
9421 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
9422 if (is_gimple_omp_offloaded (octx
->stmt
))
9424 varpool_node::get_create (decl
)->offloadable
= 1;
9428 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
9429 lock
= build_call_expr_loc (loc
, lock
, 1,
9430 build_fold_addr_expr_loc (loc
, decl
));
9432 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
9433 unlock
= build_call_expr_loc (loc
, unlock
, 1,
9434 build_fold_addr_expr_loc (loc
, decl
));
9438 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
9439 lock
= build_call_expr_loc (loc
, lock
, 0);
9441 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
9442 unlock
= build_call_expr_loc (loc
, unlock
, 0);
9445 push_gimplify_context ();
9447 block
= make_node (BLOCK
);
9448 bind
= gimple_build_bind (NULL
, NULL
, block
);
9449 gsi_replace (gsi_p
, bind
, true);
9450 gimple_bind_add_stmt (bind
, stmt
);
9452 tbody
= gimple_bind_body (bind
);
9453 gimplify_and_add (lock
, &tbody
);
9454 gimple_bind_set_body (bind
, tbody
);
9456 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9457 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9458 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9459 gimple_omp_set_body (stmt
, NULL
);
9461 tbody
= gimple_bind_body (bind
);
9462 gimplify_and_add (unlock
, &tbody
);
9463 gimple_bind_set_body (bind
, tbody
);
9465 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9467 pop_gimplify_context (bind
);
9468 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9469 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9472 /* A subroutine of lower_omp_for. Generate code to emit the predicate
9473 for a lastprivate clause. Given a loop control predicate of (V
9474 cond N2), we gate the clause on (!(V cond N2)). The lowered form
9475 is appended to *DLIST, iterator initialization is appended to
9476 *BODY_P. *CLIST is for lastprivate(conditional:) code that needs
9477 to be emitted in a critical section. */
9480 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
9481 gimple_seq
*dlist
, gimple_seq
*clist
,
9482 struct omp_context
*ctx
)
9484 tree clauses
, cond
, vinit
;
9485 enum tree_code cond_code
;
9488 cond_code
= fd
->loop
.cond_code
;
9489 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
9491 /* When possible, use a strict equality expression. This can let VRP
9492 type optimizations deduce the value and remove a copy. */
9493 if (tree_fits_shwi_p (fd
->loop
.step
))
9495 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
9496 if (step
== 1 || step
== -1)
9497 cond_code
= EQ_EXPR
;
9500 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_GRID_LOOP
9501 || gimple_omp_for_grid_phony (fd
->for_stmt
))
9502 cond
= omp_grid_lastprivate_predicate (fd
);
9505 tree n2
= fd
->loop
.n2
;
9506 if (fd
->collapse
> 1
9507 && TREE_CODE (n2
) != INTEGER_CST
9508 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
9510 struct omp_context
*taskreg_ctx
= NULL
;
9511 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
9513 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
9514 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
9515 || gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9517 if (gimple_omp_for_combined_into_p (gfor
))
9519 gcc_assert (ctx
->outer
->outer
9520 && is_parallel_ctx (ctx
->outer
->outer
));
9521 taskreg_ctx
= ctx
->outer
->outer
;
9525 struct omp_for_data outer_fd
;
9526 omp_extract_for_data (gfor
, &outer_fd
, NULL
);
9527 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
9530 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
9531 taskreg_ctx
= ctx
->outer
->outer
;
9533 else if (is_taskreg_ctx (ctx
->outer
))
9534 taskreg_ctx
= ctx
->outer
;
9538 tree taskreg_clauses
9539 = gimple_omp_taskreg_clauses (taskreg_ctx
->stmt
);
9540 tree innerc
= omp_find_clause (taskreg_clauses
,
9541 OMP_CLAUSE__LOOPTEMP_
);
9542 gcc_assert (innerc
);
9543 for (i
= 0; i
< fd
->collapse
; i
++)
9545 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
9546 OMP_CLAUSE__LOOPTEMP_
);
9547 gcc_assert (innerc
);
9549 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
9550 OMP_CLAUSE__LOOPTEMP_
);
9552 n2
= fold_convert (TREE_TYPE (n2
),
9553 lookup_decl (OMP_CLAUSE_DECL (innerc
),
9557 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
9560 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
9562 lower_lastprivate_clauses (clauses
, cond
, body_p
, &stmts
, clist
, ctx
);
9563 if (!gimple_seq_empty_p (stmts
))
9565 gimple_seq_add_seq (&stmts
, *dlist
);
9568 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
9569 vinit
= fd
->loop
.n1
;
9570 if (cond_code
== EQ_EXPR
9571 && tree_fits_shwi_p (fd
->loop
.n2
)
9572 && ! integer_zerop (fd
->loop
.n2
))
9573 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
9575 vinit
= unshare_expr (vinit
);
9577 /* Initialize the iterator variable, so that threads that don't execute
9578 any iterations don't execute the lastprivate clauses by accident. */
9579 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
9583 /* Callback for walk_gimple_seq. Find #pragma omp scan statement. */
9586 omp_find_scan (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
9587 struct walk_stmt_info
*wi
)
9589 gimple
*stmt
= gsi_stmt (*gsi_p
);
9591 *handled_ops_p
= true;
9592 switch (gimple_code (stmt
))
9596 case GIMPLE_OMP_FOR
:
9597 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_SIMD
9598 && gimple_omp_for_combined_into_p (stmt
))
9599 *handled_ops_p
= false;
9602 case GIMPLE_OMP_SCAN
:
9603 *(gimple_stmt_iterator
*) (wi
->info
) = *gsi_p
;
9604 return integer_zero_node
;
9611 /* Helper function for lower_omp_for, add transformations for a worksharing
9612 loop with scan directives inside of it.
9613 For worksharing loop not combined with simd, transform:
9614 #pragma omp for reduction(inscan,+:r) private(i)
9615 for (i = 0; i < n; i = i + 1)
9620 #pragma omp scan inclusive(r)
9626 into two worksharing loops + code to merge results:
9628 num_threads = omp_get_num_threads ();
9629 thread_num = omp_get_thread_num ();
9630 if (thread_num == 0) goto <D.2099>; else goto <D.2100>;
9635 // For UDRs this is UDR init, or if ctors are needed, copy from
9636 // var3 that has been constructed to contain the neutral element.
9640 // The _scantemp_ clauses will arrange for rpriva to be initialized to
9641 // a shared array with num_threads elements and rprivb to a local array
9642 // number of elements equal to the number of (contiguous) iterations the
9643 // current thread will perform. controlb and controlp variables are
9644 // temporaries to handle deallocation of rprivb at the end of second
9646 #pragma omp for _scantemp_(rpriva) _scantemp_(rprivb) _scantemp_(controlb) \
9647 _scantemp_(controlp) reduction(inscan,+:r) private(i) nowait
9648 for (i = 0; i < n; i = i + 1)
9651 // For UDRs this is UDR init or copy from var3.
9653 // This is the input phase from user code.
9657 // For UDRs this is UDR merge.
9659 // Rather than handing it over to the user, save to local thread's
9661 rprivb[ivar] = var2;
9662 // For exclusive scan, the above two statements are swapped.
9666 // And remember the final value from this thread's into the shared
9668 rpriva[(sizetype) thread_num] = var2;
9669 // If more than one thread, compute using Work-Efficient prefix sum
9670 // the inclusive parallel scan of the rpriva array.
9671 if (num_threads > 1) goto <D.2102>; else goto <D.2103>;
9676 num_threadsu = (unsigned int) num_threads;
9677 thread_numup1 = (unsigned int) thread_num + 1;
9680 if (twok > num_threadsu) goto <D.2110>; else goto <D.2111>;
9684 if (k == num_threadsu) goto <D.2112>; else goto <D.2111>;
9689 cplx = .MUL_OVERFLOW (thread_nump1, twok);
9690 mul = REALPART_EXPR <cplx>;
9691 ovf = IMAGPART_EXPR <cplx>;
9692 if (ovf == 0) goto <D.2116>; else goto <D.2117>;
9695 andvm1 = andv + 4294967295;
9697 if (l < num_threadsu) goto <D.2120>; else goto <D.2117>;
9699 // For UDRs this is UDR merge, performed using var2 variable as temporary,
9700 // i.e. var2 = rpriva[l - k]; UDR merge (var2, rpriva[l]); rpriva[l] = var2;
9701 rpriva[l] = rpriva[l - k] + rpriva[l];
9703 if (down == 0) goto <D.2121>; else goto <D.2122>;
9711 if (k != 0) goto <D.2108>; else goto <D.2103>;
9713 if (thread_num == 0) goto <D.2124>; else goto <D.2125>;
9715 // For UDRs this is UDR init or copy from var3.
9719 var2 = rpriva[thread_num - 1];
9722 #pragma omp for _scantemp_(controlb) _scantemp_(controlp) \
9723 reduction(inscan,+:r) private(i)
9724 for (i = 0; i < n; i = i + 1)
9727 // For UDRs, this is r = var2; UDR merge (r, rprivb[ivar]);
9728 r = var2 + rprivb[ivar];
9731 // This is the scan phase from user code.
9733 // Plus a bump of the iterator.
9739 lower_omp_for_scan (gimple_seq
*body_p
, gimple_seq
*dlist
, gomp_for
*stmt
,
9740 struct omp_for_data
*fd
, omp_context
*ctx
)
9742 bool is_for_simd
= gimple_omp_for_combined_p (stmt
);
9743 gcc_assert (ctx
->scan_inclusive
|| ctx
->scan_exclusive
);
9745 gimple_seq body
= gimple_omp_body (stmt
);
9746 gimple_stmt_iterator input1_gsi
= gsi_none ();
9747 struct walk_stmt_info wi
;
9748 memset (&wi
, 0, sizeof (wi
));
9750 wi
.info
= (void *) &input1_gsi
;
9751 walk_gimple_seq_mod (&body
, omp_find_scan
, NULL
, &wi
);
9752 gcc_assert (!gsi_end_p (input1_gsi
));
9754 gimple
*input_stmt1
= gsi_stmt (input1_gsi
);
9755 gimple_stmt_iterator gsi
= input1_gsi
;
9757 gimple_stmt_iterator scan1_gsi
= gsi
;
9758 gimple
*scan_stmt1
= gsi_stmt (gsi
);
9759 gcc_assert (scan_stmt1
&& gimple_code (scan_stmt1
) == GIMPLE_OMP_SCAN
);
9761 gimple_seq input_body
= gimple_omp_body (input_stmt1
);
9762 gimple_seq scan_body
= gimple_omp_body (scan_stmt1
);
9763 gimple_omp_set_body (input_stmt1
, NULL
);
9764 gimple_omp_set_body (scan_stmt1
, NULL
);
9765 gimple_omp_set_body (stmt
, NULL
);
9767 gomp_for
*new_stmt
= as_a
<gomp_for
*> (gimple_copy (stmt
));
9768 gimple_seq new_body
= copy_gimple_seq_and_replace_locals (body
);
9769 gimple_omp_set_body (stmt
, body
);
9770 gimple_omp_set_body (input_stmt1
, input_body
);
9772 gimple_stmt_iterator input2_gsi
= gsi_none ();
9773 memset (&wi
, 0, sizeof (wi
));
9775 wi
.info
= (void *) &input2_gsi
;
9776 walk_gimple_seq_mod (&new_body
, omp_find_scan
, NULL
, &wi
);
9777 gcc_assert (!gsi_end_p (input2_gsi
));
9779 gimple
*input_stmt2
= gsi_stmt (input2_gsi
);
9782 gimple_stmt_iterator scan2_gsi
= gsi
;
9783 gimple
*scan_stmt2
= gsi_stmt (gsi
);
9784 gcc_assert (scan_stmt2
&& gimple_code (scan_stmt2
) == GIMPLE_OMP_SCAN
);
9785 gimple_omp_set_body (scan_stmt2
, scan_body
);
9787 gimple_stmt_iterator input3_gsi
= gsi_none ();
9788 gimple_stmt_iterator scan3_gsi
= gsi_none ();
9789 gimple_stmt_iterator input4_gsi
= gsi_none ();
9790 gimple_stmt_iterator scan4_gsi
= gsi_none ();
9791 gimple
*input_stmt3
= NULL
, *scan_stmt3
= NULL
;
9792 gimple
*input_stmt4
= NULL
, *scan_stmt4
= NULL
;
9793 omp_context
*input_simd_ctx
= NULL
, *scan_simd_ctx
= NULL
;
9796 memset (&wi
, 0, sizeof (wi
));
9798 wi
.info
= (void *) &input3_gsi
;
9799 walk_gimple_seq_mod (&input_body
, omp_find_scan
, NULL
, &wi
);
9800 gcc_assert (!gsi_end_p (input3_gsi
));
9802 input_stmt3
= gsi_stmt (input3_gsi
);
9806 scan_stmt3
= gsi_stmt (gsi
);
9807 gcc_assert (scan_stmt3
&& gimple_code (scan_stmt3
) == GIMPLE_OMP_SCAN
);
9809 memset (&wi
, 0, sizeof (wi
));
9811 wi
.info
= (void *) &input4_gsi
;
9812 walk_gimple_seq_mod (&scan_body
, omp_find_scan
, NULL
, &wi
);
9813 gcc_assert (!gsi_end_p (input4_gsi
));
9815 input_stmt4
= gsi_stmt (input4_gsi
);
9819 scan_stmt4
= gsi_stmt (gsi
);
9820 gcc_assert (scan_stmt4
&& gimple_code (scan_stmt4
) == GIMPLE_OMP_SCAN
);
9822 input_simd_ctx
= maybe_lookup_ctx (input_stmt3
)->outer
;
9823 scan_simd_ctx
= maybe_lookup_ctx (input_stmt4
)->outer
;
9826 tree num_threads
= create_tmp_var (integer_type_node
);
9827 tree thread_num
= create_tmp_var (integer_type_node
);
9828 tree nthreads_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9829 tree threadnum_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9830 gimple
*g
= gimple_build_call (nthreads_decl
, 0);
9831 gimple_call_set_lhs (g
, num_threads
);
9832 gimple_seq_add_stmt (body_p
, g
);
9833 g
= gimple_build_call (threadnum_decl
, 0);
9834 gimple_call_set_lhs (g
, thread_num
);
9835 gimple_seq_add_stmt (body_p
, g
);
9837 tree ivar
= create_tmp_var (sizetype
);
9838 tree new_clauses1
= NULL_TREE
, new_clauses2
= NULL_TREE
;
9839 tree
*cp1
= &new_clauses1
, *cp2
= &new_clauses2
;
9840 tree k
= create_tmp_var (unsigned_type_node
);
9841 tree l
= create_tmp_var (unsigned_type_node
);
9843 gimple_seq clist
= NULL
, mdlist
= NULL
;
9844 gimple_seq thr01_list
= NULL
, thrn1_list
= NULL
;
9845 gimple_seq thr02_list
= NULL
, thrn2_list
= NULL
;
9846 gimple_seq scan1_list
= NULL
, input2_list
= NULL
;
9847 gimple_seq last_list
= NULL
, reduc_list
= NULL
;
9848 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9849 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
9850 && OMP_CLAUSE_REDUCTION_INSCAN (c
))
9852 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
9853 tree var
= OMP_CLAUSE_DECL (c
);
9854 tree new_var
= lookup_decl (var
, ctx
);
9855 tree var3
= NULL_TREE
;
9856 tree new_vard
= new_var
;
9857 if (omp_is_reference (var
))
9858 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
9859 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
9861 var3
= maybe_lookup_decl (new_vard
, ctx
);
9862 if (var3
== new_vard
)
9866 tree ptype
= build_pointer_type (TREE_TYPE (new_var
));
9867 tree rpriva
= create_tmp_var (ptype
);
9868 tree nc
= build_omp_clause (clause_loc
, OMP_CLAUSE__SCANTEMP_
);
9869 OMP_CLAUSE_DECL (nc
) = rpriva
;
9871 cp1
= &OMP_CLAUSE_CHAIN (nc
);
9873 tree rprivb
= create_tmp_var (ptype
);
9874 nc
= build_omp_clause (clause_loc
, OMP_CLAUSE__SCANTEMP_
);
9875 OMP_CLAUSE_DECL (nc
) = rprivb
;
9876 OMP_CLAUSE__SCANTEMP__ALLOC (nc
) = 1;
9878 cp1
= &OMP_CLAUSE_CHAIN (nc
);
9880 tree var2
= create_tmp_var_raw (TREE_TYPE (new_var
));
9881 if (new_vard
!= new_var
)
9882 TREE_ADDRESSABLE (var2
) = 1;
9883 gimple_add_tmp_var (var2
);
9885 tree x
= fold_convert_loc (clause_loc
, sizetype
, thread_num
);
9886 x
= fold_build2_loc (clause_loc
, MULT_EXPR
, sizetype
, x
,
9887 TYPE_SIZE_UNIT (TREE_TYPE (ptype
)));
9888 x
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (rpriva
), rpriva
, x
);
9889 tree rpriva_ref
= build_simple_mem_ref_loc (clause_loc
, x
);
9891 x
= fold_build2_loc (clause_loc
, PLUS_EXPR
, integer_type_node
,
9892 thread_num
, integer_minus_one_node
);
9893 x
= fold_convert_loc (clause_loc
, sizetype
, x
);
9894 x
= fold_build2_loc (clause_loc
, MULT_EXPR
, sizetype
, x
,
9895 TYPE_SIZE_UNIT (TREE_TYPE (ptype
)));
9896 x
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (rpriva
), rpriva
, x
);
9897 tree rprivam1_ref
= build_simple_mem_ref_loc (clause_loc
, x
);
9899 x
= fold_convert_loc (clause_loc
, sizetype
, l
);
9900 x
= fold_build2_loc (clause_loc
, MULT_EXPR
, sizetype
, x
,
9901 TYPE_SIZE_UNIT (TREE_TYPE (ptype
)));
9902 x
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (rpriva
), rpriva
, x
);
9903 tree rprival_ref
= build_simple_mem_ref_loc (clause_loc
, x
);
9905 x
= fold_build2_loc (clause_loc
, MINUS_EXPR
, unsigned_type_node
, l
, k
);
9906 x
= fold_convert_loc (clause_loc
, sizetype
, x
);
9907 x
= fold_build2_loc (clause_loc
, MULT_EXPR
, sizetype
, x
,
9908 TYPE_SIZE_UNIT (TREE_TYPE (ptype
)));
9909 x
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (rpriva
), rpriva
, x
);
9910 tree rprivalmk_ref
= build_simple_mem_ref_loc (clause_loc
, x
);
9912 x
= fold_build2_loc (clause_loc
, MULT_EXPR
, sizetype
, ivar
,
9913 TYPE_SIZE_UNIT (TREE_TYPE (ptype
)));
9914 x
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (rprivb
), rprivb
, x
);
9915 tree rprivb_ref
= build_simple_mem_ref_loc (clause_loc
, x
);
9917 tree var4
= is_for_simd
? new_var
: var2
;
9918 tree var5
= NULL_TREE
, var6
= NULL_TREE
;
9921 var5
= lookup_decl (var
, input_simd_ctx
);
9922 var6
= lookup_decl (var
, scan_simd_ctx
);
9923 if (new_vard
!= new_var
)
9925 var5
= build_simple_mem_ref_loc (clause_loc
, var5
);
9926 var6
= build_simple_mem_ref_loc (clause_loc
, var6
);
9929 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
9931 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
9934 x
= lang_hooks
.decls
.omp_clause_default_ctor
9935 (c
, var2
, build_outer_var_ref (var
, ctx
));
9937 gimplify_and_add (x
, &clist
);
9939 x
= build_outer_var_ref (var
, ctx
);
9940 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, unshare_expr (var4
),
9942 gimplify_and_add (x
, &thr01_list
);
9944 tree y
= (DECL_HAS_VALUE_EXPR_P (new_vard
)
9945 ? DECL_VALUE_EXPR (new_vard
) : NULL_TREE
);
9948 x
= unshare_expr (var4
);
9949 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, var3
);
9950 gimplify_and_add (x
, &thrn1_list
);
9951 x
= unshare_expr (var4
);
9952 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, var3
);
9953 gimplify_and_add (x
, &thr02_list
);
9955 else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
9957 /* Otherwise, assign to it the identity element. */
9958 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
9959 tseq
= copy_gimple_seq_and_replace_locals (tseq
);
9962 if (new_vard
!= new_var
)
9963 val
= build_fold_addr_expr_loc (clause_loc
, val
);
9964 SET_DECL_VALUE_EXPR (new_vard
, val
);
9965 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
9967 SET_DECL_VALUE_EXPR (placeholder
, error_mark_node
);
9968 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
9969 lower_omp (&tseq
, ctx
);
9970 gimple_seq_add_seq (&thrn1_list
, tseq
);
9971 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
9972 lower_omp (&tseq
, ctx
);
9973 gimple_seq_add_seq (&thr02_list
, tseq
);
9974 SET_DECL_VALUE_EXPR (placeholder
, NULL_TREE
);
9975 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
9976 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
9978 SET_DECL_VALUE_EXPR (new_vard
, y
);
9981 DECL_HAS_VALUE_EXPR_P (new_vard
) = 0;
9982 SET_DECL_VALUE_EXPR (new_vard
, NULL_TREE
);
9986 x
= unshare_expr (var4
);
9987 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, rprivam1_ref
);
9988 gimplify_and_add (x
, &thrn2_list
);
9992 x
= unshare_expr (rprivb_ref
);
9993 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, var5
);
9994 gimplify_and_add (x
, &scan1_list
);
9998 if (ctx
->scan_exclusive
)
10000 x
= unshare_expr (rprivb_ref
);
10001 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, var2
);
10002 gimplify_and_add (x
, &scan1_list
);
10005 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
10006 tseq
= copy_gimple_seq_and_replace_locals (tseq
);
10007 SET_DECL_VALUE_EXPR (placeholder
, var2
);
10008 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
10009 lower_omp (&tseq
, ctx
);
10010 gimple_seq_add_seq (&scan1_list
, tseq
);
10012 if (ctx
->scan_inclusive
)
10014 x
= unshare_expr (rprivb_ref
);
10015 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, var2
);
10016 gimplify_and_add (x
, &scan1_list
);
10020 x
= unshare_expr (rpriva_ref
);
10021 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
,
10022 unshare_expr (var4
));
10023 gimplify_and_add (x
, &mdlist
);
10025 x
= unshare_expr (is_for_simd
? var6
: new_var
);
10026 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, var4
);
10027 gimplify_and_add (x
, &input2_list
);
10030 if (new_vard
!= new_var
)
10031 val
= build_fold_addr_expr_loc (clause_loc
, val
);
10033 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
10034 tseq
= copy_gimple_seq_and_replace_locals (tseq
);
10035 SET_DECL_VALUE_EXPR (new_vard
, val
);
10036 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
10039 SET_DECL_VALUE_EXPR (placeholder
, var6
);
10040 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
10043 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
10044 lower_omp (&tseq
, ctx
);
10046 SET_DECL_VALUE_EXPR (new_vard
, y
);
10049 DECL_HAS_VALUE_EXPR_P (new_vard
) = 0;
10050 SET_DECL_VALUE_EXPR (new_vard
, NULL_TREE
);
10054 SET_DECL_VALUE_EXPR (placeholder
, new_var
);
10055 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
10056 lower_omp (&tseq
, ctx
);
10058 gimple_seq_add_seq (&input2_list
, tseq
);
10060 x
= build_outer_var_ref (var
, ctx
);
10061 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, rpriva_ref
);
10062 gimplify_and_add (x
, &last_list
);
10064 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var2
, rprivalmk_ref
);
10065 gimplify_and_add (x
, &reduc_list
);
10066 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
10067 tseq
= copy_gimple_seq_and_replace_locals (tseq
);
10069 if (new_vard
!= new_var
)
10070 val
= build_fold_addr_expr_loc (clause_loc
, val
);
10071 SET_DECL_VALUE_EXPR (new_vard
, val
);
10072 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
10073 SET_DECL_VALUE_EXPR (placeholder
, var2
);
10074 lower_omp (&tseq
, ctx
);
10075 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
10076 SET_DECL_VALUE_EXPR (placeholder
, NULL_TREE
);
10077 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
10079 SET_DECL_VALUE_EXPR (new_vard
, y
);
10082 DECL_HAS_VALUE_EXPR_P (new_vard
) = 0;
10083 SET_DECL_VALUE_EXPR (new_vard
, NULL_TREE
);
10085 gimple_seq_add_seq (&reduc_list
, tseq
);
10086 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, rprival_ref
, var2
);
10087 gimplify_and_add (x
, &reduc_list
);
10089 x
= lang_hooks
.decls
.omp_clause_dtor (c
, var2
);
10091 gimplify_and_add (x
, dlist
);
10095 x
= build_outer_var_ref (var
, ctx
);
10096 gimplify_assign (unshare_expr (var4
), x
, &thr01_list
);
10098 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
10099 gimplify_assign (unshare_expr (var4
), unshare_expr (x
),
10101 gimplify_assign (unshare_expr (var4
), x
, &thr02_list
);
10103 gimplify_assign (unshare_expr (var4
), rprivam1_ref
, &thrn2_list
);
10105 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
10106 if (code
== MINUS_EXPR
)
10110 gimplify_assign (unshare_expr (rprivb_ref
), var5
, &scan1_list
);
10113 if (ctx
->scan_exclusive
)
10114 gimplify_assign (unshare_expr (rprivb_ref
), var2
,
10116 x
= build2 (code
, TREE_TYPE (new_var
), var2
, new_var
);
10117 gimplify_assign (var2
, x
, &scan1_list
);
10118 if (ctx
->scan_inclusive
)
10119 gimplify_assign (unshare_expr (rprivb_ref
), var2
,
10123 gimplify_assign (unshare_expr (rpriva_ref
), unshare_expr (var4
),
10126 x
= build2 (code
, TREE_TYPE (new_var
), var4
, rprivb_ref
);
10127 gimplify_assign (is_for_simd
? var6
: new_var
, x
, &input2_list
);
10129 gimplify_assign (build_outer_var_ref (var
, ctx
), rpriva_ref
,
10132 x
= build2 (code
, TREE_TYPE (new_var
), rprivalmk_ref
,
10133 unshare_expr (rprival_ref
));
10134 gimplify_assign (rprival_ref
, x
, &reduc_list
);
10138 g
= gimple_build_assign (ivar
, PLUS_EXPR
, ivar
, size_one_node
);
10139 gimple_seq_add_stmt (&scan1_list
, g
);
10140 g
= gimple_build_assign (ivar
, PLUS_EXPR
, ivar
, size_one_node
);
10141 gimple_seq_add_stmt (gimple_omp_body_ptr (is_for_simd
10142 ? scan_stmt4
: scan_stmt2
), g
);
10144 tree controlb
= create_tmp_var (boolean_type_node
);
10145 tree controlp
= create_tmp_var (ptr_type_node
);
10146 tree nc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SCANTEMP_
);
10147 OMP_CLAUSE_DECL (nc
) = controlb
;
10148 OMP_CLAUSE__SCANTEMP__CONTROL (nc
) = 1;
10150 cp1
= &OMP_CLAUSE_CHAIN (nc
);
10151 nc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SCANTEMP_
);
10152 OMP_CLAUSE_DECL (nc
) = controlp
;
10153 OMP_CLAUSE__SCANTEMP__CONTROL (nc
) = 1;
10155 cp1
= &OMP_CLAUSE_CHAIN (nc
);
10156 nc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SCANTEMP_
);
10157 OMP_CLAUSE_DECL (nc
) = controlb
;
10158 OMP_CLAUSE__SCANTEMP__CONTROL (nc
) = 1;
10160 cp2
= &OMP_CLAUSE_CHAIN (nc
);
10161 nc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SCANTEMP_
);
10162 OMP_CLAUSE_DECL (nc
) = controlp
;
10163 OMP_CLAUSE__SCANTEMP__CONTROL (nc
) = 1;
10165 cp2
= &OMP_CLAUSE_CHAIN (nc
);
10167 *cp1
= gimple_omp_for_clauses (stmt
);
10168 gimple_omp_for_set_clauses (stmt
, new_clauses1
);
10169 *cp2
= gimple_omp_for_clauses (new_stmt
);
10170 gimple_omp_for_set_clauses (new_stmt
, new_clauses2
);
10174 gimple_seq_add_seq (gimple_omp_body_ptr (scan_stmt3
), scan1_list
);
10175 gimple_seq_add_seq (gimple_omp_body_ptr (input_stmt4
), input2_list
);
10177 gsi_insert_seq_after (&input3_gsi
, gimple_omp_body (input_stmt3
),
10179 gsi_remove (&input3_gsi
, true);
10180 gsi_insert_seq_after (&scan3_gsi
, gimple_omp_body (scan_stmt3
),
10182 gsi_remove (&scan3_gsi
, true);
10183 gsi_insert_seq_after (&input4_gsi
, gimple_omp_body (input_stmt4
),
10185 gsi_remove (&input4_gsi
, true);
10186 gsi_insert_seq_after (&scan4_gsi
, gimple_omp_body (scan_stmt4
),
10188 gsi_remove (&scan4_gsi
, true);
10192 gimple_omp_set_body (scan_stmt1
, scan1_list
);
10193 gimple_omp_set_body (input_stmt2
, input2_list
);
10196 gsi_insert_seq_after (&input1_gsi
, gimple_omp_body (input_stmt1
),
10198 gsi_remove (&input1_gsi
, true);
10199 gsi_insert_seq_after (&scan1_gsi
, gimple_omp_body (scan_stmt1
),
10201 gsi_remove (&scan1_gsi
, true);
10202 gsi_insert_seq_after (&input2_gsi
, gimple_omp_body (input_stmt2
),
10204 gsi_remove (&input2_gsi
, true);
10205 gsi_insert_seq_after (&scan2_gsi
, gimple_omp_body (scan_stmt2
),
10207 gsi_remove (&scan2_gsi
, true);
10209 gimple_seq_add_seq (body_p
, clist
);
10211 tree lab1
= create_artificial_label (UNKNOWN_LOCATION
);
10212 tree lab2
= create_artificial_label (UNKNOWN_LOCATION
);
10213 tree lab3
= create_artificial_label (UNKNOWN_LOCATION
);
10214 g
= gimple_build_cond (EQ_EXPR
, thread_num
, integer_zero_node
, lab1
, lab2
);
10215 gimple_seq_add_stmt (body_p
, g
);
10216 g
= gimple_build_label (lab1
);
10217 gimple_seq_add_stmt (body_p
, g
);
10218 gimple_seq_add_seq (body_p
, thr01_list
);
10219 g
= gimple_build_goto (lab3
);
10220 gimple_seq_add_stmt (body_p
, g
);
10221 g
= gimple_build_label (lab2
);
10222 gimple_seq_add_stmt (body_p
, g
);
10223 gimple_seq_add_seq (body_p
, thrn1_list
);
10224 g
= gimple_build_label (lab3
);
10225 gimple_seq_add_stmt (body_p
, g
);
10227 g
= gimple_build_assign (ivar
, size_zero_node
);
10228 gimple_seq_add_stmt (body_p
, g
);
10230 gimple_seq_add_stmt (body_p
, stmt
);
10231 gimple_seq_add_seq (body_p
, body
);
10232 gimple_seq_add_stmt (body_p
, gimple_build_omp_continue (fd
->loop
.v
,
10235 g
= gimple_build_omp_return (true);
10236 gimple_seq_add_stmt (body_p
, g
);
10237 gimple_seq_add_seq (body_p
, mdlist
);
10239 lab1
= create_artificial_label (UNKNOWN_LOCATION
);
10240 lab2
= create_artificial_label (UNKNOWN_LOCATION
);
10241 g
= gimple_build_cond (GT_EXPR
, num_threads
, integer_one_node
, lab1
, lab2
);
10242 gimple_seq_add_stmt (body_p
, g
);
10243 g
= gimple_build_label (lab1
);
10244 gimple_seq_add_stmt (body_p
, g
);
10246 g
= omp_build_barrier (NULL
);
10247 gimple_seq_add_stmt (body_p
, g
);
10249 tree down
= create_tmp_var (unsigned_type_node
);
10250 g
= gimple_build_assign (down
, build_zero_cst (unsigned_type_node
));
10251 gimple_seq_add_stmt (body_p
, g
);
10253 g
= gimple_build_assign (k
, build_one_cst (unsigned_type_node
));
10254 gimple_seq_add_stmt (body_p
, g
);
10256 tree num_threadsu
= create_tmp_var (unsigned_type_node
);
10257 g
= gimple_build_assign (num_threadsu
, NOP_EXPR
, num_threads
);
10258 gimple_seq_add_stmt (body_p
, g
);
10260 tree thread_numu
= create_tmp_var (unsigned_type_node
);
10261 g
= gimple_build_assign (thread_numu
, NOP_EXPR
, thread_num
);
10262 gimple_seq_add_stmt (body_p
, g
);
10264 tree thread_nump1
= create_tmp_var (unsigned_type_node
);
10265 g
= gimple_build_assign (thread_nump1
, PLUS_EXPR
, thread_numu
,
10266 build_int_cst (unsigned_type_node
, 1));
10267 gimple_seq_add_stmt (body_p
, g
);
10269 lab3
= create_artificial_label (UNKNOWN_LOCATION
);
10270 g
= gimple_build_label (lab3
);
10271 gimple_seq_add_stmt (body_p
, g
);
10273 tree twok
= create_tmp_var (unsigned_type_node
);
10274 g
= gimple_build_assign (twok
, LSHIFT_EXPR
, k
, integer_one_node
);
10275 gimple_seq_add_stmt (body_p
, g
);
10277 tree lab4
= create_artificial_label (UNKNOWN_LOCATION
);
10278 tree lab5
= create_artificial_label (UNKNOWN_LOCATION
);
10279 tree lab6
= create_artificial_label (UNKNOWN_LOCATION
);
10280 g
= gimple_build_cond (GT_EXPR
, twok
, num_threadsu
, lab4
, lab5
);
10281 gimple_seq_add_stmt (body_p
, g
);
10282 g
= gimple_build_label (lab4
);
10283 gimple_seq_add_stmt (body_p
, g
);
10284 g
= gimple_build_assign (down
, build_all_ones_cst (unsigned_type_node
));
10285 gimple_seq_add_stmt (body_p
, g
);
10286 g
= gimple_build_assign (k
, RSHIFT_EXPR
, k
, integer_one_node
);
10287 gimple_seq_add_stmt (body_p
, g
);
10289 g
= gimple_build_cond (EQ_EXPR
, k
, num_threadsu
, lab6
, lab5
);
10290 gimple_seq_add_stmt (body_p
, g
);
10291 g
= gimple_build_label (lab6
);
10292 gimple_seq_add_stmt (body_p
, g
);
10294 g
= gimple_build_assign (k
, RSHIFT_EXPR
, k
, integer_one_node
);
10295 gimple_seq_add_stmt (body_p
, g
);
10297 g
= gimple_build_label (lab5
);
10298 gimple_seq_add_stmt (body_p
, g
);
10300 g
= gimple_build_assign (twok
, LSHIFT_EXPR
, k
, integer_one_node
);
10301 gimple_seq_add_stmt (body_p
, g
);
10303 tree cplx
= create_tmp_var (build_complex_type (unsigned_type_node
, false));
10304 DECL_GIMPLE_REG_P (cplx
) = 1;
10305 g
= gimple_build_call_internal (IFN_MUL_OVERFLOW
, 2, thread_nump1
, twok
);
10306 gimple_call_set_lhs (g
, cplx
);
10307 gimple_seq_add_stmt (body_p
, g
);
10308 tree mul
= create_tmp_var (unsigned_type_node
);
10309 g
= gimple_build_assign (mul
, REALPART_EXPR
,
10310 build1 (REALPART_EXPR
, unsigned_type_node
, cplx
));
10311 gimple_seq_add_stmt (body_p
, g
);
10312 tree ovf
= create_tmp_var (unsigned_type_node
);
10313 g
= gimple_build_assign (ovf
, IMAGPART_EXPR
,
10314 build1 (IMAGPART_EXPR
, unsigned_type_node
, cplx
));
10315 gimple_seq_add_stmt (body_p
, g
);
10317 tree lab7
= create_artificial_label (UNKNOWN_LOCATION
);
10318 tree lab8
= create_artificial_label (UNKNOWN_LOCATION
);
10319 g
= gimple_build_cond (EQ_EXPR
, ovf
, build_zero_cst (unsigned_type_node
),
10321 gimple_seq_add_stmt (body_p
, g
);
10322 g
= gimple_build_label (lab7
);
10323 gimple_seq_add_stmt (body_p
, g
);
10325 tree andv
= create_tmp_var (unsigned_type_node
);
10326 g
= gimple_build_assign (andv
, BIT_AND_EXPR
, k
, down
);
10327 gimple_seq_add_stmt (body_p
, g
);
10328 tree andvm1
= create_tmp_var (unsigned_type_node
);
10329 g
= gimple_build_assign (andvm1
, PLUS_EXPR
, andv
,
10330 build_minus_one_cst (unsigned_type_node
));
10331 gimple_seq_add_stmt (body_p
, g
);
10333 g
= gimple_build_assign (l
, PLUS_EXPR
, mul
, andvm1
);
10334 gimple_seq_add_stmt (body_p
, g
);
10336 tree lab9
= create_artificial_label (UNKNOWN_LOCATION
);
10337 g
= gimple_build_cond (LT_EXPR
, l
, num_threadsu
, lab9
, lab8
);
10338 gimple_seq_add_stmt (body_p
, g
);
10339 g
= gimple_build_label (lab9
);
10340 gimple_seq_add_stmt (body_p
, g
);
10341 gimple_seq_add_seq (body_p
, reduc_list
);
10342 g
= gimple_build_label (lab8
);
10343 gimple_seq_add_stmt (body_p
, g
);
10345 tree lab10
= create_artificial_label (UNKNOWN_LOCATION
);
10346 tree lab11
= create_artificial_label (UNKNOWN_LOCATION
);
10347 tree lab12
= create_artificial_label (UNKNOWN_LOCATION
);
10348 g
= gimple_build_cond (EQ_EXPR
, down
, build_zero_cst (unsigned_type_node
),
10350 gimple_seq_add_stmt (body_p
, g
);
10351 g
= gimple_build_label (lab10
);
10352 gimple_seq_add_stmt (body_p
, g
);
10353 g
= gimple_build_assign (k
, LSHIFT_EXPR
, k
, integer_one_node
);
10354 gimple_seq_add_stmt (body_p
, g
);
10355 g
= gimple_build_goto (lab12
);
10356 gimple_seq_add_stmt (body_p
, g
);
10357 g
= gimple_build_label (lab11
);
10358 gimple_seq_add_stmt (body_p
, g
);
10359 g
= gimple_build_assign (k
, RSHIFT_EXPR
, k
, integer_one_node
);
10360 gimple_seq_add_stmt (body_p
, g
);
10361 g
= gimple_build_label (lab12
);
10362 gimple_seq_add_stmt (body_p
, g
);
10364 g
= omp_build_barrier (NULL
);
10365 gimple_seq_add_stmt (body_p
, g
);
10367 g
= gimple_build_cond (NE_EXPR
, k
, build_zero_cst (unsigned_type_node
),
10369 gimple_seq_add_stmt (body_p
, g
);
10371 g
= gimple_build_label (lab2
);
10372 gimple_seq_add_stmt (body_p
, g
);
10374 lab1
= create_artificial_label (UNKNOWN_LOCATION
);
10375 lab2
= create_artificial_label (UNKNOWN_LOCATION
);
10376 lab3
= create_artificial_label (UNKNOWN_LOCATION
);
10377 g
= gimple_build_cond (EQ_EXPR
, thread_num
, integer_zero_node
, lab1
, lab2
);
10378 gimple_seq_add_stmt (body_p
, g
);
10379 g
= gimple_build_label (lab1
);
10380 gimple_seq_add_stmt (body_p
, g
);
10381 gimple_seq_add_seq (body_p
, thr02_list
);
10382 g
= gimple_build_goto (lab3
);
10383 gimple_seq_add_stmt (body_p
, g
);
10384 g
= gimple_build_label (lab2
);
10385 gimple_seq_add_stmt (body_p
, g
);
10386 gimple_seq_add_seq (body_p
, thrn2_list
);
10387 g
= gimple_build_label (lab3
);
10388 gimple_seq_add_stmt (body_p
, g
);
10390 g
= gimple_build_assign (ivar
, size_zero_node
);
10391 gimple_seq_add_stmt (body_p
, g
);
10392 gimple_seq_add_stmt (body_p
, new_stmt
);
10393 gimple_seq_add_seq (body_p
, new_body
);
10395 gimple_seq new_dlist
= NULL
;
10396 lab1
= create_artificial_label (UNKNOWN_LOCATION
);
10397 lab2
= create_artificial_label (UNKNOWN_LOCATION
);
10398 tree num_threadsm1
= create_tmp_var (integer_type_node
);
10399 g
= gimple_build_assign (num_threadsm1
, PLUS_EXPR
, num_threads
,
10400 integer_minus_one_node
);
10401 gimple_seq_add_stmt (&new_dlist
, g
);
10402 g
= gimple_build_cond (EQ_EXPR
, thread_num
, num_threadsm1
, lab1
, lab2
);
10403 gimple_seq_add_stmt (&new_dlist
, g
);
10404 g
= gimple_build_label (lab1
);
10405 gimple_seq_add_stmt (&new_dlist
, g
);
10406 gimple_seq_add_seq (&new_dlist
, last_list
);
10407 g
= gimple_build_label (lab2
);
10408 gimple_seq_add_stmt (&new_dlist
, g
);
10409 gimple_seq_add_seq (&new_dlist
, *dlist
);
10410 *dlist
= new_dlist
;
10413 /* Lower code for an OMP loop directive. */
10416 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10418 tree
*rhs_p
, block
;
10419 struct omp_for_data fd
, *fdp
= NULL
;
10420 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
10422 gimple_seq omp_for_body
, body
, dlist
, tred_ilist
= NULL
, tred_dlist
= NULL
;
10423 gimple_seq cnt_list
= NULL
, clist
= NULL
;
10424 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
10427 push_gimplify_context ();
10429 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
10431 block
= make_node (BLOCK
);
10432 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10433 /* Replace at gsi right away, so that 'stmt' is no member
10434 of a sequence anymore as we're going to add to a different
10436 gsi_replace (gsi_p
, new_stmt
, true);
10438 /* Move declaration of temporaries in the loop body before we make
10440 omp_for_body
= gimple_omp_body (stmt
);
10441 if (!gimple_seq_empty_p (omp_for_body
)
10442 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
10445 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
10446 tree vars
= gimple_bind_vars (inner_bind
);
10447 gimple_bind_append_vars (new_stmt
, vars
);
10448 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10449 keep them on the inner_bind and it's block. */
10450 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
10451 if (gimple_bind_block (inner_bind
))
10452 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
10455 if (gimple_omp_for_combined_into_p (stmt
))
10457 omp_extract_for_data (stmt
, &fd
, NULL
);
10460 /* We need two temporaries with fd.loop.v type (istart/iend)
10461 and then (fd.collapse - 1) temporaries with the same
10462 type for count2 ... countN-1 vars if not constant. */
10464 tree type
= fd
.iter_type
;
10465 if (fd
.collapse
> 1
10466 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
10467 count
+= fd
.collapse
- 1;
10469 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
10470 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
10471 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
10473 tree clauses
= *pc
;
10476 = omp_find_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
10477 OMP_CLAUSE__LOOPTEMP_
);
10478 if (ctx
->simt_stmt
)
10479 simtc
= omp_find_clause (gimple_omp_for_clauses (ctx
->simt_stmt
),
10480 OMP_CLAUSE__LOOPTEMP_
);
10481 for (i
= 0; i
< count
; i
++)
10486 gcc_assert (outerc
);
10487 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
10488 outerc
= omp_find_clause (OMP_CLAUSE_CHAIN (outerc
),
10489 OMP_CLAUSE__LOOPTEMP_
);
10493 /* If there are 2 adjacent SIMD stmts, one with _simt_
10494 clause, another without, make sure they have the same
10495 decls in _looptemp_ clauses, because the outer stmt
10496 they are combined into will look up just one inner_stmt. */
10497 if (ctx
->simt_stmt
)
10498 temp
= OMP_CLAUSE_DECL (simtc
);
10500 temp
= create_tmp_var (type
);
10501 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
10503 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
10504 OMP_CLAUSE_DECL (*pc
) = temp
;
10505 pc
= &OMP_CLAUSE_CHAIN (*pc
);
10506 if (ctx
->simt_stmt
)
10507 simtc
= omp_find_clause (OMP_CLAUSE_CHAIN (simtc
),
10508 OMP_CLAUSE__LOOPTEMP_
);
10513 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10517 = omp_task_reductions_find_first (gimple_omp_for_clauses (stmt
), OMP_FOR
,
10518 OMP_CLAUSE_REDUCTION
);
10519 tree rtmp
= NULL_TREE
;
10522 tree type
= build_pointer_type (pointer_sized_int_node
);
10523 tree temp
= create_tmp_var (type
);
10524 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__REDUCTEMP_
);
10525 OMP_CLAUSE_DECL (c
) = temp
;
10526 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (stmt
);
10527 gimple_omp_for_set_clauses (stmt
, c
);
10528 lower_omp_task_reductions (ctx
, OMP_FOR
,
10529 gimple_omp_for_clauses (stmt
),
10530 &tred_ilist
, &tred_dlist
);
10532 rtmp
= make_ssa_name (type
);
10533 gimple_seq_add_stmt (&body
, gimple_build_assign (rtmp
, temp
));
10536 lower_lastprivate_conditional_clauses (gimple_omp_for_clauses_ptr (stmt
),
10539 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
10541 gimple_seq_add_seq (rclauses
? &tred_ilist
: &body
,
10542 gimple_omp_for_pre_body (stmt
));
10544 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10546 /* Lower the header expressions. At this point, we can assume that
10547 the header is of the form:
10549 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10551 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10552 using the .omp_data_s mapping, if needed. */
10553 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
10555 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
10556 if (!is_gimple_min_invariant (*rhs_p
))
10557 *rhs_p
= get_formal_tmp_var (*rhs_p
, &cnt_list
);
10558 else if (TREE_CODE (*rhs_p
) == ADDR_EXPR
)
10559 recompute_tree_invariant_for_addr_expr (*rhs_p
);
10561 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
10562 if (!is_gimple_min_invariant (*rhs_p
))
10563 *rhs_p
= get_formal_tmp_var (*rhs_p
, &cnt_list
);
10564 else if (TREE_CODE (*rhs_p
) == ADDR_EXPR
)
10565 recompute_tree_invariant_for_addr_expr (*rhs_p
);
10567 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
10568 if (!is_gimple_min_invariant (*rhs_p
))
10569 *rhs_p
= get_formal_tmp_var (*rhs_p
, &cnt_list
);
10572 gimple_seq_add_seq (&tred_ilist
, cnt_list
);
10574 gimple_seq_add_seq (&body
, cnt_list
);
10576 /* Once lowered, extract the bounds and clauses. */
10577 omp_extract_for_data (stmt
, &fd
, NULL
);
10579 if (is_gimple_omp_oacc (ctx
->stmt
)
10580 && !ctx_in_oacc_kernels_region (ctx
))
10581 lower_oacc_head_tail (gimple_location (stmt
),
10582 gimple_omp_for_clauses (stmt
),
10583 &oacc_head
, &oacc_tail
, ctx
);
10585 /* Add OpenACC partitioning and reduction markers just before the loop. */
10587 gimple_seq_add_seq (&body
, oacc_head
);
10589 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, &clist
, ctx
);
10591 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
10592 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10593 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
10594 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
10596 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
10597 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
10598 OMP_CLAUSE_LINEAR_STEP (c
)
10599 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
10603 bool phony_loop
= (gimple_omp_for_kind (stmt
) != GF_OMP_FOR_KIND_GRID_LOOP
10604 && gimple_omp_for_grid_phony (stmt
));
10605 if ((ctx
->scan_inclusive
|| ctx
->scan_exclusive
)
10606 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
10608 gcc_assert (!phony_loop
);
10609 lower_omp_for_scan (&body
, &dlist
, stmt
, &fd
, ctx
);
10614 gimple_seq_add_stmt (&body
, stmt
);
10615 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
10619 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
10622 /* After the loop, add exit clauses. */
10623 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, &clist
, ctx
);
10627 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
10628 gcall
*g
= gimple_build_call (fndecl
, 0);
10629 gimple_seq_add_stmt (&body
, g
);
10630 gimple_seq_add_seq (&body
, clist
);
10631 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
10632 g
= gimple_build_call (fndecl
, 0);
10633 gimple_seq_add_stmt (&body
, g
);
10636 if (ctx
->cancellable
)
10637 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
10639 gimple_seq_add_seq (&body
, dlist
);
10643 gimple_seq_add_seq (&tred_ilist
, body
);
10647 body
= maybe_catch_exception (body
);
10651 /* Region exit marker goes at the end of the loop body. */
10652 gimple
*g
= gimple_build_omp_return (fd
.have_nowait
);
10653 gimple_seq_add_stmt (&body
, g
);
10655 gimple_seq_add_seq (&body
, tred_dlist
);
10657 maybe_add_implicit_barrier_cancel (ctx
, g
, &body
);
10660 OMP_CLAUSE_DECL (rclauses
) = rtmp
;
10663 /* Add OpenACC joining and reduction markers just after the loop. */
10665 gimple_seq_add_seq (&body
, oacc_tail
);
10667 pop_gimplify_context (new_stmt
);
10669 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10670 maybe_remove_omp_member_access_dummy_vars (new_stmt
);
10671 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
10672 if (BLOCK_VARS (block
))
10673 TREE_USED (block
) = 1;
10675 gimple_bind_set_body (new_stmt
, body
);
10676 gimple_omp_set_body (stmt
, NULL
);
10677 gimple_omp_for_set_pre_body (stmt
, NULL
);
10680 /* Callback for walk_stmts. Check if the current statement only contains
10681 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10684 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
10685 bool *handled_ops_p
,
10686 struct walk_stmt_info
*wi
)
10688 int *info
= (int *) wi
->info
;
10689 gimple
*stmt
= gsi_stmt (*gsi_p
);
10691 *handled_ops_p
= true;
10692 switch (gimple_code (stmt
))
10698 case GIMPLE_OMP_FOR
:
10699 case GIMPLE_OMP_SECTIONS
:
10700 *info
= *info
== 0 ? 1 : -1;
10709 struct omp_taskcopy_context
10711 /* This field must be at the beginning, as we do "inheritance": Some
10712 callback functions for tree-inline.c (e.g., omp_copy_decl)
10713 receive a copy_body_data pointer that is up-casted to an
10714 omp_context pointer. */
10720 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
10722 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
10724 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
10725 return create_tmp_var (TREE_TYPE (var
));
10731 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
10733 tree name
, new_fields
= NULL
, type
, f
;
10735 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
10736 name
= DECL_NAME (TYPE_NAME (orig_type
));
10737 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
10738 TYPE_DECL
, name
, type
);
10739 TYPE_NAME (type
) = name
;
10741 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
10743 tree new_f
= copy_node (f
);
10744 DECL_CONTEXT (new_f
) = type
;
10745 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
10746 TREE_CHAIN (new_f
) = new_fields
;
10747 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10748 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10749 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
10751 new_fields
= new_f
;
10752 tcctx
->cb
.decl_map
->put (f
, new_f
);
10754 TYPE_FIELDS (type
) = nreverse (new_fields
);
10755 layout_type (type
);
10759 /* Create task copyfn. */
10762 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
10764 struct function
*child_cfun
;
10765 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
10766 tree record_type
, srecord_type
, bind
, list
;
10767 bool record_needs_remap
= false, srecord_needs_remap
= false;
10769 struct omp_taskcopy_context tcctx
;
10770 location_t loc
= gimple_location (task_stmt
);
10771 size_t looptempno
= 0;
10773 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
10774 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
10775 gcc_assert (child_cfun
->cfg
== NULL
);
10776 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
10778 /* Reset DECL_CONTEXT on function arguments. */
10779 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
10780 DECL_CONTEXT (t
) = child_fn
;
10782 /* Populate the function. */
10783 push_gimplify_context ();
10784 push_cfun (child_cfun
);
10786 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
10787 TREE_SIDE_EFFECTS (bind
) = 1;
10789 DECL_SAVED_TREE (child_fn
) = bind
;
10790 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
10792 /* Remap src and dst argument types if needed. */
10793 record_type
= ctx
->record_type
;
10794 srecord_type
= ctx
->srecord_type
;
10795 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
10796 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10798 record_needs_remap
= true;
10801 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
10802 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10804 srecord_needs_remap
= true;
10808 if (record_needs_remap
|| srecord_needs_remap
)
10810 memset (&tcctx
, '\0', sizeof (tcctx
));
10811 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
10812 tcctx
.cb
.dst_fn
= child_fn
;
10813 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
10814 gcc_checking_assert (tcctx
.cb
.src_node
);
10815 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
10816 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
10817 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
10818 tcctx
.cb
.eh_lp_nr
= 0;
10819 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
10820 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
10823 if (record_needs_remap
)
10824 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
10825 if (srecord_needs_remap
)
10826 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
10829 tcctx
.cb
.decl_map
= NULL
;
10831 arg
= DECL_ARGUMENTS (child_fn
);
10832 TREE_TYPE (arg
) = build_pointer_type (record_type
);
10833 sarg
= DECL_CHAIN (arg
);
10834 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
10836 /* First pass: initialize temporaries used in record_type and srecord_type
10837 sizes and field offsets. */
10838 if (tcctx
.cb
.decl_map
)
10839 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10840 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10844 decl
= OMP_CLAUSE_DECL (c
);
10845 p
= tcctx
.cb
.decl_map
->get (decl
);
10848 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10849 sf
= (tree
) n
->value
;
10850 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10851 src
= build_simple_mem_ref_loc (loc
, sarg
);
10852 src
= omp_build_component_ref (src
, sf
);
10853 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
10854 append_to_statement_list (t
, &list
);
10857 /* Second pass: copy shared var pointers and copy construct non-VLA
10858 firstprivate vars. */
10859 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10860 switch (OMP_CLAUSE_CODE (c
))
10862 splay_tree_key key
;
10863 case OMP_CLAUSE_SHARED
:
10864 decl
= OMP_CLAUSE_DECL (c
);
10865 key
= (splay_tree_key
) decl
;
10866 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
10867 key
= (splay_tree_key
) &DECL_UID (decl
);
10868 n
= splay_tree_lookup (ctx
->field_map
, key
);
10871 f
= (tree
) n
->value
;
10872 if (tcctx
.cb
.decl_map
)
10873 f
= *tcctx
.cb
.decl_map
->get (f
);
10874 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
10875 sf
= (tree
) n
->value
;
10876 if (tcctx
.cb
.decl_map
)
10877 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10878 src
= build_simple_mem_ref_loc (loc
, sarg
);
10879 src
= omp_build_component_ref (src
, sf
);
10880 dst
= build_simple_mem_ref_loc (loc
, arg
);
10881 dst
= omp_build_component_ref (dst
, f
);
10882 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10883 append_to_statement_list (t
, &list
);
10885 case OMP_CLAUSE_REDUCTION
:
10886 case OMP_CLAUSE_IN_REDUCTION
:
10887 decl
= OMP_CLAUSE_DECL (c
);
10888 if (TREE_CODE (decl
) == MEM_REF
)
10890 decl
= TREE_OPERAND (decl
, 0);
10891 if (TREE_CODE (decl
) == POINTER_PLUS_EXPR
)
10892 decl
= TREE_OPERAND (decl
, 0);
10893 if (TREE_CODE (decl
) == INDIRECT_REF
10894 || TREE_CODE (decl
) == ADDR_EXPR
)
10895 decl
= TREE_OPERAND (decl
, 0);
10897 key
= (splay_tree_key
) decl
;
10898 n
= splay_tree_lookup (ctx
->field_map
, key
);
10901 f
= (tree
) n
->value
;
10902 if (tcctx
.cb
.decl_map
)
10903 f
= *tcctx
.cb
.decl_map
->get (f
);
10904 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
10905 sf
= (tree
) n
->value
;
10906 if (tcctx
.cb
.decl_map
)
10907 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10908 src
= build_simple_mem_ref_loc (loc
, sarg
);
10909 src
= omp_build_component_ref (src
, sf
);
10910 if (decl
!= OMP_CLAUSE_DECL (c
)
10911 && TREE_CODE (TREE_TYPE (decl
)) == REFERENCE_TYPE
10912 && TREE_CODE (TREE_TYPE (TREE_TYPE (decl
))) == POINTER_TYPE
)
10913 src
= build_simple_mem_ref_loc (loc
, src
);
10914 dst
= build_simple_mem_ref_loc (loc
, arg
);
10915 dst
= omp_build_component_ref (dst
, f
);
10916 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10917 append_to_statement_list (t
, &list
);
10919 case OMP_CLAUSE__LOOPTEMP_
:
10920 /* Fields for first two _looptemp_ clauses are initialized by
10921 GOMP_taskloop*, the rest are handled like firstprivate. */
10922 if (looptempno
< 2)
10928 case OMP_CLAUSE__REDUCTEMP_
:
10929 case OMP_CLAUSE_FIRSTPRIVATE
:
10930 decl
= OMP_CLAUSE_DECL (c
);
10931 if (is_variable_sized (decl
))
10933 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10936 f
= (tree
) n
->value
;
10937 if (tcctx
.cb
.decl_map
)
10938 f
= *tcctx
.cb
.decl_map
->get (f
);
10939 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10942 sf
= (tree
) n
->value
;
10943 if (tcctx
.cb
.decl_map
)
10944 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10945 src
= build_simple_mem_ref_loc (loc
, sarg
);
10946 src
= omp_build_component_ref (src
, sf
);
10947 if (use_pointer_for_field (decl
, NULL
) || omp_is_reference (decl
))
10948 src
= build_simple_mem_ref_loc (loc
, src
);
10952 dst
= build_simple_mem_ref_loc (loc
, arg
);
10953 dst
= omp_build_component_ref (dst
, f
);
10954 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_FIRSTPRIVATE
)
10955 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10957 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10958 append_to_statement_list (t
, &list
);
10960 case OMP_CLAUSE_PRIVATE
:
10961 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
10963 decl
= OMP_CLAUSE_DECL (c
);
10964 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10965 f
= (tree
) n
->value
;
10966 if (tcctx
.cb
.decl_map
)
10967 f
= *tcctx
.cb
.decl_map
->get (f
);
10968 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10971 sf
= (tree
) n
->value
;
10972 if (tcctx
.cb
.decl_map
)
10973 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10974 src
= build_simple_mem_ref_loc (loc
, sarg
);
10975 src
= omp_build_component_ref (src
, sf
);
10976 if (use_pointer_for_field (decl
, NULL
))
10977 src
= build_simple_mem_ref_loc (loc
, src
);
10981 dst
= build_simple_mem_ref_loc (loc
, arg
);
10982 dst
= omp_build_component_ref (dst
, f
);
10983 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10984 append_to_statement_list (t
, &list
);
10990 /* Last pass: handle VLA firstprivates. */
10991 if (tcctx
.cb
.decl_map
)
10992 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10993 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10997 decl
= OMP_CLAUSE_DECL (c
);
10998 if (!is_variable_sized (decl
))
11000 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
11003 f
= (tree
) n
->value
;
11004 f
= *tcctx
.cb
.decl_map
->get (f
);
11005 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
11006 ind
= DECL_VALUE_EXPR (decl
);
11007 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
11008 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
11009 n
= splay_tree_lookup (ctx
->sfield_map
,
11010 (splay_tree_key
) TREE_OPERAND (ind
, 0));
11011 sf
= (tree
) n
->value
;
11012 sf
= *tcctx
.cb
.decl_map
->get (sf
);
11013 src
= build_simple_mem_ref_loc (loc
, sarg
);
11014 src
= omp_build_component_ref (src
, sf
);
11015 src
= build_simple_mem_ref_loc (loc
, src
);
11016 dst
= build_simple_mem_ref_loc (loc
, arg
);
11017 dst
= omp_build_component_ref (dst
, f
);
11018 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
11019 append_to_statement_list (t
, &list
);
11020 n
= splay_tree_lookup (ctx
->field_map
,
11021 (splay_tree_key
) TREE_OPERAND (ind
, 0));
11022 df
= (tree
) n
->value
;
11023 df
= *tcctx
.cb
.decl_map
->get (df
);
11024 ptr
= build_simple_mem_ref_loc (loc
, arg
);
11025 ptr
= omp_build_component_ref (ptr
, df
);
11026 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
11027 build_fold_addr_expr_loc (loc
, dst
));
11028 append_to_statement_list (t
, &list
);
11031 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
11032 append_to_statement_list (t
, &list
);
11034 if (tcctx
.cb
.decl_map
)
11035 delete tcctx
.cb
.decl_map
;
11036 pop_gimplify_context (NULL
);
11037 BIND_EXPR_BODY (bind
) = list
;
11042 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
11046 size_t cnt
[4] = { 0, 0, 0, 0 }, idx
= 2, i
;
11048 clauses
= omp_find_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
11049 gcc_assert (clauses
);
11050 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11051 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
11052 switch (OMP_CLAUSE_DEPEND_KIND (c
))
11054 case OMP_CLAUSE_DEPEND_LAST
:
11055 /* Lowering already done at gimplification. */
11057 case OMP_CLAUSE_DEPEND_IN
:
11060 case OMP_CLAUSE_DEPEND_OUT
:
11061 case OMP_CLAUSE_DEPEND_INOUT
:
11064 case OMP_CLAUSE_DEPEND_MUTEXINOUTSET
:
11067 case OMP_CLAUSE_DEPEND_DEPOBJ
:
11070 case OMP_CLAUSE_DEPEND_SOURCE
:
11071 case OMP_CLAUSE_DEPEND_SINK
:
11074 gcc_unreachable ();
11076 if (cnt
[1] || cnt
[3])
11078 size_t total
= cnt
[0] + cnt
[1] + cnt
[2] + cnt
[3];
11079 tree type
= build_array_type_nelts (ptr_type_node
, total
+ idx
);
11080 tree array
= create_tmp_var (type
);
11081 TREE_ADDRESSABLE (array
) = 1;
11082 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
11086 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, 0));
11087 gimple_seq_add_stmt (iseq
, g
);
11088 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
11091 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, total
));
11092 gimple_seq_add_stmt (iseq
, g
);
11093 for (i
= 0; i
< (idx
== 5 ? 3 : 1); i
++)
11095 r
= build4 (ARRAY_REF
, ptr_type_node
, array
,
11096 size_int (i
+ 1 + (idx
== 5)), NULL_TREE
, NULL_TREE
);
11097 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, cnt
[i
]));
11098 gimple_seq_add_stmt (iseq
, g
);
11100 for (i
= 0; i
< 4; i
++)
11104 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11105 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
11109 switch (OMP_CLAUSE_DEPEND_KIND (c
))
11111 case OMP_CLAUSE_DEPEND_IN
:
11115 case OMP_CLAUSE_DEPEND_OUT
:
11116 case OMP_CLAUSE_DEPEND_INOUT
:
11120 case OMP_CLAUSE_DEPEND_MUTEXINOUTSET
:
11124 case OMP_CLAUSE_DEPEND_DEPOBJ
:
11129 gcc_unreachable ();
11131 tree t
= OMP_CLAUSE_DECL (c
);
11132 t
= fold_convert (ptr_type_node
, t
);
11133 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
11134 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
11135 NULL_TREE
, NULL_TREE
);
11136 g
= gimple_build_assign (r
, t
);
11137 gimple_seq_add_stmt (iseq
, g
);
11140 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
11141 OMP_CLAUSE_DEPEND_KIND (c
) = OMP_CLAUSE_DEPEND_LAST
;
11142 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
11143 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
11145 tree clobber
= build_clobber (type
);
11146 g
= gimple_build_assign (array
, clobber
);
11147 gimple_seq_add_stmt (oseq
, g
);
11150 /* Lower the OpenMP parallel or task directive in the current statement
11151 in GSI_P. CTX holds context information for the directive. */
11154 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11158 gimple
*stmt
= gsi_stmt (*gsi_p
);
11159 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
11160 gimple_seq par_body
;
11161 location_t loc
= gimple_location (stmt
);
11163 clauses
= gimple_omp_taskreg_clauses (stmt
);
11164 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11165 && gimple_omp_task_taskwait_p (stmt
))
11173 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
11174 par_body
= gimple_bind_body (par_bind
);
11176 child_fn
= ctx
->cb
.dst_fn
;
11177 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11178 && !gimple_omp_parallel_combined_p (stmt
))
11180 struct walk_stmt_info wi
;
11183 memset (&wi
, 0, sizeof (wi
));
11185 wi
.val_only
= true;
11186 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
11188 gimple_omp_parallel_set_combined_p (stmt
, true);
11190 gimple_seq dep_ilist
= NULL
;
11191 gimple_seq dep_olist
= NULL
;
11192 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11193 && omp_find_clause (clauses
, OMP_CLAUSE_DEPEND
))
11195 push_gimplify_context ();
11196 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11197 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
11198 &dep_ilist
, &dep_olist
);
11201 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11202 && gimple_omp_task_taskwait_p (stmt
))
11206 gsi_replace (gsi_p
, dep_bind
, true);
11207 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11208 gimple_bind_add_stmt (dep_bind
, stmt
);
11209 gimple_bind_add_seq (dep_bind
, dep_olist
);
11210 pop_gimplify_context (dep_bind
);
11215 if (ctx
->srecord_type
)
11216 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
11218 gimple_seq tskred_ilist
= NULL
;
11219 gimple_seq tskred_olist
= NULL
;
11220 if ((is_task_ctx (ctx
)
11221 && gimple_omp_task_taskloop_p (ctx
->stmt
)
11222 && omp_find_clause (gimple_omp_task_clauses (ctx
->stmt
),
11223 OMP_CLAUSE_REDUCTION
))
11224 || (is_parallel_ctx (ctx
)
11225 && omp_find_clause (gimple_omp_parallel_clauses (stmt
),
11226 OMP_CLAUSE__REDUCTEMP_
)))
11228 if (dep_bind
== NULL
)
11230 push_gimplify_context ();
11231 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11233 lower_omp_task_reductions (ctx
, is_task_ctx (ctx
) ? OMP_TASKLOOP
11235 gimple_omp_taskreg_clauses (ctx
->stmt
),
11236 &tskred_ilist
, &tskred_olist
);
11239 push_gimplify_context ();
11241 gimple_seq par_olist
= NULL
;
11242 gimple_seq par_ilist
= NULL
;
11243 gimple_seq par_rlist
= NULL
;
11244 bool phony_construct
= gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11245 && gimple_omp_parallel_grid_phony (as_a
<gomp_parallel
*> (stmt
));
11246 if (phony_construct
&& ctx
->record_type
)
11248 gcc_checking_assert (!ctx
->receiver_decl
);
11249 ctx
->receiver_decl
= create_tmp_var
11250 (build_reference_type (ctx
->record_type
), ".omp_rec");
11252 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
11253 lower_omp (&par_body
, ctx
);
11254 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
11255 lower_reduction_clauses (clauses
, &par_rlist
, NULL
, ctx
);
11257 /* Declare all the variables created by mapping and the variables
11258 declared in the scope of the parallel body. */
11259 record_vars_into (ctx
->block_vars
, child_fn
);
11260 maybe_remove_omp_member_access_dummy_vars (par_bind
);
11261 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
11263 if (ctx
->record_type
)
11266 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
11267 : ctx
->record_type
, ".omp_data_o");
11268 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11269 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11270 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
11273 gimple_seq olist
= NULL
;
11274 gimple_seq ilist
= NULL
;
11275 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
11276 lower_send_shared_vars (&ilist
, &olist
, ctx
);
11278 if (ctx
->record_type
)
11280 tree clobber
= build_clobber (TREE_TYPE (ctx
->sender_decl
));
11281 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11285 /* Once all the expansions are done, sequence all the different
11286 fragments inside gimple_omp_body. */
11288 gimple_seq new_body
= NULL
;
11290 if (ctx
->record_type
)
11292 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11293 /* fixup_child_record_type might have changed receiver_decl's type. */
11294 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11295 gimple_seq_add_stmt (&new_body
,
11296 gimple_build_assign (ctx
->receiver_decl
, t
));
11299 gimple_seq_add_seq (&new_body
, par_ilist
);
11300 gimple_seq_add_seq (&new_body
, par_body
);
11301 gimple_seq_add_seq (&new_body
, par_rlist
);
11302 if (ctx
->cancellable
)
11303 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
11304 gimple_seq_add_seq (&new_body
, par_olist
);
11305 new_body
= maybe_catch_exception (new_body
);
11306 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
11307 gimple_seq_add_stmt (&new_body
,
11308 gimple_build_omp_continue (integer_zero_node
,
11309 integer_zero_node
));
11310 if (!phony_construct
)
11312 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11313 gimple_omp_set_body (stmt
, new_body
);
11316 if (dep_bind
&& gimple_bind_block (par_bind
) == NULL_TREE
)
11317 bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11319 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
11320 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
11321 gimple_bind_add_seq (bind
, ilist
);
11322 if (!phony_construct
)
11323 gimple_bind_add_stmt (bind
, stmt
);
11325 gimple_bind_add_seq (bind
, new_body
);
11326 gimple_bind_add_seq (bind
, olist
);
11328 pop_gimplify_context (NULL
);
11332 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11333 gimple_bind_add_seq (dep_bind
, tskred_ilist
);
11334 gimple_bind_add_stmt (dep_bind
, bind
);
11335 gimple_bind_add_seq (dep_bind
, tskred_olist
);
11336 gimple_bind_add_seq (dep_bind
, dep_olist
);
11337 pop_gimplify_context (dep_bind
);
11341 /* Lower the GIMPLE_OMP_TARGET in the current statement
11342 in GSI_P. CTX holds context information for the directive. */
11345 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11348 tree child_fn
, t
, c
;
11349 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
11350 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
11351 gimple_seq tgt_body
, olist
, ilist
, fplist
, new_body
;
11352 location_t loc
= gimple_location (stmt
);
11353 bool offloaded
, data_region
;
11354 unsigned int map_cnt
= 0;
11356 offloaded
= is_gimple_omp_offloaded (stmt
);
11357 switch (gimple_omp_target_kind (stmt
))
11359 case GF_OMP_TARGET_KIND_REGION
:
11360 case GF_OMP_TARGET_KIND_UPDATE
:
11361 case GF_OMP_TARGET_KIND_ENTER_DATA
:
11362 case GF_OMP_TARGET_KIND_EXIT_DATA
:
11363 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11364 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11365 case GF_OMP_TARGET_KIND_OACC_SERIAL
:
11366 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11367 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11368 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
11369 data_region
= false;
11371 case GF_OMP_TARGET_KIND_DATA
:
11372 case GF_OMP_TARGET_KIND_OACC_DATA
:
11373 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
11374 data_region
= true;
11377 gcc_unreachable ();
11380 clauses
= gimple_omp_target_clauses (stmt
);
11382 gimple_seq dep_ilist
= NULL
;
11383 gimple_seq dep_olist
= NULL
;
11384 if (omp_find_clause (clauses
, OMP_CLAUSE_DEPEND
))
11386 push_gimplify_context ();
11387 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11388 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
11389 &dep_ilist
, &dep_olist
);
11396 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
11397 tgt_body
= gimple_bind_body (tgt_bind
);
11399 else if (data_region
)
11400 tgt_body
= gimple_omp_body (stmt
);
11401 child_fn
= ctx
->cb
.dst_fn
;
11403 push_gimplify_context ();
11406 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11407 switch (OMP_CLAUSE_CODE (c
))
11413 case OMP_CLAUSE_MAP
:
11415 /* First check what we're prepared to handle in the following. */
11416 switch (OMP_CLAUSE_MAP_KIND (c
))
11418 case GOMP_MAP_ALLOC
:
11420 case GOMP_MAP_FROM
:
11421 case GOMP_MAP_TOFROM
:
11422 case GOMP_MAP_POINTER
:
11423 case GOMP_MAP_TO_PSET
:
11424 case GOMP_MAP_DELETE
:
11425 case GOMP_MAP_RELEASE
:
11426 case GOMP_MAP_ALWAYS_TO
:
11427 case GOMP_MAP_ALWAYS_FROM
:
11428 case GOMP_MAP_ALWAYS_TOFROM
:
11429 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
11430 case GOMP_MAP_FIRSTPRIVATE_REFERENCE
:
11431 case GOMP_MAP_STRUCT
:
11432 case GOMP_MAP_ALWAYS_POINTER
:
11434 case GOMP_MAP_IF_PRESENT
:
11435 case GOMP_MAP_FORCE_ALLOC
:
11436 case GOMP_MAP_FORCE_TO
:
11437 case GOMP_MAP_FORCE_FROM
:
11438 case GOMP_MAP_FORCE_TOFROM
:
11439 case GOMP_MAP_FORCE_PRESENT
:
11440 case GOMP_MAP_FORCE_DEVICEPTR
:
11441 case GOMP_MAP_DEVICE_RESIDENT
:
11442 case GOMP_MAP_LINK
:
11443 case GOMP_MAP_ATTACH
:
11444 case GOMP_MAP_DETACH
:
11445 case GOMP_MAP_FORCE_DETACH
:
11446 gcc_assert (is_gimple_omp_oacc (stmt
));
11449 gcc_unreachable ();
11453 case OMP_CLAUSE_TO
:
11454 case OMP_CLAUSE_FROM
:
11456 var
= OMP_CLAUSE_DECL (c
);
11459 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11460 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11461 && (OMP_CLAUSE_MAP_KIND (c
)
11462 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
11467 if (DECL_SIZE (var
)
11468 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
11470 tree var2
= DECL_VALUE_EXPR (var
);
11471 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
11472 var2
= TREE_OPERAND (var2
, 0);
11473 gcc_assert (DECL_P (var2
));
11478 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11479 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
11480 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
11482 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11484 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
11485 && varpool_node::get_create (var
)->offloadable
)
11488 tree type
= build_pointer_type (TREE_TYPE (var
));
11489 tree new_var
= lookup_decl (var
, ctx
);
11490 x
= create_tmp_var_raw (type
, get_name (new_var
));
11491 gimple_add_tmp_var (x
);
11492 x
= build_simple_mem_ref (x
);
11493 SET_DECL_VALUE_EXPR (new_var
, x
);
11494 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11499 if (!maybe_lookup_field (var
, ctx
))
11502 /* Don't remap compute constructs' reduction variables, because the
11503 intermediate result must be local to each gang. */
11504 if (offloaded
&& !(OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11505 && OMP_CLAUSE_MAP_IN_REDUCTION (c
)))
11507 x
= build_receiver_ref (var
, true, ctx
);
11508 tree new_var
= lookup_decl (var
, ctx
);
11510 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11511 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11512 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11513 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11514 x
= build_simple_mem_ref (x
);
11515 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
11517 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
11518 if (omp_is_reference (new_var
)
11519 && (TREE_CODE (TREE_TYPE (new_var
)) != POINTER_TYPE
11520 || DECL_BY_REFERENCE (var
)))
11522 /* Create a local object to hold the instance
11524 tree type
= TREE_TYPE (TREE_TYPE (new_var
));
11525 const char *id
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
11526 tree inst
= create_tmp_var (type
, id
);
11527 gimplify_assign (inst
, fold_indirect_ref (x
), &fplist
);
11528 x
= build_fold_addr_expr (inst
);
11530 gimplify_assign (new_var
, x
, &fplist
);
11532 else if (DECL_P (new_var
))
11534 SET_DECL_VALUE_EXPR (new_var
, x
);
11535 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11538 gcc_unreachable ();
11543 case OMP_CLAUSE_FIRSTPRIVATE
:
11544 if (is_oacc_parallel_or_serial (ctx
))
11545 goto oacc_firstprivate
;
11547 var
= OMP_CLAUSE_DECL (c
);
11548 if (!omp_is_reference (var
)
11549 && !is_gimple_reg_type (TREE_TYPE (var
)))
11551 tree new_var
= lookup_decl (var
, ctx
);
11552 if (is_variable_sized (var
))
11554 tree pvar
= DECL_VALUE_EXPR (var
);
11555 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
11556 pvar
= TREE_OPERAND (pvar
, 0);
11557 gcc_assert (DECL_P (pvar
));
11558 tree new_pvar
= lookup_decl (pvar
, ctx
);
11559 x
= build_fold_indirect_ref (new_pvar
);
11560 TREE_THIS_NOTRAP (x
) = 1;
11563 x
= build_receiver_ref (var
, true, ctx
);
11564 SET_DECL_VALUE_EXPR (new_var
, x
);
11565 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11569 case OMP_CLAUSE_PRIVATE
:
11570 if (is_gimple_omp_oacc (ctx
->stmt
))
11572 var
= OMP_CLAUSE_DECL (c
);
11573 if (is_variable_sized (var
))
11575 tree new_var
= lookup_decl (var
, ctx
);
11576 tree pvar
= DECL_VALUE_EXPR (var
);
11577 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
11578 pvar
= TREE_OPERAND (pvar
, 0);
11579 gcc_assert (DECL_P (pvar
));
11580 tree new_pvar
= lookup_decl (pvar
, ctx
);
11581 x
= build_fold_indirect_ref (new_pvar
);
11582 TREE_THIS_NOTRAP (x
) = 1;
11583 SET_DECL_VALUE_EXPR (new_var
, x
);
11584 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11588 case OMP_CLAUSE_USE_DEVICE_PTR
:
11589 case OMP_CLAUSE_USE_DEVICE_ADDR
:
11590 case OMP_CLAUSE_IS_DEVICE_PTR
:
11591 var
= OMP_CLAUSE_DECL (c
);
11593 if (is_variable_sized (var
))
11595 tree new_var
= lookup_decl (var
, ctx
);
11596 tree pvar
= DECL_VALUE_EXPR (var
);
11597 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
11598 pvar
= TREE_OPERAND (pvar
, 0);
11599 gcc_assert (DECL_P (pvar
));
11600 tree new_pvar
= lookup_decl (pvar
, ctx
);
11601 x
= build_fold_indirect_ref (new_pvar
);
11602 TREE_THIS_NOTRAP (x
) = 1;
11603 SET_DECL_VALUE_EXPR (new_var
, x
);
11604 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11606 else if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_ADDR
11607 && !omp_is_reference (var
)
11608 && !omp_is_allocatable_or_ptr (var
)
11609 && !lang_hooks
.decls
.omp_array_data (var
, true))
11610 || TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11612 tree new_var
= lookup_decl (var
, ctx
);
11613 tree type
= build_pointer_type (TREE_TYPE (var
));
11614 x
= create_tmp_var_raw (type
, get_name (new_var
));
11615 gimple_add_tmp_var (x
);
11616 x
= build_simple_mem_ref (x
);
11617 SET_DECL_VALUE_EXPR (new_var
, x
);
11618 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11622 tree new_var
= lookup_decl (var
, ctx
);
11623 x
= create_tmp_var_raw (TREE_TYPE (new_var
), get_name (new_var
));
11624 gimple_add_tmp_var (x
);
11625 SET_DECL_VALUE_EXPR (new_var
, x
);
11626 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11633 target_nesting_level
++;
11634 lower_omp (&tgt_body
, ctx
);
11635 target_nesting_level
--;
11637 else if (data_region
)
11638 lower_omp (&tgt_body
, ctx
);
11642 /* Declare all the variables created by mapping and the variables
11643 declared in the scope of the target body. */
11644 record_vars_into (ctx
->block_vars
, child_fn
);
11645 maybe_remove_omp_member_access_dummy_vars (tgt_bind
);
11646 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
11651 if (ctx
->record_type
)
11654 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
11655 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11656 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11657 t
= make_tree_vec (3);
11658 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
11659 TREE_VEC_ELT (t
, 1)
11660 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
11661 ".omp_data_sizes");
11662 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
11663 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
11664 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
11665 tree tkind_type
= short_unsigned_type_node
;
11666 int talign_shift
= 8;
11667 TREE_VEC_ELT (t
, 2)
11668 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
11669 ".omp_data_kinds");
11670 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
11671 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
11672 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
11673 gimple_omp_target_set_data_arg (stmt
, t
);
11675 vec
<constructor_elt
, va_gc
> *vsize
;
11676 vec
<constructor_elt
, va_gc
> *vkind
;
11677 vec_alloc (vsize
, map_cnt
);
11678 vec_alloc (vkind
, map_cnt
);
11679 unsigned int map_idx
= 0;
11681 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11682 switch (OMP_CLAUSE_CODE (c
))
11684 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
11685 unsigned int talign
;
11690 case OMP_CLAUSE_MAP
:
11691 case OMP_CLAUSE_TO
:
11692 case OMP_CLAUSE_FROM
:
11693 oacc_firstprivate_map
:
11695 ovar
= OMP_CLAUSE_DECL (c
);
11696 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11697 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
11698 || (OMP_CLAUSE_MAP_KIND (c
)
11699 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
11701 if (!DECL_P (ovar
))
11703 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11704 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11706 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
11707 == get_base_address (ovar
));
11708 nc
= OMP_CLAUSE_CHAIN (c
);
11709 ovar
= OMP_CLAUSE_DECL (nc
);
11713 tree x
= build_sender_ref (ovar
, ctx
);
11715 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
11716 gimplify_assign (x
, v
, &ilist
);
11722 if (DECL_SIZE (ovar
)
11723 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
11725 tree ovar2
= DECL_VALUE_EXPR (ovar
);
11726 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
11727 ovar2
= TREE_OPERAND (ovar2
, 0);
11728 gcc_assert (DECL_P (ovar2
));
11731 if (!maybe_lookup_field (ovar
, ctx
))
11735 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
11736 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
11737 talign
= DECL_ALIGN_UNIT (ovar
);
11740 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11741 x
= build_sender_ref (ovar
, ctx
);
11743 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11744 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11745 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11746 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
11748 gcc_assert (offloaded
);
11750 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
11751 mark_addressable (avar
);
11752 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
11753 talign
= DECL_ALIGN_UNIT (avar
);
11754 avar
= build_fold_addr_expr (avar
);
11755 gimplify_assign (x
, avar
, &ilist
);
11757 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
11759 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
11760 if (!omp_is_reference (var
))
11762 if (is_gimple_reg (var
)
11763 && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
11764 TREE_NO_WARNING (var
) = 1;
11765 var
= build_fold_addr_expr (var
);
11768 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
11769 gimplify_assign (x
, var
, &ilist
);
11771 else if (is_gimple_reg (var
))
11773 gcc_assert (offloaded
);
11774 tree avar
= create_tmp_var (TREE_TYPE (var
));
11775 mark_addressable (avar
);
11776 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
11777 if (GOMP_MAP_COPY_TO_P (map_kind
)
11778 || map_kind
== GOMP_MAP_POINTER
11779 || map_kind
== GOMP_MAP_TO_PSET
11780 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11782 /* If we need to initialize a temporary
11783 with VAR because it is not addressable, and
11784 the variable hasn't been initialized yet, then
11785 we'll get a warning for the store to avar.
11786 Don't warn in that case, the mapping might
11788 TREE_NO_WARNING (var
) = 1;
11789 gimplify_assign (avar
, var
, &ilist
);
11791 avar
= build_fold_addr_expr (avar
);
11792 gimplify_assign (x
, avar
, &ilist
);
11793 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
11794 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11795 && !TYPE_READONLY (TREE_TYPE (var
)))
11797 x
= unshare_expr (x
);
11798 x
= build_simple_mem_ref (x
);
11799 gimplify_assign (var
, x
, &olist
);
11804 /* While MAP is handled explicitly by the FE,
11805 for 'target update', only the identified is passed. */
11806 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FROM
11807 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_TO
)
11808 && (omp_is_allocatable_or_ptr (var
)
11809 && omp_check_optional_argument (var
, false)))
11810 var
= build_fold_indirect_ref (var
);
11811 else if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_FROM
11812 && OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_TO
)
11813 || (!omp_is_allocatable_or_ptr (var
)
11814 && !omp_check_optional_argument (var
, false)))
11815 var
= build_fold_addr_expr (var
);
11816 gimplify_assign (x
, var
, &ilist
);
11820 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
11822 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
11823 s
= TREE_TYPE (ovar
);
11824 if (TREE_CODE (s
) == REFERENCE_TYPE
11825 || omp_check_optional_argument (ovar
, false))
11827 s
= TYPE_SIZE_UNIT (s
);
11830 s
= OMP_CLAUSE_SIZE (c
);
11831 if (s
== NULL_TREE
)
11832 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11833 s
= fold_convert (size_type_node
, s
);
11834 purpose
= size_int (map_idx
++);
11835 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11836 if (TREE_CODE (s
) != INTEGER_CST
)
11837 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11839 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
11840 switch (OMP_CLAUSE_CODE (c
))
11842 case OMP_CLAUSE_MAP
:
11843 tkind
= OMP_CLAUSE_MAP_KIND (c
);
11844 tkind_zero
= tkind
;
11845 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
11848 case GOMP_MAP_ALLOC
:
11849 case GOMP_MAP_IF_PRESENT
:
11851 case GOMP_MAP_FROM
:
11852 case GOMP_MAP_TOFROM
:
11853 case GOMP_MAP_ALWAYS_TO
:
11854 case GOMP_MAP_ALWAYS_FROM
:
11855 case GOMP_MAP_ALWAYS_TOFROM
:
11856 case GOMP_MAP_RELEASE
:
11857 case GOMP_MAP_FORCE_TO
:
11858 case GOMP_MAP_FORCE_FROM
:
11859 case GOMP_MAP_FORCE_TOFROM
:
11860 case GOMP_MAP_FORCE_PRESENT
:
11861 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
11863 case GOMP_MAP_DELETE
:
11864 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
11868 if (tkind_zero
!= tkind
)
11870 if (integer_zerop (s
))
11871 tkind
= tkind_zero
;
11872 else if (integer_nonzerop (s
))
11873 tkind_zero
= tkind
;
11876 case OMP_CLAUSE_FIRSTPRIVATE
:
11877 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
11878 tkind
= GOMP_MAP_TO
;
11879 tkind_zero
= tkind
;
11881 case OMP_CLAUSE_TO
:
11882 tkind
= GOMP_MAP_TO
;
11883 tkind_zero
= tkind
;
11885 case OMP_CLAUSE_FROM
:
11886 tkind
= GOMP_MAP_FROM
;
11887 tkind_zero
= tkind
;
11890 gcc_unreachable ();
11892 gcc_checking_assert (tkind
11893 < (HOST_WIDE_INT_C (1U) << talign_shift
));
11894 gcc_checking_assert (tkind_zero
11895 < (HOST_WIDE_INT_C (1U) << talign_shift
));
11896 talign
= ceil_log2 (talign
);
11897 tkind
|= talign
<< talign_shift
;
11898 tkind_zero
|= talign
<< talign_shift
;
11899 gcc_checking_assert (tkind
11900 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11901 gcc_checking_assert (tkind_zero
11902 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11903 if (tkind
== tkind_zero
)
11904 x
= build_int_cstu (tkind_type
, tkind
);
11907 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
11908 x
= build3 (COND_EXPR
, tkind_type
,
11909 fold_build2 (EQ_EXPR
, boolean_type_node
,
11910 unshare_expr (s
), size_zero_node
),
11911 build_int_cstu (tkind_type
, tkind_zero
),
11912 build_int_cstu (tkind_type
, tkind
));
11914 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
11919 case OMP_CLAUSE_FIRSTPRIVATE
:
11920 if (is_oacc_parallel_or_serial (ctx
))
11921 goto oacc_firstprivate_map
;
11922 ovar
= OMP_CLAUSE_DECL (c
);
11923 if (omp_is_reference (ovar
))
11924 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
11926 talign
= DECL_ALIGN_UNIT (ovar
);
11927 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11928 x
= build_sender_ref (ovar
, ctx
);
11929 tkind
= GOMP_MAP_FIRSTPRIVATE
;
11930 type
= TREE_TYPE (ovar
);
11931 if (omp_is_reference (ovar
))
11932 type
= TREE_TYPE (type
);
11933 if ((INTEGRAL_TYPE_P (type
)
11934 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
11935 || TREE_CODE (type
) == POINTER_TYPE
)
11937 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
11939 if (omp_is_reference (var
))
11940 t
= build_simple_mem_ref (var
);
11941 else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
11942 TREE_NO_WARNING (var
) = 1;
11943 if (TREE_CODE (type
) != POINTER_TYPE
)
11944 t
= fold_convert (pointer_sized_int_node
, t
);
11945 t
= fold_convert (TREE_TYPE (x
), t
);
11946 gimplify_assign (x
, t
, &ilist
);
11948 else if (omp_is_reference (var
))
11949 gimplify_assign (x
, var
, &ilist
);
11950 else if (is_gimple_reg (var
))
11952 tree avar
= create_tmp_var (TREE_TYPE (var
));
11953 mark_addressable (avar
);
11954 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
11955 TREE_NO_WARNING (var
) = 1;
11956 gimplify_assign (avar
, var
, &ilist
);
11957 avar
= build_fold_addr_expr (avar
);
11958 gimplify_assign (x
, avar
, &ilist
);
11962 var
= build_fold_addr_expr (var
);
11963 gimplify_assign (x
, var
, &ilist
);
11965 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
11967 else if (omp_is_reference (ovar
))
11968 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
11970 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11971 s
= fold_convert (size_type_node
, s
);
11972 purpose
= size_int (map_idx
++);
11973 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11974 if (TREE_CODE (s
) != INTEGER_CST
)
11975 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11977 gcc_checking_assert (tkind
11978 < (HOST_WIDE_INT_C (1U) << talign_shift
));
11979 talign
= ceil_log2 (talign
);
11980 tkind
|= talign
<< talign_shift
;
11981 gcc_checking_assert (tkind
11982 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11983 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
11984 build_int_cstu (tkind_type
, tkind
));
11987 case OMP_CLAUSE_USE_DEVICE_PTR
:
11988 case OMP_CLAUSE_USE_DEVICE_ADDR
:
11989 case OMP_CLAUSE_IS_DEVICE_PTR
:
11990 ovar
= OMP_CLAUSE_DECL (c
);
11991 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11993 if (lang_hooks
.decls
.omp_array_data (ovar
, true))
11995 tkind
= (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_IS_DEVICE_PTR
11996 ? GOMP_MAP_USE_DEVICE_PTR
: GOMP_MAP_FIRSTPRIVATE_INT
);
11997 x
= build_sender_ref ((splay_tree_key
) &DECL_NAME (ovar
), ctx
);
11999 else if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_IS_DEVICE_PTR
)
12001 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
12002 x
= build_sender_ref ((splay_tree_key
) &DECL_UID (ovar
), ctx
);
12006 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
12007 x
= build_sender_ref (ovar
, ctx
);
12010 if (is_gimple_omp_oacc (ctx
->stmt
))
12012 gcc_assert (tkind
== GOMP_MAP_USE_DEVICE_PTR
);
12014 if (OMP_CLAUSE_USE_DEVICE_PTR_IF_PRESENT (c
))
12015 tkind
= GOMP_MAP_USE_DEVICE_PTR_IF_PRESENT
;
12018 type
= TREE_TYPE (ovar
);
12019 if (lang_hooks
.decls
.omp_array_data (ovar
, true))
12020 var
= lang_hooks
.decls
.omp_array_data (ovar
, false);
12021 else if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_ADDR
12022 && !omp_is_reference (ovar
)
12023 && !omp_is_allocatable_or_ptr (ovar
))
12024 || TREE_CODE (type
) == ARRAY_TYPE
)
12025 var
= build_fold_addr_expr (var
);
12028 if (omp_is_reference (ovar
)
12029 || omp_check_optional_argument (ovar
, false)
12030 || omp_is_allocatable_or_ptr (ovar
))
12032 type
= TREE_TYPE (type
);
12033 if (TREE_CODE (type
) != ARRAY_TYPE
12034 && ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_USE_DEVICE_ADDR
12035 && !omp_is_allocatable_or_ptr (ovar
))
12036 || (omp_is_reference (ovar
)
12037 && omp_is_allocatable_or_ptr (ovar
))))
12038 var
= build_simple_mem_ref (var
);
12039 var
= fold_convert (TREE_TYPE (x
), var
);
12043 present
= omp_check_optional_argument (ovar
, true);
12046 tree null_label
= create_artificial_label (UNKNOWN_LOCATION
);
12047 tree notnull_label
= create_artificial_label (UNKNOWN_LOCATION
);
12048 tree opt_arg_label
= create_artificial_label (UNKNOWN_LOCATION
);
12049 tree new_x
= unshare_expr (x
);
12050 gimplify_expr (&present
, &ilist
, NULL
, is_gimple_val
,
12052 gcond
*cond
= gimple_build_cond_from_tree (present
,
12055 gimple_seq_add_stmt (&ilist
, cond
);
12056 gimple_seq_add_stmt (&ilist
, gimple_build_label (null_label
));
12057 gimplify_assign (new_x
, null_pointer_node
, &ilist
);
12058 gimple_seq_add_stmt (&ilist
, gimple_build_goto (opt_arg_label
));
12059 gimple_seq_add_stmt (&ilist
,
12060 gimple_build_label (notnull_label
));
12061 gimplify_assign (x
, var
, &ilist
);
12062 gimple_seq_add_stmt (&ilist
,
12063 gimple_build_label (opt_arg_label
));
12066 gimplify_assign (x
, var
, &ilist
);
12068 purpose
= size_int (map_idx
++);
12069 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
12070 gcc_checking_assert (tkind
12071 < (HOST_WIDE_INT_C (1U) << talign_shift
));
12072 gcc_checking_assert (tkind
12073 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
12074 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
12075 build_int_cstu (tkind_type
, tkind
));
12079 gcc_assert (map_idx
== map_cnt
);
12081 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
12082 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
12083 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
12084 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
12085 for (int i
= 1; i
<= 2; i
++)
12086 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
12088 gimple_seq initlist
= NULL
;
12089 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
12090 TREE_VEC_ELT (t
, i
)),
12091 &initlist
, true, NULL_TREE
);
12092 gimple_seq_add_seq (&ilist
, initlist
);
12094 tree clobber
= build_clobber (TREE_TYPE (TREE_VEC_ELT (t
, i
)));
12095 gimple_seq_add_stmt (&olist
,
12096 gimple_build_assign (TREE_VEC_ELT (t
, i
),
12100 tree clobber
= build_clobber (ctx
->record_type
);
12101 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
12105 /* Once all the expansions are done, sequence all the different
12106 fragments inside gimple_omp_body. */
12111 && ctx
->record_type
)
12113 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
12114 /* fixup_child_record_type might have changed receiver_decl's type. */
12115 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
12116 gimple_seq_add_stmt (&new_body
,
12117 gimple_build_assign (ctx
->receiver_decl
, t
));
12119 gimple_seq_add_seq (&new_body
, fplist
);
12121 if (offloaded
|| data_region
)
12123 tree prev
= NULL_TREE
;
12124 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
12125 switch (OMP_CLAUSE_CODE (c
))
12130 case OMP_CLAUSE_FIRSTPRIVATE
:
12131 if (is_gimple_omp_oacc (ctx
->stmt
))
12133 var
= OMP_CLAUSE_DECL (c
);
12134 if (omp_is_reference (var
)
12135 || is_gimple_reg_type (TREE_TYPE (var
)))
12137 tree new_var
= lookup_decl (var
, ctx
);
12139 type
= TREE_TYPE (var
);
12140 if (omp_is_reference (var
))
12141 type
= TREE_TYPE (type
);
12142 if ((INTEGRAL_TYPE_P (type
)
12143 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
12144 || TREE_CODE (type
) == POINTER_TYPE
)
12146 x
= build_receiver_ref (var
, false, ctx
);
12147 if (TREE_CODE (type
) != POINTER_TYPE
)
12148 x
= fold_convert (pointer_sized_int_node
, x
);
12149 x
= fold_convert (type
, x
);
12150 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
12152 if (omp_is_reference (var
))
12154 tree v
= create_tmp_var_raw (type
, get_name (var
));
12155 gimple_add_tmp_var (v
);
12156 TREE_ADDRESSABLE (v
) = 1;
12157 gimple_seq_add_stmt (&new_body
,
12158 gimple_build_assign (v
, x
));
12159 x
= build_fold_addr_expr (v
);
12161 gimple_seq_add_stmt (&new_body
,
12162 gimple_build_assign (new_var
, x
));
12166 x
= build_receiver_ref (var
, !omp_is_reference (var
), ctx
);
12167 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
12169 gimple_seq_add_stmt (&new_body
,
12170 gimple_build_assign (new_var
, x
));
12173 else if (is_variable_sized (var
))
12175 tree pvar
= DECL_VALUE_EXPR (var
);
12176 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
12177 pvar
= TREE_OPERAND (pvar
, 0);
12178 gcc_assert (DECL_P (pvar
));
12179 tree new_var
= lookup_decl (pvar
, ctx
);
12180 x
= build_receiver_ref (var
, false, ctx
);
12181 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
12182 gimple_seq_add_stmt (&new_body
,
12183 gimple_build_assign (new_var
, x
));
12186 case OMP_CLAUSE_PRIVATE
:
12187 if (is_gimple_omp_oacc (ctx
->stmt
))
12189 var
= OMP_CLAUSE_DECL (c
);
12190 if (omp_is_reference (var
))
12192 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
12193 tree new_var
= lookup_decl (var
, ctx
);
12194 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
12195 if (TREE_CONSTANT (x
))
12197 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
12199 gimple_add_tmp_var (x
);
12200 TREE_ADDRESSABLE (x
) = 1;
12201 x
= build_fold_addr_expr_loc (clause_loc
, x
);
12206 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
12207 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
12208 gimple_seq_add_stmt (&new_body
,
12209 gimple_build_assign (new_var
, x
));
12212 case OMP_CLAUSE_USE_DEVICE_PTR
:
12213 case OMP_CLAUSE_USE_DEVICE_ADDR
:
12214 case OMP_CLAUSE_IS_DEVICE_PTR
:
12216 gimple_seq assign_body
;
12217 bool is_array_data
;
12218 bool do_optional_check
;
12219 assign_body
= NULL
;
12220 do_optional_check
= false;
12221 var
= OMP_CLAUSE_DECL (c
);
12222 is_array_data
= lang_hooks
.decls
.omp_array_data (var
, true) != NULL
;
12224 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_IS_DEVICE_PTR
)
12225 x
= build_sender_ref (is_array_data
12226 ? (splay_tree_key
) &DECL_NAME (var
)
12227 : (splay_tree_key
) &DECL_UID (var
), ctx
);
12229 x
= build_receiver_ref (var
, false, ctx
);
12233 bool is_ref
= omp_is_reference (var
);
12234 do_optional_check
= true;
12235 /* First, we copy the descriptor data from the host; then
12236 we update its data to point to the target address. */
12237 new_var
= lookup_decl (var
, ctx
);
12238 new_var
= DECL_VALUE_EXPR (new_var
);
12243 var
= build_fold_indirect_ref (var
);
12244 gimplify_expr (&var
, &assign_body
, NULL
, is_gimple_val
,
12246 v
= create_tmp_var_raw (TREE_TYPE (var
), get_name (var
));
12247 gimple_add_tmp_var (v
);
12248 TREE_ADDRESSABLE (v
) = 1;
12249 gimple_seq_add_stmt (&assign_body
,
12250 gimple_build_assign (v
, var
));
12251 tree rhs
= build_fold_addr_expr (v
);
12252 gimple_seq_add_stmt (&assign_body
,
12253 gimple_build_assign (new_var
, rhs
));
12256 gimple_seq_add_stmt (&assign_body
,
12257 gimple_build_assign (new_var
, var
));
12259 tree v2
= lang_hooks
.decls
.omp_array_data (unshare_expr (v
), false);
12261 gimplify_expr (&x
, &assign_body
, NULL
, is_gimple_val
, fb_rvalue
);
12262 gimple_seq_add_stmt (&assign_body
,
12263 gimple_build_assign (v2
, x
));
12265 else if (is_variable_sized (var
))
12267 tree pvar
= DECL_VALUE_EXPR (var
);
12268 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
12269 pvar
= TREE_OPERAND (pvar
, 0);
12270 gcc_assert (DECL_P (pvar
));
12271 new_var
= lookup_decl (pvar
, ctx
);
12272 gimplify_expr (&x
, &assign_body
, NULL
, is_gimple_val
, fb_rvalue
);
12273 gimple_seq_add_stmt (&assign_body
,
12274 gimple_build_assign (new_var
, x
));
12276 else if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_ADDR
12277 && !omp_is_reference (var
)
12278 && !omp_is_allocatable_or_ptr (var
))
12279 || TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
12281 new_var
= lookup_decl (var
, ctx
);
12282 new_var
= DECL_VALUE_EXPR (new_var
);
12283 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
12284 new_var
= TREE_OPERAND (new_var
, 0);
12285 gcc_assert (DECL_P (new_var
));
12286 gimplify_expr (&x
, &assign_body
, NULL
, is_gimple_val
, fb_rvalue
);
12287 gimple_seq_add_stmt (&assign_body
,
12288 gimple_build_assign (new_var
, x
));
12292 tree type
= TREE_TYPE (var
);
12293 new_var
= lookup_decl (var
, ctx
);
12294 if (omp_is_reference (var
))
12296 type
= TREE_TYPE (type
);
12297 if (TREE_CODE (type
) != ARRAY_TYPE
12298 && (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_USE_DEVICE_ADDR
12299 || (omp_is_reference (var
)
12300 && omp_is_allocatable_or_ptr (var
))))
12302 tree v
= create_tmp_var_raw (type
, get_name (var
));
12303 gimple_add_tmp_var (v
);
12304 TREE_ADDRESSABLE (v
) = 1;
12305 x
= fold_convert (type
, x
);
12306 gimplify_expr (&x
, &assign_body
, NULL
, is_gimple_val
,
12308 gimple_seq_add_stmt (&assign_body
,
12309 gimple_build_assign (v
, x
));
12310 x
= build_fold_addr_expr (v
);
12311 do_optional_check
= true;
12314 new_var
= DECL_VALUE_EXPR (new_var
);
12315 x
= fold_convert (TREE_TYPE (new_var
), x
);
12316 gimplify_expr (&x
, &assign_body
, NULL
, is_gimple_val
, fb_rvalue
);
12317 gimple_seq_add_stmt (&assign_body
,
12318 gimple_build_assign (new_var
, x
));
12321 present
= (do_optional_check
12322 ? omp_check_optional_argument (OMP_CLAUSE_DECL (c
), true)
12326 tree null_label
= create_artificial_label (UNKNOWN_LOCATION
);
12327 tree notnull_label
= create_artificial_label (UNKNOWN_LOCATION
);
12328 tree opt_arg_label
= create_artificial_label (UNKNOWN_LOCATION
);
12329 glabel
*null_glabel
= gimple_build_label (null_label
);
12330 glabel
*notnull_glabel
= gimple_build_label (notnull_label
);
12331 ggoto
*opt_arg_ggoto
= gimple_build_goto (opt_arg_label
);
12332 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
12334 gimplify_expr (&present
, &new_body
, NULL
, is_gimple_val
,
12336 gcond
*cond
= gimple_build_cond_from_tree (present
,
12339 gimple_seq_add_stmt (&new_body
, cond
);
12340 gimple_seq_add_stmt (&new_body
, null_glabel
);
12341 gimplify_assign (new_var
, null_pointer_node
, &new_body
);
12342 gimple_seq_add_stmt (&new_body
, opt_arg_ggoto
);
12343 gimple_seq_add_stmt (&new_body
, notnull_glabel
);
12344 gimple_seq_add_seq (&new_body
, assign_body
);
12345 gimple_seq_add_stmt (&new_body
,
12346 gimple_build_label (opt_arg_label
));
12349 gimple_seq_add_seq (&new_body
, assign_body
);
12352 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
12353 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
12354 are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
12355 or references to VLAs. */
12356 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
12357 switch (OMP_CLAUSE_CODE (c
))
12362 case OMP_CLAUSE_MAP
:
12363 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
12364 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
12366 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
12367 poly_int64 offset
= 0;
12369 var
= OMP_CLAUSE_DECL (c
);
12371 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
12372 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
12374 && varpool_node::get_create (var
)->offloadable
)
12376 if (TREE_CODE (var
) == INDIRECT_REF
12377 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
12378 var
= TREE_OPERAND (var
, 0);
12379 if (TREE_CODE (var
) == COMPONENT_REF
)
12381 var
= get_addr_base_and_unit_offset (var
, &offset
);
12382 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
12384 else if (DECL_SIZE (var
)
12385 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
12387 tree var2
= DECL_VALUE_EXPR (var
);
12388 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
12389 var2
= TREE_OPERAND (var2
, 0);
12390 gcc_assert (DECL_P (var2
));
12393 tree new_var
= lookup_decl (var
, ctx
), x
;
12394 tree type
= TREE_TYPE (new_var
);
12396 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
12397 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
12400 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
12402 new_var
= build2 (MEM_REF
, type
,
12403 build_fold_addr_expr (new_var
),
12404 build_int_cst (build_pointer_type (type
),
12407 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
12409 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
12410 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
12411 new_var
= build2 (MEM_REF
, type
,
12412 build_fold_addr_expr (new_var
),
12413 build_int_cst (build_pointer_type (type
),
12417 is_ref
= omp_is_reference (var
);
12418 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
12420 bool ref_to_array
= false;
12423 type
= TREE_TYPE (type
);
12424 if (TREE_CODE (type
) == ARRAY_TYPE
)
12426 type
= build_pointer_type (type
);
12427 ref_to_array
= true;
12430 else if (TREE_CODE (type
) == ARRAY_TYPE
)
12432 tree decl2
= DECL_VALUE_EXPR (new_var
);
12433 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
12434 decl2
= TREE_OPERAND (decl2
, 0);
12435 gcc_assert (DECL_P (decl2
));
12437 type
= TREE_TYPE (new_var
);
12439 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
12440 x
= fold_convert_loc (clause_loc
, type
, x
);
12441 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
12443 tree bias
= OMP_CLAUSE_SIZE (c
);
12445 bias
= lookup_decl (bias
, ctx
);
12446 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
12447 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
12449 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
12450 TREE_TYPE (x
), x
, bias
);
12453 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
12454 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
12455 if (is_ref
&& !ref_to_array
)
12457 tree t
= create_tmp_var_raw (type
, get_name (var
));
12458 gimple_add_tmp_var (t
);
12459 TREE_ADDRESSABLE (t
) = 1;
12460 gimple_seq_add_stmt (&new_body
,
12461 gimple_build_assign (t
, x
));
12462 x
= build_fold_addr_expr_loc (clause_loc
, t
);
12464 gimple_seq_add_stmt (&new_body
,
12465 gimple_build_assign (new_var
, x
));
12468 else if (OMP_CLAUSE_CHAIN (c
)
12469 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
12471 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
12472 == GOMP_MAP_FIRSTPRIVATE_POINTER
12473 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
12474 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
12477 case OMP_CLAUSE_PRIVATE
:
12478 var
= OMP_CLAUSE_DECL (c
);
12479 if (is_variable_sized (var
))
12481 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
12482 tree new_var
= lookup_decl (var
, ctx
);
12483 tree pvar
= DECL_VALUE_EXPR (var
);
12484 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
12485 pvar
= TREE_OPERAND (pvar
, 0);
12486 gcc_assert (DECL_P (pvar
));
12487 tree new_pvar
= lookup_decl (pvar
, ctx
);
12488 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
12489 tree al
= size_int (DECL_ALIGN (var
));
12490 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
12491 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
12492 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
12493 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
12494 gimple_seq_add_stmt (&new_body
,
12495 gimple_build_assign (new_pvar
, x
));
12497 else if (omp_is_reference (var
) && !is_gimple_omp_oacc (ctx
->stmt
))
12499 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
12500 tree new_var
= lookup_decl (var
, ctx
);
12501 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
12502 if (TREE_CONSTANT (x
))
12507 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
12508 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
12509 tree al
= size_int (TYPE_ALIGN (rtype
));
12510 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
12513 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
12514 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
12515 gimple_seq_add_stmt (&new_body
,
12516 gimple_build_assign (new_var
, x
));
12521 gimple_seq fork_seq
= NULL
;
12522 gimple_seq join_seq
= NULL
;
12524 if (is_oacc_parallel_or_serial (ctx
))
12526 /* If there are reductions on the offloaded region itself, treat
12527 them as a dummy GANG loop. */
12528 tree level
= build_int_cst (integer_type_node
, GOMP_DIM_GANG
);
12530 lower_oacc_reductions (gimple_location (ctx
->stmt
), clauses
, level
,
12531 false, NULL
, NULL
, &fork_seq
, &join_seq
, ctx
);
12534 gimple_seq_add_seq (&new_body
, fork_seq
);
12535 gimple_seq_add_seq (&new_body
, tgt_body
);
12536 gimple_seq_add_seq (&new_body
, join_seq
);
12539 new_body
= maybe_catch_exception (new_body
);
12541 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
12542 gimple_omp_set_body (stmt
, new_body
);
12545 bind
= gimple_build_bind (NULL
, NULL
,
12546 tgt_bind
? gimple_bind_block (tgt_bind
)
12548 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
12549 gimple_bind_add_seq (bind
, ilist
);
12550 gimple_bind_add_stmt (bind
, stmt
);
12551 gimple_bind_add_seq (bind
, olist
);
12553 pop_gimplify_context (NULL
);
12557 gimple_bind_add_seq (dep_bind
, dep_ilist
);
12558 gimple_bind_add_stmt (dep_bind
, bind
);
12559 gimple_bind_add_seq (dep_bind
, dep_olist
);
12560 pop_gimplify_context (dep_bind
);
12564 /* Expand code for an OpenMP teams directive. */
12567 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
12569 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
12570 push_gimplify_context ();
12572 tree block
= make_node (BLOCK
);
12573 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
12574 gsi_replace (gsi_p
, bind
, true);
12575 gimple_seq bind_body
= NULL
;
12576 gimple_seq dlist
= NULL
;
12577 gimple_seq olist
= NULL
;
12579 tree num_teams
= omp_find_clause (gimple_omp_teams_clauses (teams_stmt
),
12580 OMP_CLAUSE_NUM_TEAMS
);
12581 if (num_teams
== NULL_TREE
)
12582 num_teams
= build_int_cst (unsigned_type_node
, 0);
12585 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
12586 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
12587 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
12589 tree thread_limit
= omp_find_clause (gimple_omp_teams_clauses (teams_stmt
),
12590 OMP_CLAUSE_THREAD_LIMIT
);
12591 if (thread_limit
== NULL_TREE
)
12592 thread_limit
= build_int_cst (unsigned_type_node
, 0);
12595 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
12596 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
12597 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
12601 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
12602 &bind_body
, &dlist
, ctx
, NULL
);
12603 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
12604 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
,
12606 if (!gimple_omp_teams_grid_phony (teams_stmt
))
12608 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
12609 location_t loc
= gimple_location (teams_stmt
);
12610 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
12611 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
12612 gimple_set_location (call
, loc
);
12613 gimple_seq_add_stmt (&bind_body
, call
);
12616 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
12617 gimple_omp_set_body (teams_stmt
, NULL
);
12618 gimple_seq_add_seq (&bind_body
, olist
);
12619 gimple_seq_add_seq (&bind_body
, dlist
);
12620 if (!gimple_omp_teams_grid_phony (teams_stmt
))
12621 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
12622 gimple_bind_set_body (bind
, bind_body
);
12624 pop_gimplify_context (bind
);
12626 gimple_bind_append_vars (bind
, ctx
->block_vars
);
12627 BLOCK_VARS (block
) = ctx
->block_vars
;
12628 if (BLOCK_VARS (block
))
12629 TREE_USED (block
) = 1;
12632 /* Expand code within an artificial GIMPLE_OMP_GRID_BODY OMP construct. */
12635 lower_omp_grid_body (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
12637 gimple
*stmt
= gsi_stmt (*gsi_p
);
12638 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
12639 gimple_seq_add_stmt (gimple_omp_body_ptr (stmt
),
12640 gimple_build_omp_return (false));
12644 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
12645 regimplified. If DATA is non-NULL, lower_omp_1 is outside
12646 of OMP context, but with task_shared_vars set. */
12649 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
12654 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
12655 if (VAR_P (t
) && data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
12658 if (task_shared_vars
12660 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
12663 /* If a global variable has been privatized, TREE_CONSTANT on
12664 ADDR_EXPR might be wrong. */
12665 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
12666 recompute_tree_invariant_for_addr_expr (t
);
12668 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
12672 /* Data to be communicated between lower_omp_regimplify_operands and
12673 lower_omp_regimplify_operands_p. */
12675 struct lower_omp_regimplify_operands_data
12681 /* Helper function for lower_omp_regimplify_operands. Find
12682 omp_member_access_dummy_var vars and adjust temporarily their
12683 DECL_VALUE_EXPRs if needed. */
12686 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
12689 tree t
= omp_member_access_dummy_var (*tp
);
12692 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
12693 lower_omp_regimplify_operands_data
*ldata
12694 = (lower_omp_regimplify_operands_data
*) wi
->info
;
12695 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
12698 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
12699 ldata
->decls
->safe_push (*tp
);
12700 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
12701 SET_DECL_VALUE_EXPR (*tp
, v
);
12704 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
12708 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
12709 of omp_member_access_dummy_var vars during regimplification. */
12712 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
12713 gimple_stmt_iterator
*gsi_p
)
12715 auto_vec
<tree
, 10> decls
;
12718 struct walk_stmt_info wi
;
12719 memset (&wi
, '\0', sizeof (wi
));
12720 struct lower_omp_regimplify_operands_data data
;
12722 data
.decls
= &decls
;
12724 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
12726 gimple_regimplify_operands (stmt
, gsi_p
);
12727 while (!decls
.is_empty ())
12729 tree t
= decls
.pop ();
12730 tree v
= decls
.pop ();
12731 SET_DECL_VALUE_EXPR (t
, v
);
12736 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
12738 gimple
*stmt
= gsi_stmt (*gsi_p
);
12739 struct walk_stmt_info wi
;
12742 if (gimple_has_location (stmt
))
12743 input_location
= gimple_location (stmt
);
12745 if (task_shared_vars
)
12746 memset (&wi
, '\0', sizeof (wi
));
12748 /* If we have issued syntax errors, avoid doing any heavy lifting.
12749 Just replace the OMP directives with a NOP to avoid
12750 confusing RTL expansion. */
12751 if (seen_error () && is_gimple_omp (stmt
))
12753 gsi_replace (gsi_p
, gimple_build_nop (), true);
12757 switch (gimple_code (stmt
))
12761 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
12762 if ((ctx
|| task_shared_vars
)
12763 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
12764 lower_omp_regimplify_p
,
12765 ctx
? NULL
: &wi
, NULL
)
12766 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
12767 lower_omp_regimplify_p
,
12768 ctx
? NULL
: &wi
, NULL
)))
12769 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
12773 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
12775 case GIMPLE_EH_FILTER
:
12776 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
12779 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
12780 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
12782 case GIMPLE_TRANSACTION
:
12783 lower_omp (gimple_transaction_body_ptr (as_a
<gtransaction
*> (stmt
)),
12787 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
12788 maybe_remove_omp_member_access_dummy_vars (as_a
<gbind
*> (stmt
));
12790 case GIMPLE_OMP_PARALLEL
:
12791 case GIMPLE_OMP_TASK
:
12792 ctx
= maybe_lookup_ctx (stmt
);
12794 if (ctx
->cancellable
)
12795 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
12796 lower_omp_taskreg (gsi_p
, ctx
);
12798 case GIMPLE_OMP_FOR
:
12799 ctx
= maybe_lookup_ctx (stmt
);
12801 if (ctx
->cancellable
)
12802 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
12803 lower_omp_for (gsi_p
, ctx
);
12805 case GIMPLE_OMP_SECTIONS
:
12806 ctx
= maybe_lookup_ctx (stmt
);
12808 if (ctx
->cancellable
)
12809 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
12810 lower_omp_sections (gsi_p
, ctx
);
12812 case GIMPLE_OMP_SINGLE
:
12813 ctx
= maybe_lookup_ctx (stmt
);
12815 lower_omp_single (gsi_p
, ctx
);
12817 case GIMPLE_OMP_MASTER
:
12818 ctx
= maybe_lookup_ctx (stmt
);
12820 lower_omp_master (gsi_p
, ctx
);
12822 case GIMPLE_OMP_TASKGROUP
:
12823 ctx
= maybe_lookup_ctx (stmt
);
12825 lower_omp_taskgroup (gsi_p
, ctx
);
12827 case GIMPLE_OMP_ORDERED
:
12828 ctx
= maybe_lookup_ctx (stmt
);
12830 lower_omp_ordered (gsi_p
, ctx
);
12832 case GIMPLE_OMP_SCAN
:
12833 ctx
= maybe_lookup_ctx (stmt
);
12835 lower_omp_scan (gsi_p
, ctx
);
12837 case GIMPLE_OMP_CRITICAL
:
12838 ctx
= maybe_lookup_ctx (stmt
);
12840 lower_omp_critical (gsi_p
, ctx
);
12842 case GIMPLE_OMP_ATOMIC_LOAD
:
12843 if ((ctx
|| task_shared_vars
)
12844 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
12845 as_a
<gomp_atomic_load
*> (stmt
)),
12846 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
12847 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
12849 case GIMPLE_OMP_TARGET
:
12850 ctx
= maybe_lookup_ctx (stmt
);
12852 lower_omp_target (gsi_p
, ctx
);
12854 case GIMPLE_OMP_TEAMS
:
12855 ctx
= maybe_lookup_ctx (stmt
);
12857 if (gimple_omp_teams_host (as_a
<gomp_teams
*> (stmt
)))
12858 lower_omp_taskreg (gsi_p
, ctx
);
12860 lower_omp_teams (gsi_p
, ctx
);
12862 case GIMPLE_OMP_GRID_BODY
:
12863 ctx
= maybe_lookup_ctx (stmt
);
12865 lower_omp_grid_body (gsi_p
, ctx
);
12869 call_stmt
= as_a
<gcall
*> (stmt
);
12870 fndecl
= gimple_call_fndecl (call_stmt
);
12872 && fndecl_built_in_p (fndecl
, BUILT_IN_NORMAL
))
12873 switch (DECL_FUNCTION_CODE (fndecl
))
12875 case BUILT_IN_GOMP_BARRIER
:
12879 case BUILT_IN_GOMP_CANCEL
:
12880 case BUILT_IN_GOMP_CANCELLATION_POINT
:
12883 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
12884 cctx
= cctx
->outer
;
12885 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
12886 if (!cctx
->cancellable
)
12888 if (DECL_FUNCTION_CODE (fndecl
)
12889 == BUILT_IN_GOMP_CANCELLATION_POINT
)
12891 stmt
= gimple_build_nop ();
12892 gsi_replace (gsi_p
, stmt
, false);
12896 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
12898 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
12899 gimple_call_set_fndecl (call_stmt
, fndecl
);
12900 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
12903 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
12904 gimple_call_set_lhs (call_stmt
, lhs
);
12905 tree fallthru_label
;
12906 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
12908 g
= gimple_build_label (fallthru_label
);
12909 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
12910 g
= gimple_build_cond (NE_EXPR
, lhs
,
12911 fold_convert (TREE_TYPE (lhs
),
12912 boolean_false_node
),
12913 cctx
->cancel_label
, fallthru_label
);
12914 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
12921 case GIMPLE_ASSIGN
:
12922 for (omp_context
*up
= ctx
; up
; up
= up
->outer
)
12924 if (gimple_code (up
->stmt
) == GIMPLE_OMP_ORDERED
12925 || gimple_code (up
->stmt
) == GIMPLE_OMP_CRITICAL
12926 || gimple_code (up
->stmt
) == GIMPLE_OMP_TASKGROUP
12927 || gimple_code (up
->stmt
) == GIMPLE_OMP_SECTION
12928 || gimple_code (up
->stmt
) == GIMPLE_OMP_SCAN
12929 || (gimple_code (up
->stmt
) == GIMPLE_OMP_TARGET
12930 && (gimple_omp_target_kind (up
->stmt
)
12931 == GF_OMP_TARGET_KIND_DATA
)))
12933 else if (!up
->lastprivate_conditional_map
)
12935 tree lhs
= get_base_address (gimple_assign_lhs (stmt
));
12936 if (TREE_CODE (lhs
) == MEM_REF
12937 && DECL_P (TREE_OPERAND (lhs
, 0))
12938 && TREE_CODE (TREE_TYPE (TREE_OPERAND (lhs
,
12939 0))) == REFERENCE_TYPE
)
12940 lhs
= TREE_OPERAND (lhs
, 0);
12942 if (tree
*v
= up
->lastprivate_conditional_map
->get (lhs
))
12945 if (up
->combined_into_simd_safelen1
)
12948 if (gimple_code (up
->stmt
) == GIMPLE_OMP_SCAN
)
12951 if (gimple_code (up
->stmt
) == GIMPLE_OMP_FOR
)
12952 clauses
= gimple_omp_for_clauses (up
->stmt
);
12954 clauses
= gimple_omp_sections_clauses (up
->stmt
);
12955 tree c
= omp_find_clause (clauses
, OMP_CLAUSE__CONDTEMP_
);
12956 if (!OMP_CLAUSE__CONDTEMP__ITER (c
))
12957 c
= omp_find_clause (OMP_CLAUSE_CHAIN (c
),
12958 OMP_CLAUSE__CONDTEMP_
);
12959 gcc_assert (OMP_CLAUSE__CONDTEMP__ITER (c
));
12960 gimple
*g
= gimple_build_assign (*v
, OMP_CLAUSE_DECL (c
));
12961 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
12968 if ((ctx
|| task_shared_vars
)
12969 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
12972 /* Just remove clobbers, this should happen only if we have
12973 "privatized" local addressable variables in SIMD regions,
12974 the clobber isn't needed in that case and gimplifying address
12975 of the ARRAY_REF into a pointer and creating MEM_REF based
12976 clobber would create worse code than we get with the clobber
12978 if (gimple_clobber_p (stmt
))
12980 gsi_replace (gsi_p
, gimple_build_nop (), true);
12983 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
12990 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
12992 location_t saved_location
= input_location
;
12993 gimple_stmt_iterator gsi
;
12994 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
12995 lower_omp_1 (&gsi
, ctx
);
12996 /* During gimplification, we haven't folded statments inside offloading
12997 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
12998 if (target_nesting_level
|| taskreg_nesting_level
)
12999 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
13001 input_location
= saved_location
;
13004 /* Main entry point. */
13006 static unsigned int
13007 execute_lower_omp (void)
13013 /* This pass always runs, to provide PROP_gimple_lomp.
13014 But often, there is nothing to do. */
13015 if (flag_openacc
== 0 && flag_openmp
== 0
13016 && flag_openmp_simd
== 0)
13019 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
13020 delete_omp_context
);
13022 body
= gimple_body (current_function_decl
);
13024 if (hsa_gen_requested_p ())
13025 omp_grid_gridify_all_targets (&body
);
13027 scan_omp (&body
, NULL
);
13028 gcc_assert (taskreg_nesting_level
== 0);
13029 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
13030 finish_taskreg_scan (ctx
);
13031 taskreg_contexts
.release ();
13033 if (all_contexts
->root
)
13035 if (task_shared_vars
)
13036 push_gimplify_context ();
13037 lower_omp (&body
, NULL
);
13038 if (task_shared_vars
)
13039 pop_gimplify_context (NULL
);
13044 splay_tree_delete (all_contexts
);
13045 all_contexts
= NULL
;
13047 BITMAP_FREE (task_shared_vars
);
13048 BITMAP_FREE (global_nonaddressable_vars
);
13050 /* If current function is a method, remove artificial dummy VAR_DECL created
13051 for non-static data member privatization, they aren't needed for
13052 debuginfo nor anything else, have been already replaced everywhere in the
13053 IL and cause problems with LTO. */
13054 if (DECL_ARGUMENTS (current_function_decl
)
13055 && DECL_ARTIFICIAL (DECL_ARGUMENTS (current_function_decl
))
13056 && (TREE_CODE (TREE_TYPE (DECL_ARGUMENTS (current_function_decl
)))
13058 remove_member_access_dummy_vars (DECL_INITIAL (current_function_decl
));
13064 const pass_data pass_data_lower_omp
=
13066 GIMPLE_PASS
, /* type */
13067 "omplower", /* name */
13068 OPTGROUP_OMP
, /* optinfo_flags */
13069 TV_NONE
, /* tv_id */
13070 PROP_gimple_any
, /* properties_required */
13071 PROP_gimple_lomp
| PROP_gimple_lomp_dev
, /* properties_provided */
13072 0, /* properties_destroyed */
13073 0, /* todo_flags_start */
13074 0, /* todo_flags_finish */
13077 class pass_lower_omp
: public gimple_opt_pass
13080 pass_lower_omp (gcc::context
*ctxt
)
13081 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
13084 /* opt_pass methods: */
13085 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
13087 }; // class pass_lower_omp
13089 } // anon namespace
13092 make_pass_lower_omp (gcc::context
*ctxt
)
13094 return new pass_lower_omp (ctxt
);
13097 /* The following is a utility to diagnose structured block violations.
13098 It is not part of the "omplower" pass, as that's invoked too late. It
13099 should be invoked by the respective front ends after gimplification. */
13101 static splay_tree all_labels
;
13103 /* Check for mismatched contexts and generate an error if needed. Return
13104 true if an error is detected. */
13107 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
13108 gimple
*branch_ctx
, gimple
*label_ctx
)
13110 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
13111 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
13113 if (label_ctx
== branch_ctx
)
13116 const char* kind
= NULL
;
13120 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
13121 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
13123 gcc_checking_assert (kind
== NULL
);
13129 gcc_checking_assert (flag_openmp
|| flag_openmp_simd
);
13133 /* Previously we kept track of the label's entire context in diagnose_sb_[12]
13134 so we could traverse it and issue a correct "exit" or "enter" error
13135 message upon a structured block violation.
13137 We built the context by building a list with tree_cons'ing, but there is
13138 no easy counterpart in gimple tuples. It seems like far too much work
13139 for issuing exit/enter error messages. If someone really misses the
13140 distinct error message... patches welcome. */
13143 /* Try to avoid confusing the user by producing and error message
13144 with correct "exit" or "enter" verbiage. We prefer "exit"
13145 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
13146 if (branch_ctx
== NULL
)
13152 if (TREE_VALUE (label_ctx
) == branch_ctx
)
13157 label_ctx
= TREE_CHAIN (label_ctx
);
13162 error ("invalid exit from %s structured block", kind
);
13164 error ("invalid entry to %s structured block", kind
);
13167 /* If it's obvious we have an invalid entry, be specific about the error. */
13168 if (branch_ctx
== NULL
)
13169 error ("invalid entry to %s structured block", kind
);
13172 /* Otherwise, be vague and lazy, but efficient. */
13173 error ("invalid branch to/from %s structured block", kind
);
13176 gsi_replace (gsi_p
, gimple_build_nop (), false);
13180 /* Pass 1: Create a minimal tree of structured blocks, and record
13181 where each label is found. */
13184 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
13185 struct walk_stmt_info
*wi
)
13187 gimple
*context
= (gimple
*) wi
->info
;
13188 gimple
*inner_context
;
13189 gimple
*stmt
= gsi_stmt (*gsi_p
);
13191 *handled_ops_p
= true;
13193 switch (gimple_code (stmt
))
13197 case GIMPLE_OMP_PARALLEL
:
13198 case GIMPLE_OMP_TASK
:
13199 case GIMPLE_OMP_SECTIONS
:
13200 case GIMPLE_OMP_SINGLE
:
13201 case GIMPLE_OMP_SECTION
:
13202 case GIMPLE_OMP_MASTER
:
13203 case GIMPLE_OMP_ORDERED
:
13204 case GIMPLE_OMP_SCAN
:
13205 case GIMPLE_OMP_CRITICAL
:
13206 case GIMPLE_OMP_TARGET
:
13207 case GIMPLE_OMP_TEAMS
:
13208 case GIMPLE_OMP_TASKGROUP
:
13209 /* The minimal context here is just the current OMP construct. */
13210 inner_context
= stmt
;
13211 wi
->info
= inner_context
;
13212 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
13213 wi
->info
= context
;
13216 case GIMPLE_OMP_FOR
:
13217 inner_context
= stmt
;
13218 wi
->info
= inner_context
;
13219 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
13221 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
13222 diagnose_sb_1
, NULL
, wi
);
13223 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
13224 wi
->info
= context
;
13228 splay_tree_insert (all_labels
,
13229 (splay_tree_key
) gimple_label_label (
13230 as_a
<glabel
*> (stmt
)),
13231 (splay_tree_value
) context
);
13241 /* Pass 2: Check each branch and see if its context differs from that of
13242 the destination label's context. */
13245 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
13246 struct walk_stmt_info
*wi
)
13248 gimple
*context
= (gimple
*) wi
->info
;
13250 gimple
*stmt
= gsi_stmt (*gsi_p
);
13252 *handled_ops_p
= true;
13254 switch (gimple_code (stmt
))
13258 case GIMPLE_OMP_PARALLEL
:
13259 case GIMPLE_OMP_TASK
:
13260 case GIMPLE_OMP_SECTIONS
:
13261 case GIMPLE_OMP_SINGLE
:
13262 case GIMPLE_OMP_SECTION
:
13263 case GIMPLE_OMP_MASTER
:
13264 case GIMPLE_OMP_ORDERED
:
13265 case GIMPLE_OMP_SCAN
:
13266 case GIMPLE_OMP_CRITICAL
:
13267 case GIMPLE_OMP_TARGET
:
13268 case GIMPLE_OMP_TEAMS
:
13269 case GIMPLE_OMP_TASKGROUP
:
13271 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
13272 wi
->info
= context
;
13275 case GIMPLE_OMP_FOR
:
13277 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
13279 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
13280 diagnose_sb_2
, NULL
, wi
);
13281 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
13282 wi
->info
= context
;
13287 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
13288 tree lab
= gimple_cond_true_label (cond_stmt
);
13291 n
= splay_tree_lookup (all_labels
,
13292 (splay_tree_key
) lab
);
13293 diagnose_sb_0 (gsi_p
, context
,
13294 n
? (gimple
*) n
->value
: NULL
);
13296 lab
= gimple_cond_false_label (cond_stmt
);
13299 n
= splay_tree_lookup (all_labels
,
13300 (splay_tree_key
) lab
);
13301 diagnose_sb_0 (gsi_p
, context
,
13302 n
? (gimple
*) n
->value
: NULL
);
13309 tree lab
= gimple_goto_dest (stmt
);
13310 if (TREE_CODE (lab
) != LABEL_DECL
)
13313 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
13314 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
13318 case GIMPLE_SWITCH
:
13320 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
13322 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
13324 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
13325 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
13326 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
13332 case GIMPLE_RETURN
:
13333 diagnose_sb_0 (gsi_p
, context
, NULL
);
13343 static unsigned int
13344 diagnose_omp_structured_block_errors (void)
13346 struct walk_stmt_info wi
;
13347 gimple_seq body
= gimple_body (current_function_decl
);
13349 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
13351 memset (&wi
, 0, sizeof (wi
));
13352 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
13354 memset (&wi
, 0, sizeof (wi
));
13355 wi
.want_locations
= true;
13356 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
13358 gimple_set_body (current_function_decl
, body
);
13360 splay_tree_delete (all_labels
);
13368 const pass_data pass_data_diagnose_omp_blocks
=
13370 GIMPLE_PASS
, /* type */
13371 "*diagnose_omp_blocks", /* name */
13372 OPTGROUP_OMP
, /* optinfo_flags */
13373 TV_NONE
, /* tv_id */
13374 PROP_gimple_any
, /* properties_required */
13375 0, /* properties_provided */
13376 0, /* properties_destroyed */
13377 0, /* todo_flags_start */
13378 0, /* todo_flags_finish */
13381 class pass_diagnose_omp_blocks
: public gimple_opt_pass
13384 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
13385 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
13388 /* opt_pass methods: */
13389 virtual bool gate (function
*)
13391 return flag_openacc
|| flag_openmp
|| flag_openmp_simd
;
13393 virtual unsigned int execute (function
*)
13395 return diagnose_omp_structured_block_errors ();
13398 }; // class pass_diagnose_omp_blocks
13400 } // anon namespace
13403 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
13405 return new pass_diagnose_omp_blocks (ctxt
);
13409 #include "gt-omp-low.h"