1 /* Bits of OpenMP and OpenACC handling that is specific to device offloading
2 and a lowering pass for OpenACC device directives.
4 Copyright (C) 2005-2019 Free Software Foundation, Inc.
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
29 #include "tree-pass.h"
32 #include "pretty-print.h"
33 #include "diagnostic-core.h"
34 #include "fold-const.h"
35 #include "internal-fn.h"
36 #include "langhooks.h"
38 #include "gimple-iterator.h"
39 #include "gimplify-me.h"
40 #include "gimple-walk.h"
42 #include "tree-into-ssa.h"
43 #include "tree-nested.h"
44 #include "stor-layout.h"
45 #include "common/common-target.h"
46 #include "omp-general.h"
47 #include "omp-offload.h"
48 #include "lto-section-names.h"
49 #include "gomp-constants.h"
50 #include "gimple-pretty-print.h"
52 #include "stringpool.h"
56 /* Describe the OpenACC looping structure of a function. The entire
57 function is held in a 'NULL' loop. */
61 oacc_loop
*parent
; /* Containing loop. */
63 oacc_loop
*child
; /* First inner loop. */
65 oacc_loop
*sibling
; /* Next loop within same parent. */
67 location_t loc
; /* Location of the loop start. */
69 gcall
*marker
; /* Initial head marker. */
71 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
72 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
74 tree routine
; /* Pseudo-loop enclosing a routine. */
76 unsigned mask
; /* Partitioning mask. */
77 unsigned e_mask
; /* Partitioning of element loops (when tiling). */
78 unsigned inner
; /* Partitioning of inner loops. */
79 unsigned flags
; /* Partitioning flags. */
80 vec
<gcall
*> ifns
; /* Contained loop abstraction functions. */
81 tree chunk_size
; /* Chunk size. */
82 gcall
*head_end
; /* Final marker of head sequence. */
85 /* Holds offload tables with decls. */
86 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
88 /* Return level at which oacc routine may spawn a partitioned loop, or
89 -1 if it is not a routine (i.e. is an offload fn). */
92 oacc_fn_attrib_level (tree attr
)
94 tree pos
= TREE_VALUE (attr
);
96 if (!TREE_PURPOSE (pos
))
100 for (ix
= 0; ix
!= GOMP_DIM_MAX
;
101 ix
++, pos
= TREE_CHAIN (pos
))
102 if (!integer_zerop (TREE_PURPOSE (pos
)))
108 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
109 adds their addresses and sizes to constructor-vector V_CTOR. */
112 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
113 vec
<constructor_elt
, va_gc
> *v_ctor
)
115 unsigned len
= vec_safe_length (v_decls
);
116 for (unsigned i
= 0; i
< len
; i
++)
118 tree it
= (*v_decls
)[i
];
119 bool is_var
= VAR_P (it
);
122 #ifdef ACCEL_COMPILER
123 && DECL_HAS_VALUE_EXPR_P (it
)
125 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it
));
127 tree size
= NULL_TREE
;
129 size
= fold_convert (const_ptr_type_node
, DECL_SIZE_UNIT (it
));
133 addr
= build_fold_addr_expr (it
);
136 #ifdef ACCEL_COMPILER
137 /* For "omp declare target link" vars add address of the pointer to
138 the target table, instead of address of the var. */
139 tree value_expr
= DECL_VALUE_EXPR (it
);
140 tree link_ptr_decl
= TREE_OPERAND (value_expr
, 0);
141 varpool_node::finalize_decl (link_ptr_decl
);
142 addr
= build_fold_addr_expr (link_ptr_decl
);
144 addr
= build_fold_addr_expr (it
);
147 /* Most significant bit of the size marks "omp declare target link"
148 vars in host and target tables. */
149 unsigned HOST_WIDE_INT isize
= tree_to_uhwi (size
);
150 isize
|= 1ULL << (int_size_in_bytes (const_ptr_type_node
)
151 * BITS_PER_UNIT
- 1);
152 size
= wide_int_to_tree (const_ptr_type_node
, isize
);
155 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, addr
);
157 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, size
);
161 /* Create new symbols containing (address, size) pairs for global variables,
162 marked with "omp declare target" attribute, as well as addresses for the
163 functions, which are outlined offloading regions. */
165 omp_finish_file (void)
167 unsigned num_funcs
= vec_safe_length (offload_funcs
);
168 unsigned num_vars
= vec_safe_length (offload_vars
);
170 if (num_funcs
== 0 && num_vars
== 0)
173 if (targetm_common
.have_named_sections
)
175 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
176 vec_alloc (v_f
, num_funcs
);
177 vec_alloc (v_v
, num_vars
* 2);
179 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
180 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
182 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
184 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
186 SET_TYPE_ALIGN (vars_decl_type
, TYPE_ALIGN (pointer_sized_int_node
));
187 SET_TYPE_ALIGN (funcs_decl_type
, TYPE_ALIGN (pointer_sized_int_node
));
188 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
189 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
190 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
191 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
192 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
193 get_identifier (".offload_func_table"),
195 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
196 get_identifier (".offload_var_table"),
198 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
199 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
200 otherwise a joint table in a binary will contain padding between
201 tables from multiple object files. */
202 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
203 SET_DECL_ALIGN (funcs_decl
, TYPE_ALIGN (funcs_decl_type
));
204 SET_DECL_ALIGN (vars_decl
, TYPE_ALIGN (vars_decl_type
));
205 DECL_INITIAL (funcs_decl
) = ctor_f
;
206 DECL_INITIAL (vars_decl
) = ctor_v
;
207 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
208 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
210 varpool_node::finalize_decl (vars_decl
);
211 varpool_node::finalize_decl (funcs_decl
);
215 for (unsigned i
= 0; i
< num_funcs
; i
++)
217 tree it
= (*offload_funcs
)[i
];
218 targetm
.record_offload_symbol (it
);
220 for (unsigned i
= 0; i
< num_vars
; i
++)
222 tree it
= (*offload_vars
)[i
];
223 targetm
.record_offload_symbol (it
);
228 /* Call dim_pos (POS == true) or dim_size (POS == false) builtins for
229 axis DIM. Return a tmp var holding the result. */
232 oacc_dim_call (bool pos
, int dim
, gimple_seq
*seq
)
234 tree arg
= build_int_cst (unsigned_type_node
, dim
);
235 tree size
= create_tmp_var (integer_type_node
);
236 enum internal_fn fn
= pos
? IFN_GOACC_DIM_POS
: IFN_GOACC_DIM_SIZE
;
237 gimple
*call
= gimple_build_call_internal (fn
, 1, arg
);
239 gimple_call_set_lhs (call
, size
);
240 gimple_seq_add_stmt (seq
, call
);
245 /* Find the number of threads (POS = false), or thread number (POS =
246 true) for an OpenACC region partitioned as MASK. Setup code
247 required for the calculation is added to SEQ. */
250 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
252 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
255 /* Start at gang level, and examine relevant dimension indices. */
256 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
257 if (GOMP_DIM_MASK (ix
) & mask
)
261 /* We had an outer index, so scale that by the size of
263 tree n
= oacc_dim_call (false, ix
, seq
);
264 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
268 /* Determine index in this dimension. */
269 tree id
= oacc_dim_call (true, ix
, seq
);
271 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
277 if (res
== NULL_TREE
)
278 res
= integer_zero_node
;
283 /* Transform IFN_GOACC_LOOP calls to actual code. See
284 expand_oacc_for for where these are generated. At the vector
285 level, we stride loops, such that each member of a warp will
286 operate on adjacent iterations. At the worker and gang level,
287 each gang/warp executes a set of contiguous iterations. Chunking
288 can override this such that each iteration engine executes a
289 contiguous chunk, and then moves on to stride to the next chunk. */
292 oacc_xform_loop (gcall
*call
)
294 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
295 enum ifn_goacc_loop_kind code
296 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
297 tree dir
= gimple_call_arg (call
, 1);
298 tree range
= gimple_call_arg (call
, 2);
299 tree step
= gimple_call_arg (call
, 3);
300 tree chunk_size
= NULL_TREE
;
301 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
302 tree lhs
= gimple_call_lhs (call
);
303 tree type
= NULL_TREE
;
304 tree diff_type
= TREE_TYPE (range
);
306 gimple_seq seq
= NULL
;
307 bool chunking
= false, striding
= true;
308 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
309 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
311 /* Skip lowering if return value of IFN_GOACC_LOOP call is not used. */
314 gsi_replace_with_seq (&gsi
, seq
, true);
318 type
= TREE_TYPE (lhs
);
320 #ifdef ACCEL_COMPILER
321 chunk_size
= gimple_call_arg (call
, 4);
322 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
323 || integer_zerop (chunk_size
)) /* Default (also static). */
325 /* If we're at the gang level, we want each to execute a
326 contiguous run of iterations. Otherwise we want each element
328 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
333 /* Chunk of size 1 is striding. */
334 striding
= integer_onep (chunk_size
);
335 chunking
= !striding
;
339 /* striding=true, chunking=true
341 striding=true, chunking=false
343 striding=false,chunking=true
344 -> chunks=ceil (range/(chunksize*threads*step))
345 striding=false,chunking=false
346 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
347 push_gimplify_context (true);
351 default: gcc_unreachable ();
353 case IFN_GOACC_LOOP_CHUNKS
:
355 r
= build_int_cst (type
, 1);
359 = (range - dir) / (chunks * step * num_threads) + dir */
360 tree per
= oacc_thread_numbers (false, mask
, &seq
);
361 per
= fold_convert (type
, per
);
362 chunk_size
= fold_convert (type
, chunk_size
);
363 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
364 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
365 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
366 r
= build2 (PLUS_EXPR
, type
, r
, per
);
367 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
371 case IFN_GOACC_LOOP_STEP
:
373 /* If striding, step by the entire compute volume, otherwise
374 step by the inner volume. */
375 unsigned volume
= striding
? mask
: inner_mask
;
377 r
= oacc_thread_numbers (false, volume
, &seq
);
378 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
382 case IFN_GOACC_LOOP_OFFSET
:
383 /* Enable vectorization on non-SIMT targets. */
385 && outer_mask
== GOMP_DIM_MASK (GOMP_DIM_VECTOR
)
386 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
388 && (flag_tree_loop_vectorize
389 || !global_options_set
.x_flag_tree_loop_vectorize
))
391 basic_block bb
= gsi_bb (gsi
);
392 class loop
*parent
= bb
->loop_father
;
393 class loop
*body
= parent
->inner
;
395 parent
->force_vectorize
= true;
396 parent
->safelen
= INT_MAX
;
398 /* "Chunking loops" may have inner loops. */
401 body
->force_vectorize
= true;
402 body
->safelen
= INT_MAX
;
405 cfun
->has_force_vectorize_loops
= true;
409 r
= oacc_thread_numbers (true, mask
, &seq
);
410 r
= fold_convert (diff_type
, r
);
414 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
415 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
416 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
417 inner_size
, outer_size
);
419 volume
= fold_convert (diff_type
, volume
);
421 chunk_size
= fold_convert (diff_type
, chunk_size
);
424 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
426 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
427 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
428 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
431 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
432 fold_convert (diff_type
, inner_size
));
433 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
434 r
= fold_convert (diff_type
, r
);
435 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
437 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
438 inner
= fold_convert (diff_type
, inner
);
439 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
443 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
445 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
446 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
448 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
451 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
452 if (type
!= diff_type
)
453 r
= fold_convert (type
, r
);
456 case IFN_GOACC_LOOP_BOUND
:
461 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
462 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
463 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
464 inner_size
, outer_size
);
466 volume
= fold_convert (diff_type
, volume
);
468 chunk_size
= fold_convert (diff_type
, chunk_size
);
471 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
473 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
474 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
475 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
478 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
479 fold_convert (diff_type
, inner_size
));
481 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
483 tree offset
= gimple_call_arg (call
, 6);
484 r
= build2 (PLUS_EXPR
, diff_type
, r
,
485 fold_convert (diff_type
, offset
));
486 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
487 diff_type
, r
, range
);
489 if (diff_type
!= type
)
490 r
= fold_convert (type
, r
);
494 gimplify_assign (lhs
, r
, &seq
);
496 pop_gimplify_context (NULL
);
498 gsi_replace_with_seq (&gsi
, seq
, true);
501 /* Transform a GOACC_TILE call. Determines the element loop span for
502 the specified loop of the nest. This is 1 if we're not tiling.
504 GOACC_TILE (collapse_count, loop_no, tile_arg, gwv_tile, gwv_element); */
507 oacc_xform_tile (gcall
*call
)
509 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
510 unsigned collapse
= tree_to_uhwi (gimple_call_arg (call
, 0));
511 /* Inner loops have higher loop_nos. */
512 unsigned loop_no
= tree_to_uhwi (gimple_call_arg (call
, 1));
513 tree tile_size
= gimple_call_arg (call
, 2);
514 unsigned e_mask
= tree_to_uhwi (gimple_call_arg (call
, 4));
515 tree lhs
= gimple_call_lhs (call
);
516 tree type
= TREE_TYPE (lhs
);
517 gimple_seq seq
= NULL
;
518 tree span
= build_int_cst (type
, 1);
521 & ~(GOMP_DIM_MASK (GOMP_DIM_VECTOR
)
522 | GOMP_DIM_MASK (GOMP_DIM_WORKER
))));
523 push_gimplify_context (!seen_error ());
525 #ifndef ACCEL_COMPILER
526 /* Partitioning disabled on host compilers. */
530 /* Not paritioning. */
531 span
= integer_one_node
;
532 else if (!integer_zerop (tile_size
))
533 /* User explicitly specified size. */
537 /* Pick a size based on the paritioning of the element loop and
538 the number of loop nests. */
539 tree first_size
= NULL_TREE
;
540 tree second_size
= NULL_TREE
;
542 if (e_mask
& GOMP_DIM_MASK (GOMP_DIM_VECTOR
))
543 first_size
= oacc_dim_call (false, GOMP_DIM_VECTOR
, &seq
);
544 if (e_mask
& GOMP_DIM_MASK (GOMP_DIM_WORKER
))
545 second_size
= oacc_dim_call (false, GOMP_DIM_WORKER
, &seq
);
549 first_size
= second_size
;
550 second_size
= NULL_TREE
;
553 if (loop_no
+ 1 == collapse
)
556 if (!loop_no
&& second_size
)
557 span
= fold_build2 (MULT_EXPR
, TREE_TYPE (span
),
560 else if (loop_no
+ 2 == collapse
)
566 /* There's no obvious element size for this loop. Options
567 are 1, first_size or some non-unity constant (32 is my
568 favourite). We should gather some statistics. */
572 span
= fold_convert (type
, span
);
573 gimplify_assign (lhs
, span
, &seq
);
575 pop_gimplify_context (NULL
);
577 gsi_replace_with_seq (&gsi
, seq
, true);
580 /* Default partitioned and minimum partitioned dimensions. */
582 static int oacc_default_dims
[GOMP_DIM_MAX
];
583 static int oacc_min_dims
[GOMP_DIM_MAX
];
586 oacc_get_default_dim (int dim
)
588 gcc_assert (0 <= dim
&& dim
< GOMP_DIM_MAX
);
589 return oacc_default_dims
[dim
];
593 oacc_get_min_dim (int dim
)
595 gcc_assert (0 <= dim
&& dim
< GOMP_DIM_MAX
);
596 return oacc_min_dims
[dim
];
599 /* Parse the default dimension parameter. This is a set of
600 :-separated optional compute dimensions. Each specified dimension
601 is a positive integer. When device type support is added, it is
602 planned to be a comma separated list of such compute dimensions,
603 with all but the first prefixed by the colon-terminated device
607 oacc_parse_default_dims (const char *dims
)
611 for (ix
= GOMP_DIM_MAX
; ix
--;)
613 oacc_default_dims
[ix
] = -1;
614 oacc_min_dims
[ix
] = 1;
617 #ifndef ACCEL_COMPILER
618 /* Cannot be overridden on the host. */
623 const char *pos
= dims
;
625 for (ix
= 0; *pos
&& ix
!= GOMP_DIM_MAX
; ix
++)
640 val
= strtol (pos
, CONST_CAST (char **, &eptr
), 10);
641 if (errno
|| val
<= 0 || (int) val
!= val
)
644 oacc_default_dims
[ix
] = (int) val
;
650 error_at (UNKNOWN_LOCATION
,
651 "%<-fopenacc-dim%> operand is malformed at %qs", pos
);
655 /* Allow the backend to validate the dimensions. */
656 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_default_dims
, -1, 0);
657 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_min_dims
, -2, 0);
660 /* Validate and update the dimensions for offloaded FN. ATTRS is the
661 raw attribute. DIMS is an array of dimensions, which is filled in.
662 LEVEL is the partitioning level of a routine, or -1 for an offload
663 region itself. USED is the mask of partitioned execution in the
667 oacc_validate_dims (tree fn
, tree attrs
, int *dims
, int level
, unsigned used
)
669 tree purpose
[GOMP_DIM_MAX
];
671 tree pos
= TREE_VALUE (attrs
);
673 /* Make sure the attribute creator attached the dimension
677 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
679 purpose
[ix
] = TREE_PURPOSE (pos
);
680 tree val
= TREE_VALUE (pos
);
681 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
682 pos
= TREE_CHAIN (pos
);
685 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, level
, used
);
687 /* Default anything left to 1 or a partitioned default. */
688 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
691 /* The OpenACC spec says 'If the [num_gangs] clause is not
692 specified, an implementation-defined default will be used;
693 the default may depend on the code within the construct.'
694 (2.5.6). Thus an implementation is free to choose
695 non-unity default for a parallel region that doesn't have
696 any gang-partitioned loops. However, it appears that there
697 is a sufficient body of user code that expects non-gang
698 partitioned regions to not execute in gang-redundant mode.
699 So we (a) don't warn about the non-portability and (b) pick
700 the minimum permissible dimension size when there is no
701 partitioned execution. Otherwise we pick the global
702 default for the dimension, which the user can control. The
703 same wording and logic applies to num_workers and
704 vector_length, however the worker- or vector- single
705 execution doesn't have the same impact as gang-redundant
706 execution. (If the minimum gang-level partioning is not 1,
707 the target is probably too confusing.) */
708 dims
[ix
] = (used
& GOMP_DIM_MASK (ix
)
709 ? oacc_default_dims
[ix
] : oacc_min_dims
[ix
]);
715 /* Replace the attribute with new values. */
717 for (ix
= GOMP_DIM_MAX
; ix
--;)
718 pos
= tree_cons (purpose
[ix
],
719 build_int_cst (integer_type_node
, dims
[ix
]), pos
);
720 oacc_replace_fn_attrib (fn
, pos
);
724 /* Create an empty OpenACC loop structure at LOC. */
727 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
729 oacc_loop
*loop
= XCNEW (oacc_loop
);
731 loop
->parent
= parent
;
735 loop
->sibling
= parent
->child
;
736 parent
->child
= loop
;
743 /* Create an outermost, dummy OpenACC loop for offloaded function
747 new_oacc_loop_outer (tree decl
)
749 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
752 /* Start a new OpenACC loop structure beginning at head marker HEAD.
753 Link into PARENT loop. Return the new loop. */
756 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
758 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
760 loop
->marker
= marker
;
762 /* TODO: This is where device_type flattening would occur for the loop
765 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
767 tree chunk_size
= integer_zero_node
;
768 if (loop
->flags
& OLF_GANG_STATIC
)
769 chunk_size
= gimple_call_arg (marker
, 4);
770 loop
->chunk_size
= chunk_size
;
775 /* Create a dummy loop encompassing a call to a openACC routine.
776 Extract the routine's partitioning requirements. */
779 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
781 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
782 int level
= oacc_fn_attrib_level (attrs
);
784 gcc_assert (level
>= 0);
787 loop
->routine
= decl
;
788 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
789 ^ (GOMP_DIM_MASK (level
) - 1));
792 /* Finish off the current OpenACC loop ending at tail marker TAIL.
793 Return the parent loop. */
796 finish_oacc_loop (oacc_loop
*loop
)
798 /* If the loop has been collapsed, don't partition it. */
799 if (loop
->ifns
.is_empty ())
800 loop
->mask
= loop
->flags
= 0;
804 /* Free all OpenACC loop structures within LOOP (inclusive). */
807 free_oacc_loop (oacc_loop
*loop
)
810 free_oacc_loop (loop
->sibling
);
812 free_oacc_loop (loop
->child
);
814 loop
->ifns
.release ();
818 /* Dump out the OpenACC loop head or tail beginning at FROM. */
821 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
822 const char *title
, int level
)
824 enum ifn_unique_kind kind
825 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
827 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
828 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
830 gimple
*stmt
= gsi_stmt (gsi
);
832 if (gimple_call_internal_p (stmt
, IFN_UNIQUE
))
834 enum ifn_unique_kind k
835 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
836 (gimple_call_arg (stmt
, 0)));
838 if (k
== kind
&& stmt
!= from
)
841 print_gimple_stmt (file
, stmt
, depth
* 2 + 2);
844 while (gsi_end_p (gsi
))
845 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
849 /* Dump OpenACC loop LOOP, its children, and its siblings. */
852 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
856 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
857 loop
->flags
, loop
->mask
,
858 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
861 print_gimple_stmt (file
, loop
->marker
, depth
* 2);
864 fprintf (file
, "%*sRoutine %s:%u:%s\n",
865 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
866 DECL_SOURCE_LINE (loop
->routine
),
867 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
869 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
871 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
872 for (ix
= GOMP_DIM_MAX
; ix
--;)
874 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
877 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
879 dump_oacc_loop (file
, loop
->sibling
, depth
);
882 void debug_oacc_loop (oacc_loop
*);
884 /* Dump loops to stderr. */
887 debug_oacc_loop (oacc_loop
*loop
)
889 dump_oacc_loop (stderr
, loop
, 0);
892 /* Provide diagnostics on OpenACC loop LOOP, its children, and its
896 inform_oacc_loop (const oacc_loop
*loop
)
899 = loop
->mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
) ? " gang" : "";
901 = loop
->mask
& GOMP_DIM_MASK (GOMP_DIM_WORKER
) ? " worker" : "";
903 = loop
->mask
& GOMP_DIM_MASK (GOMP_DIM_VECTOR
) ? " vector" : "";
904 const char *seq
= loop
->mask
== 0 ? " seq" : "";
905 const dump_user_location_t loc
906 = dump_user_location_t::from_location_t (loop
->loc
);
907 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, loc
,
908 "assigned OpenACC%s%s%s%s loop parallelism\n", gang
, worker
,
912 inform_oacc_loop (loop
->child
);
914 inform_oacc_loop (loop
->sibling
);
917 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
918 structures as we go. By construction these loops are properly
922 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
927 if (bb
->flags
& BB_VISITED
)
931 bb
->flags
|= BB_VISITED
;
933 /* Scan for loop markers. */
934 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
937 gimple
*stmt
= gsi_stmt (gsi
);
939 if (!is_gimple_call (stmt
))
942 gcall
*call
= as_a
<gcall
*> (stmt
);
944 /* If this is a routine, make a dummy loop for it. */
945 if (tree decl
= gimple_call_fndecl (call
))
946 if (tree attrs
= oacc_get_fn_attrib (decl
))
948 gcc_assert (!marker
);
949 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
952 if (!gimple_call_internal_p (call
))
955 switch (gimple_call_internal_fn (call
))
962 /* Record the abstraction function, so we can manipulate it
964 loop
->ifns
.safe_push (call
);
968 enum ifn_unique_kind kind
969 = (enum ifn_unique_kind
) (TREE_INT_CST_LOW
970 (gimple_call_arg (call
, 0)));
971 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
972 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
974 if (gimple_call_num_args (call
) == 2)
976 gcc_assert (marker
&& !remaining
);
978 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
979 loop
= finish_oacc_loop (loop
);
981 loop
->head_end
= call
;
985 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
989 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
990 loop
= new_oacc_loop (loop
, call
);
993 gcc_assert (count
== remaining
);
997 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
998 loop
->heads
[marker
] = call
;
1000 loop
->tails
[remaining
] = call
;
1007 if (remaining
|| marker
)
1009 bb
= single_succ (bb
);
1010 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
1014 /* Walk successor blocks. */
1018 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1019 oacc_loop_discover_walk (loop
, e
->dest
);
1022 /* LOOP is the first sibling. Reverse the order in place and return
1023 the new first sibling. Recurse to child loops. */
1026 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
1028 oacc_loop
*last
= NULL
;
1032 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
1034 oacc_loop
*next
= loop
->sibling
;
1035 loop
->sibling
= last
;
1044 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
1045 the current function. */
1048 oacc_loop_discovery ()
1050 /* Clear basic block flags, in particular BB_VISITED which we're going to use
1051 in the following. */
1054 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
1055 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1057 /* The siblings were constructed in reverse order, reverse them so
1058 that diagnostics come out in an unsurprising order. */
1059 top
= oacc_loop_sibling_nreverse (top
);
1064 /* Transform the abstract internal function markers starting at FROM
1065 to be for partitioning level LEVEL. Stop when we meet another HEAD
1069 oacc_loop_xform_head_tail (gcall
*from
, int level
)
1071 enum ifn_unique_kind kind
1072 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
1073 tree replacement
= build_int_cst (unsigned_type_node
, level
);
1075 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
1077 gimple
*stmt
= gsi_stmt (gsi
);
1079 if (gimple_call_internal_p (stmt
, IFN_UNIQUE
))
1081 enum ifn_unique_kind k
1082 = ((enum ifn_unique_kind
)
1083 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
1085 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
1086 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
1087 else if (k
== kind
&& stmt
!= from
)
1090 else if (gimple_call_internal_p (stmt
, IFN_GOACC_REDUCTION
))
1091 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
1094 while (gsi_end_p (gsi
))
1095 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
1099 /* Process the discovered OpenACC loops, setting the correct
1100 partitioning level etc. */
1103 oacc_loop_process (oacc_loop
*loop
)
1106 oacc_loop_process (loop
->child
);
1108 if (loop
->mask
&& !loop
->routine
)
1111 tree mask_arg
= build_int_cst (unsigned_type_node
, loop
->mask
);
1112 tree e_mask_arg
= build_int_cst (unsigned_type_node
, loop
->e_mask
);
1113 tree chunk_arg
= loop
->chunk_size
;
1116 for (ix
= 0; loop
->ifns
.iterate (ix
, &call
); ix
++)
1117 switch (gimple_call_internal_fn (call
))
1119 case IFN_GOACC_LOOP
:
1121 bool is_e
= gimple_call_arg (call
, 5) == integer_minus_one_node
;
1122 gimple_call_set_arg (call
, 5, is_e
? e_mask_arg
: mask_arg
);
1124 gimple_call_set_arg (call
, 4, chunk_arg
);
1128 case IFN_GOACC_TILE
:
1129 gimple_call_set_arg (call
, 3, mask_arg
);
1130 gimple_call_set_arg (call
, 4, e_mask_arg
);
1137 unsigned dim
= GOMP_DIM_GANG
;
1138 unsigned mask
= loop
->mask
| loop
->e_mask
;
1139 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& mask
; ix
++)
1141 while (!(GOMP_DIM_MASK (dim
) & mask
))
1144 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
1145 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
1147 mask
^= GOMP_DIM_MASK (dim
);
1152 oacc_loop_process (loop
->sibling
);
1155 /* Walk the OpenACC loop heirarchy checking and assigning the
1156 programmer-specified partitionings. OUTER_MASK is the partitioning
1157 this loop is contained within. Return mask of partitioning
1158 encountered. If any auto loops are discovered, set GOMP_DIM_MAX
1162 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
1164 unsigned this_mask
= loop
->mask
;
1165 unsigned mask_all
= 0;
1168 #ifdef ACCEL_COMPILER
1169 /* When device_type is supported, we want the device compiler to be
1170 noisy, if the loop parameters are device_type-specific. */
1176 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
1177 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
1178 bool tiling
= (loop
->flags
& OLF_TILE
) != 0;
1180 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
1181 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
1183 /* Apply auto partitioning if this is a non-partitioned regular
1184 loop, or (no more than) single axis tiled loop. */
1186 = !seq_par
&& this_mask
== (tiling
? this_mask
& -this_mask
: 0);
1188 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
1191 error_at (loop
->loc
,
1193 ? G_("%<seq%> overrides other OpenACC loop specifiers")
1194 : G_("%<auto%> conflicts with other OpenACC loop "
1197 loop
->flags
&= ~OLF_AUTO
;
1201 &= ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
1206 if (maybe_auto
&& (loop
->flags
& OLF_INDEPENDENT
))
1208 loop
->flags
|= OLF_AUTO
;
1209 mask_all
|= GOMP_DIM_MASK (GOMP_DIM_MAX
);
1213 if (this_mask
& outer_mask
)
1215 const oacc_loop
*outer
;
1216 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
1217 if ((outer
->mask
| outer
->e_mask
) & this_mask
)
1224 error_at (loop
->loc
,
1226 ? G_("routine call uses same OpenACC parallelism"
1227 " as containing loop")
1228 : G_("inner loop uses same OpenACC parallelism"
1229 " as containing loop"));
1230 inform (outer
->loc
, "containing loop here");
1233 error_at (loop
->loc
,
1235 ? G_("routine call uses OpenACC parallelism disallowed"
1236 " by containing routine")
1237 : G_("loop uses OpenACC parallelism disallowed"
1238 " by containing routine"));
1241 inform (DECL_SOURCE_LOCATION (loop
->routine
),
1242 "routine %qD declared here", loop
->routine
);
1244 this_mask
&= ~outer_mask
;
1248 unsigned outermost
= least_bit_hwi (this_mask
);
1250 if (outermost
&& outermost
<= outer_mask
)
1254 error_at (loop
->loc
,
1255 "incorrectly nested OpenACC loop parallelism");
1257 const oacc_loop
*outer
;
1258 for (outer
= loop
->parent
;
1259 outer
->flags
&& outer
->flags
< outermost
;
1260 outer
= outer
->parent
)
1262 inform (outer
->loc
, "containing loop here");
1265 this_mask
&= ~outermost
;
1269 mask_all
|= this_mask
;
1271 if (loop
->flags
& OLF_TILE
)
1273 /* When tiling, vector goes to the element loop, and failing
1274 that we put worker there. The std doesn't contemplate
1275 specifying all three. We choose to put worker and vector on
1276 the element loops in that case. */
1277 unsigned this_e_mask
= this_mask
& GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
1278 if (!this_e_mask
|| this_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
))
1279 this_e_mask
|= this_mask
& GOMP_DIM_MASK (GOMP_DIM_WORKER
);
1281 loop
->e_mask
= this_e_mask
;
1282 this_mask
^= this_e_mask
;
1285 loop
->mask
= this_mask
;
1288 fprintf (dump_file
, "Loop %s:%d user specified %d & %d\n",
1289 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
),
1290 loop
->mask
, loop
->e_mask
);
1294 unsigned tmp_mask
= outer_mask
| this_mask
| loop
->e_mask
;
1295 loop
->inner
= oacc_loop_fixed_partitions (loop
->child
, tmp_mask
);
1296 mask_all
|= loop
->inner
;
1300 mask_all
|= oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
);
1305 /* Walk the OpenACC loop heirarchy to assign auto-partitioned loops.
1306 OUTER_MASK is the partitioning this loop is contained within.
1307 OUTER_ASSIGN is true if an outer loop is being auto-partitioned.
1308 Return the cumulative partitioning used by this loop, siblings and
1312 oacc_loop_auto_partitions (oacc_loop
*loop
, unsigned outer_mask
,
1315 bool assign
= (loop
->flags
& OLF_AUTO
) && (loop
->flags
& OLF_INDEPENDENT
);
1317 bool tiling
= loop
->flags
& OLF_TILE
;
1319 #ifdef ACCEL_COMPILER
1320 /* When device_type is supported, we want the device compiler to be
1321 noisy, if the loop parameters are device_type-specific. */
1325 if (assign
&& (!outer_assign
|| loop
->inner
))
1327 /* Allocate outermost and non-innermost loops at the outermost
1328 non-innermost available level. */
1329 unsigned this_mask
= GOMP_DIM_MASK (GOMP_DIM_GANG
);
1331 /* Find the first outermost available partition. */
1332 while (this_mask
<= outer_mask
)
1335 /* Grab two axes if tiling, and we've not assigned anything */
1336 if (tiling
&& !(loop
->mask
| loop
->e_mask
))
1337 this_mask
|= this_mask
<< 1;
1339 /* Prohibit the innermost partitioning at the moment. */
1340 this_mask
&= GOMP_DIM_MASK (GOMP_DIM_MAX
- 1) - 1;
1342 /* Don't use any dimension explicitly claimed by an inner loop. */
1343 this_mask
&= ~loop
->inner
;
1345 if (tiling
&& !loop
->e_mask
)
1347 /* If we got two axes, allocate the inner one to the element
1349 loop
->e_mask
= this_mask
& (this_mask
<< 1);
1350 this_mask
^= loop
->e_mask
;
1353 loop
->mask
|= this_mask
;
1358 unsigned tmp_mask
= outer_mask
| loop
->mask
| loop
->e_mask
;
1359 loop
->inner
= oacc_loop_auto_partitions (loop
->child
, tmp_mask
,
1360 outer_assign
| assign
);
1363 if (assign
&& (!loop
->mask
|| (tiling
&& !loop
->e_mask
) || !outer_assign
))
1365 /* Allocate the loop at the innermost available level. Note
1366 that we do this even if we already assigned this loop the
1367 outermost available level above. That way we'll partition
1368 this along 2 axes, if they are available. */
1369 unsigned this_mask
= 0;
1371 /* Determine the outermost partitioning used within this loop. */
1372 this_mask
= loop
->inner
| GOMP_DIM_MASK (GOMP_DIM_MAX
);
1373 this_mask
= least_bit_hwi (this_mask
);
1375 /* Pick the partitioning just inside that one. */
1378 /* And avoid picking one use by an outer loop. */
1379 this_mask
&= ~outer_mask
;
1381 /* If tiling and we failed completely above, grab the next one
1382 too. Making sure it doesn't hit an outer loop. */
1385 this_mask
&= ~(loop
->e_mask
| loop
->mask
);
1386 unsigned tile_mask
= ((this_mask
>> 1)
1387 & ~(outer_mask
| loop
->e_mask
| loop
->mask
));
1389 if (tile_mask
|| loop
->mask
)
1391 loop
->e_mask
|= this_mask
;
1392 this_mask
= tile_mask
;
1394 if (!loop
->e_mask
&& noisy
)
1395 warning_at (loop
->loc
, 0,
1396 "insufficient partitioning available"
1397 " to parallelize element loop");
1400 loop
->mask
|= this_mask
;
1401 if (!loop
->mask
&& noisy
)
1402 warning_at (loop
->loc
, 0,
1404 ? G_("insufficient partitioning available"
1405 " to parallelize tile loop")
1406 : G_("insufficient partitioning available"
1407 " to parallelize loop"));
1410 if (assign
&& dump_file
)
1411 fprintf (dump_file
, "Auto loop %s:%d assigned %d & %d\n",
1412 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
),
1413 loop
->mask
, loop
->e_mask
);
1415 unsigned inner_mask
= 0;
1418 inner_mask
|= oacc_loop_auto_partitions (loop
->sibling
,
1419 outer_mask
, outer_assign
);
1421 inner_mask
|= loop
->inner
| loop
->mask
| loop
->e_mask
;
1426 /* Walk the OpenACC loop heirarchy to check and assign partitioning
1427 axes. Return mask of partitioning. */
1430 oacc_loop_partition (oacc_loop
*loop
, unsigned outer_mask
)
1432 unsigned mask_all
= oacc_loop_fixed_partitions (loop
, outer_mask
);
1434 if (mask_all
& GOMP_DIM_MASK (GOMP_DIM_MAX
))
1436 mask_all
^= GOMP_DIM_MASK (GOMP_DIM_MAX
);
1437 mask_all
|= oacc_loop_auto_partitions (loop
, outer_mask
, false);
1442 /* Default fork/join early expander. Delete the function calls if
1443 there is no RTL expander. */
1446 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
1447 const int *ARG_UNUSED (dims
), bool is_fork
)
1450 return targetm
.have_oacc_fork ();
1452 return targetm
.have_oacc_join ();
1455 /* Default goacc.reduction early expander.
1457 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
1458 If RES_PTR is not integer-zerop:
1459 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
1460 TEARDOWN - emit '*RES_PTR = VAR'
1465 default_goacc_reduction (gcall
*call
)
1467 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
1468 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
1469 tree lhs
= gimple_call_lhs (call
);
1470 tree var
= gimple_call_arg (call
, 2);
1471 gimple_seq seq
= NULL
;
1473 if (code
== IFN_GOACC_REDUCTION_SETUP
1474 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
1476 /* Setup and Teardown need to copy from/to the receiver object,
1478 tree ref_to_res
= gimple_call_arg (call
, 1);
1480 if (!integer_zerop (ref_to_res
))
1482 tree dst
= build_simple_mem_ref (ref_to_res
);
1485 if (code
== IFN_GOACC_REDUCTION_SETUP
)
1491 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
1495 /* Copy VAR to LHS, if there is an LHS. */
1497 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
1499 gsi_replace_with_seq (&gsi
, seq
, true);
1502 /* Main entry point for oacc transformations which run on the device
1503 compiler after LTO, so we know what the target device is at this
1504 point (including the host fallback). */
1507 execute_oacc_device_lower ()
1509 tree attrs
= oacc_get_fn_attrib (current_function_decl
);
1512 /* Not an offloaded function. */
1515 /* Parse the default dim argument exactly once. */
1516 if ((const void *)flag_openacc_dims
!= &flag_openacc_dims
)
1518 oacc_parse_default_dims (flag_openacc_dims
);
1519 flag_openacc_dims
= (char *)&flag_openacc_dims
;
1522 bool is_oacc_kernels
1523 = (lookup_attribute ("oacc kernels",
1524 DECL_ATTRIBUTES (current_function_decl
)) != NULL
);
1525 bool is_oacc_kernels_parallelized
1526 = (lookup_attribute ("oacc kernels parallelized",
1527 DECL_ATTRIBUTES (current_function_decl
)) != NULL
);
1529 /* Unparallelized OpenACC kernels constructs must get launched as 1 x 1 x 1
1530 kernels, so remove the parallelism dimensions function attributes
1531 potentially set earlier on. */
1532 if (is_oacc_kernels
&& !is_oacc_kernels_parallelized
)
1534 oacc_set_fn_attrib (current_function_decl
, NULL
, NULL
);
1535 attrs
= oacc_get_fn_attrib (current_function_decl
);
1538 /* Discover, partition and process the loops. */
1539 oacc_loop
*loops
= oacc_loop_discovery ();
1540 int fn_level
= oacc_fn_attrib_level (attrs
);
1545 fprintf (dump_file
, "Function is OpenACC routine level %d\n",
1547 else if (is_oacc_kernels
)
1548 fprintf (dump_file
, "Function is %s OpenACC kernels offload\n",
1549 (is_oacc_kernels_parallelized
1550 ? "parallelized" : "unparallelized"));
1552 fprintf (dump_file
, "Function is OpenACC parallel offload\n");
1555 unsigned outer_mask
= fn_level
>= 0 ? GOMP_DIM_MASK (fn_level
) - 1 : 0;
1556 unsigned used_mask
= oacc_loop_partition (loops
, outer_mask
);
1557 /* OpenACC kernels constructs are special: they currently don't use the
1558 generic oacc_loop infrastructure and attribute/dimension processing. */
1559 if (is_oacc_kernels
&& is_oacc_kernels_parallelized
)
1561 /* Parallelized OpenACC kernels constructs use gang parallelism. See
1562 also tree-parloops.c:create_parallel_loop. */
1563 used_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
1566 int dims
[GOMP_DIM_MAX
];
1567 oacc_validate_dims (current_function_decl
, attrs
, dims
, fn_level
, used_mask
);
1571 const char *comma
= "Compute dimensions [";
1572 for (int ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++, comma
= ", ")
1573 fprintf (dump_file
, "%s%d", comma
, dims
[ix
]);
1574 fprintf (dump_file
, "]\n");
1577 oacc_loop_process (loops
);
1580 fprintf (dump_file
, "OpenACC loops\n");
1581 dump_oacc_loop (dump_file
, loops
, 0);
1582 fprintf (dump_file
, "\n");
1584 if (dump_enabled_p ())
1586 oacc_loop
*l
= loops
;
1587 /* OpenACC kernels constructs are special: they currently don't use the
1588 generic oacc_loop infrastructure. */
1589 if (is_oacc_kernels
)
1591 /* Create a fake oacc_loop for diagnostic purposes. */
1592 l
= new_oacc_loop_raw (NULL
,
1593 DECL_SOURCE_LOCATION (current_function_decl
));
1594 l
->mask
= used_mask
;
1598 /* Skip the outermost, dummy OpenACC loop */
1602 inform_oacc_loop (l
);
1603 if (is_oacc_kernels
)
1607 /* Offloaded targets may introduce new basic blocks, which require
1608 dominance information to update SSA. */
1609 calculate_dominance_info (CDI_DOMINATORS
);
1611 /* Now lower internal loop functions to target-specific code
1614 FOR_ALL_BB_FN (bb
, cfun
)
1615 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
1617 gimple
*stmt
= gsi_stmt (gsi
);
1618 if (!is_gimple_call (stmt
))
1624 gcall
*call
= as_a
<gcall
*> (stmt
);
1625 if (!gimple_call_internal_p (call
))
1631 /* Rewind to allow rescan. */
1633 bool rescan
= false, remove
= false;
1634 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
1640 case IFN_GOACC_TILE
:
1641 oacc_xform_tile (call
);
1645 case IFN_GOACC_LOOP
:
1646 oacc_xform_loop (call
);
1650 case IFN_GOACC_REDUCTION
:
1651 /* Mark the function for SSA renaming. */
1652 mark_virtual_operands_for_renaming (cfun
);
1654 /* If the level is -1, this ended up being an unused
1655 axis. Handle as a default. */
1656 if (integer_minus_onep (gimple_call_arg (call
, 3)))
1657 default_goacc_reduction (call
);
1659 targetm
.goacc
.reduction (call
);
1665 enum ifn_unique_kind kind
1666 = ((enum ifn_unique_kind
)
1667 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
1674 case IFN_UNIQUE_OACC_FORK
:
1675 case IFN_UNIQUE_OACC_JOIN
:
1676 if (integer_minus_onep (gimple_call_arg (call
, 2)))
1678 else if (!targetm
.goacc
.fork_join
1679 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
1683 case IFN_UNIQUE_OACC_HEAD_MARK
:
1684 case IFN_UNIQUE_OACC_TAIL_MARK
:
1692 if (gsi_end_p (gsi
))
1693 /* We rewound past the beginning of the BB. */
1694 gsi
= gsi_start_bb (bb
);
1696 /* Undo the rewind. */
1701 if (gimple_vdef (call
))
1702 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
1703 if (gimple_call_lhs (call
))
1705 /* Propagate the data dependency var. */
1706 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
1707 gimple_call_arg (call
, 1));
1708 gsi_replace (&gsi
, ass
, false);
1711 gsi_remove (&gsi
, true);
1714 /* If not rescanning, advance over the call. */
1718 free_oacc_loop (loops
);
1723 /* Default launch dimension validator. Force everything to 1. A
1724 backend that wants to provide larger dimensions must override this
1728 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
1729 int ARG_UNUSED (fn_level
),
1730 unsigned ARG_UNUSED (used
))
1732 bool changed
= false;
1734 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
1746 /* Default dimension bound is unknown on accelerator and 1 on host. */
1749 default_goacc_dim_limit (int ARG_UNUSED (axis
))
1751 #ifdef ACCEL_COMPILER
1760 const pass_data pass_data_oacc_device_lower
=
1762 GIMPLE_PASS
, /* type */
1763 "oaccdevlow", /* name */
1764 OPTGROUP_OMP
, /* optinfo_flags */
1765 TV_NONE
, /* tv_id */
1766 PROP_cfg
, /* properties_required */
1767 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
1768 0, /* properties_destroyed */
1769 0, /* todo_flags_start */
1770 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
1773 class pass_oacc_device_lower
: public gimple_opt_pass
1776 pass_oacc_device_lower (gcc::context
*ctxt
)
1777 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
1780 /* opt_pass methods: */
1781 virtual bool gate (function
*) { return flag_openacc
; };
1783 virtual unsigned int execute (function
*)
1785 return execute_oacc_device_lower ();
1788 }; // class pass_oacc_device_lower
1793 make_pass_oacc_device_lower (gcc::context
*ctxt
)
1795 return new pass_oacc_device_lower (ctxt
);
1799 /* Rewrite GOMP_SIMT_ENTER_ALLOC call given by GSI and remove the preceding
1800 GOMP_SIMT_ENTER call identifying the privatized variables, which are
1801 turned to structure fields and receive a DECL_VALUE_EXPR accordingly.
1802 Set *REGIMPLIFY to true, except if no privatized variables were seen. */
1805 ompdevlow_adjust_simt_enter (gimple_stmt_iterator
*gsi
, bool *regimplify
)
1807 gimple
*alloc_stmt
= gsi_stmt (*gsi
);
1808 tree simtrec
= gimple_call_lhs (alloc_stmt
);
1809 tree simduid
= gimple_call_arg (alloc_stmt
, 0);
1810 gimple
*enter_stmt
= SSA_NAME_DEF_STMT (simduid
);
1811 gcc_assert (gimple_call_internal_p (enter_stmt
, IFN_GOMP_SIMT_ENTER
));
1812 tree rectype
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1813 TYPE_ARTIFICIAL (rectype
) = TYPE_NAMELESS (rectype
) = 1;
1814 TREE_ADDRESSABLE (rectype
) = 1;
1815 TREE_TYPE (simtrec
) = build_pointer_type (rectype
);
1816 for (unsigned i
= 1; i
< gimple_call_num_args (enter_stmt
); i
++)
1818 tree
*argp
= gimple_call_arg_ptr (enter_stmt
, i
);
1819 if (*argp
== null_pointer_node
)
1821 gcc_assert (TREE_CODE (*argp
) == ADDR_EXPR
1822 && VAR_P (TREE_OPERAND (*argp
, 0)));
1823 tree var
= TREE_OPERAND (*argp
, 0);
1825 tree field
= build_decl (DECL_SOURCE_LOCATION (var
), FIELD_DECL
,
1826 DECL_NAME (var
), TREE_TYPE (var
));
1827 SET_DECL_ALIGN (field
, DECL_ALIGN (var
));
1828 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1829 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1831 insert_field_into_struct (rectype
, field
);
1833 tree t
= build_simple_mem_ref (simtrec
);
1834 t
= build3 (COMPONENT_REF
, TREE_TYPE (var
), t
, field
, NULL
);
1835 TREE_THIS_VOLATILE (t
) = TREE_THIS_VOLATILE (var
);
1836 SET_DECL_VALUE_EXPR (var
, t
);
1837 DECL_HAS_VALUE_EXPR_P (var
) = 1;
1840 layout_type (rectype
);
1841 tree size
= TYPE_SIZE_UNIT (rectype
);
1842 tree align
= build_int_cst (TREE_TYPE (size
), TYPE_ALIGN_UNIT (rectype
));
1845 = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC
, 2, size
, align
);
1846 gimple_call_set_lhs (alloc_stmt
, simtrec
);
1847 gsi_replace (gsi
, alloc_stmt
, false);
1848 gimple_stmt_iterator enter_gsi
= gsi_for_stmt (enter_stmt
);
1849 enter_stmt
= gimple_build_assign (simduid
, gimple_call_arg (enter_stmt
, 0));
1850 gsi_replace (&enter_gsi
, enter_stmt
, false);
1854 if (single_imm_use (simtrec
, &use
, &exit_stmt
))
1856 gcc_assert (gimple_call_internal_p (exit_stmt
, IFN_GOMP_SIMT_EXIT
));
1857 gimple_stmt_iterator exit_gsi
= gsi_for_stmt (exit_stmt
);
1858 tree clobber
= build_constructor (rectype
, NULL
);
1859 TREE_THIS_VOLATILE (clobber
) = 1;
1860 exit_stmt
= gimple_build_assign (build_simple_mem_ref (simtrec
), clobber
);
1861 gsi_insert_before (&exit_gsi
, exit_stmt
, GSI_SAME_STMT
);
1864 gcc_checking_assert (has_zero_uses (simtrec
));
1867 /* Callback for walk_gimple_stmt used to scan for SIMT-privatized variables. */
1870 find_simtpriv_var_op (tree
*tp
, int *walk_subtrees
, void *)
1875 && DECL_HAS_VALUE_EXPR_P (t
)
1876 && lookup_attribute ("omp simt private", DECL_ATTRIBUTES (t
)))
1884 /* Cleanup uses of SIMT placeholder internal functions: on non-SIMT targets,
1885 VF is 1 and LANE is 0; on SIMT targets, VF is folded to a constant, and
1886 LANE is kept to be expanded to RTL later on. Also cleanup all other SIMT
1887 internal functions on non-SIMT targets, and likewise some SIMD internal
1888 functions on SIMT targets. */
1891 execute_omp_device_lower ()
1893 int vf
= targetm
.simt
.vf
? targetm
.simt
.vf () : 1;
1894 bool regimplify
= false;
1896 gimple_stmt_iterator gsi
;
1897 FOR_EACH_BB_FN (bb
, cfun
)
1898 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1900 gimple
*stmt
= gsi_stmt (gsi
);
1901 if (!is_gimple_call (stmt
) || !gimple_call_internal_p (stmt
))
1903 tree lhs
= gimple_call_lhs (stmt
), rhs
= NULL_TREE
;
1904 tree type
= lhs
? TREE_TYPE (lhs
) : integer_type_node
;
1905 switch (gimple_call_internal_fn (stmt
))
1907 case IFN_GOMP_USE_SIMT
:
1908 rhs
= vf
== 1 ? integer_zero_node
: integer_one_node
;
1910 case IFN_GOMP_SIMT_ENTER
:
1911 rhs
= vf
== 1 ? gimple_call_arg (stmt
, 0) : NULL_TREE
;
1912 goto simtreg_enter_exit
;
1913 case IFN_GOMP_SIMT_ENTER_ALLOC
:
1915 ompdevlow_adjust_simt_enter (&gsi
, ®implify
);
1916 rhs
= vf
== 1 ? null_pointer_node
: NULL_TREE
;
1917 goto simtreg_enter_exit
;
1918 case IFN_GOMP_SIMT_EXIT
:
1922 unlink_stmt_vdef (stmt
);
1924 case IFN_GOMP_SIMT_LANE
:
1925 case IFN_GOMP_SIMT_LAST_LANE
:
1926 rhs
= vf
== 1 ? build_zero_cst (type
) : NULL_TREE
;
1928 case IFN_GOMP_SIMT_VF
:
1929 rhs
= build_int_cst (type
, vf
);
1931 case IFN_GOMP_SIMT_ORDERED_PRED
:
1932 rhs
= vf
== 1 ? integer_zero_node
: NULL_TREE
;
1934 unlink_stmt_vdef (stmt
);
1936 case IFN_GOMP_SIMT_VOTE_ANY
:
1937 case IFN_GOMP_SIMT_XCHG_BFLY
:
1938 case IFN_GOMP_SIMT_XCHG_IDX
:
1939 rhs
= vf
== 1 ? gimple_call_arg (stmt
, 0) : NULL_TREE
;
1941 case IFN_GOMP_SIMD_LANE
:
1942 case IFN_GOMP_SIMD_LAST_LANE
:
1943 rhs
= vf
!= 1 ? build_zero_cst (type
) : NULL_TREE
;
1945 case IFN_GOMP_SIMD_VF
:
1946 rhs
= vf
!= 1 ? build_one_cst (type
) : NULL_TREE
;
1953 stmt
= lhs
? gimple_build_assign (lhs
, rhs
) : gimple_build_nop ();
1954 gsi_replace (&gsi
, stmt
, false);
1957 FOR_EACH_BB_REVERSE_FN (bb
, cfun
)
1958 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1959 if (walk_gimple_stmt (&gsi
, NULL
, find_simtpriv_var_op
, NULL
))
1961 if (gimple_clobber_p (gsi_stmt (gsi
)))
1962 gsi_remove (&gsi
, true);
1964 gimple_regimplify_operands (gsi_stmt (gsi
), &gsi
);
1967 cfun
->has_force_vectorize_loops
= false;
1973 const pass_data pass_data_omp_device_lower
=
1975 GIMPLE_PASS
, /* type */
1976 "ompdevlow", /* name */
1977 OPTGROUP_OMP
, /* optinfo_flags */
1978 TV_NONE
, /* tv_id */
1979 PROP_cfg
, /* properties_required */
1980 PROP_gimple_lomp_dev
, /* properties_provided */
1981 0, /* properties_destroyed */
1982 0, /* todo_flags_start */
1983 TODO_update_ssa
, /* todo_flags_finish */
1986 class pass_omp_device_lower
: public gimple_opt_pass
1989 pass_omp_device_lower (gcc::context
*ctxt
)
1990 : gimple_opt_pass (pass_data_omp_device_lower
, ctxt
)
1993 /* opt_pass methods: */
1994 virtual bool gate (function
*fun
)
1996 return !(fun
->curr_properties
& PROP_gimple_lomp_dev
);
1998 virtual unsigned int execute (function
*)
2000 return execute_omp_device_lower ();
2003 }; // class pass_expand_omp_ssa
2008 make_pass_omp_device_lower (gcc::context
*ctxt
)
2010 return new pass_omp_device_lower (ctxt
);
2013 /* "omp declare target link" handling pass. */
2017 const pass_data pass_data_omp_target_link
=
2019 GIMPLE_PASS
, /* type */
2020 "omptargetlink", /* name */
2021 OPTGROUP_OMP
, /* optinfo_flags */
2022 TV_NONE
, /* tv_id */
2023 PROP_ssa
, /* properties_required */
2024 0, /* properties_provided */
2025 0, /* properties_destroyed */
2026 0, /* todo_flags_start */
2027 TODO_update_ssa
, /* todo_flags_finish */
2030 class pass_omp_target_link
: public gimple_opt_pass
2033 pass_omp_target_link (gcc::context
*ctxt
)
2034 : gimple_opt_pass (pass_data_omp_target_link
, ctxt
)
2037 /* opt_pass methods: */
2038 virtual bool gate (function
*fun
)
2040 #ifdef ACCEL_COMPILER
2041 return offloading_function_p (fun
->decl
);
2048 virtual unsigned execute (function
*);
2051 /* Callback for walk_gimple_stmt used to scan for link var operands. */
2054 find_link_var_op (tree
*tp
, int *walk_subtrees
, void *)
2059 && DECL_HAS_VALUE_EXPR_P (t
)
2060 && is_global_var (t
)
2061 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t
)))
2071 pass_omp_target_link::execute (function
*fun
)
2074 FOR_EACH_BB_FN (bb
, fun
)
2076 gimple_stmt_iterator gsi
;
2077 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
2078 if (walk_gimple_stmt (&gsi
, NULL
, find_link_var_op
, NULL
))
2079 gimple_regimplify_operands (gsi_stmt (gsi
), &gsi
);
2088 make_pass_omp_target_link (gcc::context
*ctxt
)
2090 return new pass_omp_target_link (ctxt
);