1 /* OMP constructs' SIMD clone supporting code.
3 Copyright (C) 2005-2018 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
44 #include "tree-into-ssa.h"
47 #include "symbol-summary.h"
48 #include "ipa-param-manipulation.h"
51 #include "stringpool.h"
53 #include "omp-simd-clone.h"
55 /* Return the number of elements in vector type VECTYPE, which is associated
56 with a SIMD clone. At present these always have a constant length. */
58 static unsigned HOST_WIDE_INT
59 simd_clone_subparts (tree vectype
)
61 return TYPE_VECTOR_SUBPARTS (vectype
).to_constant ();
64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
65 of arguments to reserve space for. */
67 static struct cgraph_simd_clone
*
68 simd_clone_struct_alloc (int nargs
)
70 struct cgraph_simd_clone
*clone_info
;
71 size_t len
= (sizeof (struct cgraph_simd_clone
)
72 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
73 clone_info
= (struct cgraph_simd_clone
*)
74 ggc_internal_cleared_alloc (len
);
78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
81 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
82 struct cgraph_simd_clone
*from
)
84 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
85 + ((from
->nargs
- from
->inbranch
)
86 * sizeof (struct cgraph_simd_clone_arg
))));
89 /* Return vector of parameter types of function FNDECL. This uses
90 TYPE_ARG_TYPES if available, otherwise falls back to types of
91 DECL_ARGUMENTS types. */
94 simd_clone_vector_of_formal_parm_types (tree fndecl
)
96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
97 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
98 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
101 FOR_EACH_VEC_ELT (args
, i
, arg
)
102 args
[i
] = TREE_TYPE (args
[i
]);
106 /* Given a simd function in NODE, extract the simd specific
107 information from the OMP clauses passed in CLAUSES, and return
108 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
109 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
110 otherwise set to FALSE. */
112 static struct cgraph_simd_clone
*
113 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
114 bool *inbranch_specified
)
116 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
119 *inbranch_specified
= false;
122 if (n
> 0 && args
.last () == void_type_node
)
125 /* Allocate one more than needed just in case this is an in-branch
126 clone which will require a mask argument. */
127 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
128 clone_info
->nargs
= n
;
133 clauses
= TREE_VALUE (clauses
);
134 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
137 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
139 switch (OMP_CLAUSE_CODE (t
))
141 case OMP_CLAUSE_INBRANCH
:
142 clone_info
->inbranch
= 1;
143 *inbranch_specified
= true;
145 case OMP_CLAUSE_NOTINBRANCH
:
146 clone_info
->inbranch
= 0;
147 *inbranch_specified
= true;
149 case OMP_CLAUSE_SIMDLEN
:
151 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
153 case OMP_CLAUSE_LINEAR
:
155 tree decl
= OMP_CLAUSE_DECL (t
);
156 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
157 int argno
= TREE_INT_CST_LOW (decl
);
158 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
160 enum cgraph_simd_clone_arg_type arg_type
;
161 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
162 switch (OMP_CLAUSE_LINEAR_KIND (t
))
164 case OMP_CLAUSE_LINEAR_REF
:
166 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
168 case OMP_CLAUSE_LINEAR_UVAL
:
170 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
172 case OMP_CLAUSE_LINEAR_VAL
:
173 case OMP_CLAUSE_LINEAR_DEFAULT
:
175 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
181 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
182 clone_info
->args
[argno
].arg_type
= arg_type
;
183 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
184 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
185 && clone_info
->args
[argno
].linear_step
< n
);
189 if (POINTER_TYPE_P (args
[argno
]))
190 step
= fold_convert (ssizetype
, step
);
191 if (!tree_fits_shwi_p (step
))
193 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
194 "ignoring large linear step");
198 else if (integer_zerop (step
))
200 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
201 "ignoring zero linear step");
207 enum cgraph_simd_clone_arg_type arg_type
;
208 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
209 switch (OMP_CLAUSE_LINEAR_KIND (t
))
211 case OMP_CLAUSE_LINEAR_REF
:
213 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
215 case OMP_CLAUSE_LINEAR_UVAL
:
217 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
219 case OMP_CLAUSE_LINEAR_VAL
:
220 case OMP_CLAUSE_LINEAR_DEFAULT
:
222 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
228 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
229 clone_info
->args
[argno
].arg_type
= arg_type
;
230 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
235 case OMP_CLAUSE_UNIFORM
:
237 tree decl
= OMP_CLAUSE_DECL (t
);
238 int argno
= tree_to_uhwi (decl
);
239 clone_info
->args
[argno
].arg_type
240 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
243 case OMP_CLAUSE_ALIGNED
:
245 tree decl
= OMP_CLAUSE_DECL (t
);
246 int argno
= tree_to_uhwi (decl
);
247 clone_info
->args
[argno
].alignment
248 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
257 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node
->decl
))))
259 warning_at (DECL_SOURCE_LOCATION (node
->decl
), 0,
260 "ignoring %<#pragma omp declare simd%> on function "
261 "with %<_Atomic%> qualified return type");
266 for (unsigned int argno
= 0; argno
< clone_info
->nargs
; argno
++)
267 if (TYPE_ATOMIC (args
[argno
])
268 && clone_info
->args
[argno
].arg_type
!= SIMD_CLONE_ARG_TYPE_UNIFORM
)
270 warning_at (DECL_SOURCE_LOCATION (node
->decl
), 0,
271 "ignoring %<#pragma omp declare simd%> on function "
272 "with %<_Atomic%> qualified non-%<uniform%> argument");
281 /* Given a SIMD clone in NODE, calculate the characteristic data
282 type and return the coresponding type. The characteristic data
283 type is computed as described in the Intel Vector ABI. */
286 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
287 struct cgraph_simd_clone
*clone_info
)
289 tree type
= integer_type_node
;
290 tree fndecl
= node
->decl
;
292 /* a) For non-void function, the characteristic data type is the
294 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
295 type
= TREE_TYPE (TREE_TYPE (fndecl
));
297 /* b) If the function has any non-uniform, non-linear parameters,
298 then the characteristic data type is the type of the first
302 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
303 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
304 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
312 /* c) If the characteristic data type determined by a) or b) above
313 is struct, union, or class type which is pass-by-value (except
314 for the type that maps to the built-in complex data type), the
315 characteristic data type is int. */
316 if (RECORD_OR_UNION_TYPE_P (type
)
317 && !aggregate_value_p (type
, NULL
)
318 && TREE_CODE (type
) != COMPLEX_TYPE
)
319 return integer_type_node
;
321 /* d) If none of the above three classes is applicable, the
322 characteristic data type is int. */
326 /* e) For Intel Xeon Phi native and offload compilation, if the
327 resulting characteristic data type is 8-bit or 16-bit integer
328 data type, the characteristic data type is int. */
329 /* Well, we don't handle Xeon Phi yet. */
333 simd_clone_mangle (struct cgraph_node
*node
,
334 struct cgraph_simd_clone
*clone_info
)
336 char vecsize_mangle
= clone_info
->vecsize_mangle
;
337 char mask
= clone_info
->inbranch
? 'M' : 'N';
338 unsigned int simdlen
= clone_info
->simdlen
;
342 gcc_assert (vecsize_mangle
&& simdlen
);
344 pp_string (&pp
, "_ZGV");
345 pp_character (&pp
, vecsize_mangle
);
346 pp_character (&pp
, mask
);
347 pp_decimal_int (&pp
, simdlen
);
349 for (n
= 0; n
< clone_info
->nargs
; ++n
)
351 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
353 switch (arg
.arg_type
)
355 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
356 pp_character (&pp
, 'u');
358 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
359 pp_character (&pp
, 'l');
361 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
362 pp_character (&pp
, 'R');
364 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
365 pp_character (&pp
, 'L');
367 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
368 pp_character (&pp
, 'U');
371 gcc_assert (arg
.linear_step
!= 0);
372 if (arg
.linear_step
> 1)
373 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
374 else if (arg
.linear_step
< 0)
376 pp_character (&pp
, 'n');
377 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
381 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
382 pp_string (&pp
, "ls");
383 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
385 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
386 pp_string (&pp
, "Rs");
387 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
389 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
390 pp_string (&pp
, "Ls");
391 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
393 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
394 pp_string (&pp
, "Us");
395 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
398 pp_character (&pp
, 'v');
402 pp_character (&pp
, 'a');
403 pp_decimal_int (&pp
, arg
.alignment
);
408 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
411 pp_string (&pp
, str
);
412 str
= pp_formatted_text (&pp
);
414 /* If there already is a SIMD clone with the same mangled name, don't
415 add another one. This can happen e.g. for
416 #pragma omp declare simd
417 #pragma omp declare simd simdlen(8)
419 if the simdlen is assumed to be 8 for the first one, etc. */
420 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
421 clone
= clone
->simdclone
->next_clone
)
422 if (id_equal (DECL_ASSEMBLER_NAME (clone
->decl
), str
))
425 return get_identifier (str
);
428 /* Create a simd clone of OLD_NODE and return it. */
430 static struct cgraph_node
*
431 simd_clone_create (struct cgraph_node
*old_node
)
433 struct cgraph_node
*new_node
;
434 if (old_node
->definition
)
436 if (!old_node
->has_gimple_body_p ())
438 old_node
->get_body ();
439 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
445 tree old_decl
= old_node
->decl
;
446 tree new_decl
= copy_node (old_node
->decl
);
447 DECL_NAME (new_decl
) = clone_function_name_numbered (old_decl
,
449 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
450 SET_DECL_RTL (new_decl
, NULL
);
451 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
452 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
453 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
454 if (old_node
->in_other_partition
)
455 new_node
->in_other_partition
= 1;
457 if (new_node
== NULL
)
460 DECL_BUILT_IN_CLASS (new_node
->decl
) = NOT_BUILT_IN
;
461 DECL_FUNCTION_CODE (new_node
->decl
) = (enum built_in_function
) 0;
462 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
463 DECL_COMDAT (new_node
->decl
) = DECL_COMDAT (old_node
->decl
);
464 DECL_WEAK (new_node
->decl
) = DECL_WEAK (old_node
->decl
);
465 DECL_EXTERNAL (new_node
->decl
) = DECL_EXTERNAL (old_node
->decl
);
466 DECL_VISIBILITY_SPECIFIED (new_node
->decl
)
467 = DECL_VISIBILITY_SPECIFIED (old_node
->decl
);
468 DECL_VISIBILITY (new_node
->decl
) = DECL_VISIBILITY (old_node
->decl
);
469 DECL_DLLIMPORT_P (new_node
->decl
) = DECL_DLLIMPORT_P (old_node
->decl
);
470 if (DECL_ONE_ONLY (old_node
->decl
))
471 make_decl_one_only (new_node
->decl
, DECL_ASSEMBLER_NAME (new_node
->decl
));
473 /* The method cgraph_version_clone_with_body () will force the new
474 symbol local. Undo this, and inherit external visibility from
476 new_node
->local
.local
= old_node
->local
.local
;
477 new_node
->externally_visible
= old_node
->externally_visible
;
482 /* Adjust the return type of the given function to its appropriate
483 vector counterpart. Returns a simd array to be used throughout the
484 function as a return value. */
487 simd_clone_adjust_return_type (struct cgraph_node
*node
)
489 tree fndecl
= node
->decl
;
490 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
494 /* Adjust the function return type. */
495 if (orig_rettype
== void_type_node
)
497 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
498 t
= TREE_TYPE (TREE_TYPE (fndecl
));
499 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
500 veclen
= node
->simdclone
->vecsize_int
;
502 veclen
= node
->simdclone
->vecsize_float
;
503 veclen
/= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t
));
504 if (veclen
> node
->simdclone
->simdlen
)
505 veclen
= node
->simdclone
->simdlen
;
506 if (POINTER_TYPE_P (t
))
507 t
= pointer_sized_int_node
;
508 if (veclen
== node
->simdclone
->simdlen
)
509 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
512 t
= build_vector_type (t
, veclen
);
513 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
515 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
516 if (!node
->definition
)
519 t
= DECL_RESULT (fndecl
);
520 /* Adjust the DECL_RESULT. */
521 gcc_assert (TREE_TYPE (t
) != void_type_node
);
522 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
525 tree atype
= build_array_type_nelts (orig_rettype
,
526 node
->simdclone
->simdlen
);
527 if (veclen
!= node
->simdclone
->simdlen
)
528 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
530 /* Set up a SIMD array to use as the return value. */
531 tree retval
= create_tmp_var_raw (atype
, "retval");
532 gimple_add_tmp_var (retval
);
536 /* Each vector argument has a corresponding array to be used locally
537 as part of the eventual loop. Create such temporary array and
540 PREFIX is the prefix to be used for the temporary.
542 TYPE is the inner element type.
544 SIMDLEN is the number of elements. */
547 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
549 tree atype
= build_array_type_nelts (type
, simdlen
);
550 tree avar
= create_tmp_var_raw (atype
, prefix
);
551 gimple_add_tmp_var (avar
);
555 /* Modify the function argument types to their corresponding vector
556 counterparts if appropriate. Also, create one array for each simd
557 argument to be used locally when using the function arguments as
560 NODE is the function whose arguments are to be adjusted.
562 Returns an adjustment vector that will be filled describing how the
563 argument types will be adjusted. */
565 static ipa_parm_adjustment_vec
566 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
569 ipa_parm_adjustment_vec adjustments
;
571 if (node
->definition
)
572 args
= ipa_get_vector_of_formal_parms (node
->decl
);
574 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
575 adjustments
.create (args
.length ());
576 unsigned i
, j
, veclen
;
577 struct ipa_parm_adjustment adj
;
578 struct cgraph_simd_clone
*sc
= node
->simdclone
;
580 for (i
= 0; i
< sc
->nargs
; ++i
)
582 memset (&adj
, 0, sizeof (adj
));
584 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
588 sc
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
589 sc
->args
[i
].orig_type
= parm_type
;
591 switch (sc
->args
[i
].arg_type
)
594 /* No adjustment necessary for scalar arguments. */
595 adj
.op
= IPA_PARM_OP_COPY
;
597 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
598 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
599 if (node
->definition
)
600 sc
->args
[i
].simd_array
601 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
602 TREE_TYPE (parm_type
),
604 adj
.op
= IPA_PARM_OP_COPY
;
606 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
607 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
608 case SIMD_CLONE_ARG_TYPE_VECTOR
:
609 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
610 veclen
= sc
->vecsize_int
;
612 veclen
= sc
->vecsize_float
;
613 veclen
/= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type
));
614 if (veclen
> sc
->simdlen
)
615 veclen
= sc
->simdlen
;
616 adj
.arg_prefix
= "simd";
617 if (POINTER_TYPE_P (parm_type
))
618 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
620 adj
.type
= build_vector_type (parm_type
, veclen
);
621 sc
->args
[i
].vector_type
= adj
.type
;
622 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
624 adjustments
.safe_push (adj
);
627 memset (&adj
, 0, sizeof (adj
));
628 adj
.op
= IPA_PARM_OP_NEW
;
629 adj
.arg_prefix
= "simd";
631 adj
.type
= sc
->args
[i
].vector_type
;
635 if (node
->definition
)
636 sc
->args
[i
].simd_array
637 = create_tmp_simd_array (DECL_NAME (parm
)
638 ? IDENTIFIER_POINTER (DECL_NAME (parm
))
639 : NULL
, parm_type
, sc
->simdlen
);
641 adjustments
.safe_push (adj
);
646 tree base_type
= simd_clone_compute_base_data_type (sc
->origin
, sc
);
648 memset (&adj
, 0, sizeof (adj
));
649 adj
.op
= IPA_PARM_OP_NEW
;
650 adj
.arg_prefix
= "mask";
653 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
654 veclen
= sc
->vecsize_int
;
656 veclen
= sc
->vecsize_float
;
657 veclen
/= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type
));
658 if (veclen
> sc
->simdlen
)
659 veclen
= sc
->simdlen
;
660 if (sc
->mask_mode
!= VOIDmode
)
662 = lang_hooks
.types
.type_for_mode (sc
->mask_mode
, 1);
663 else if (POINTER_TYPE_P (base_type
))
664 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
666 adj
.type
= build_vector_type (base_type
, veclen
);
667 adjustments
.safe_push (adj
);
669 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
670 adjustments
.safe_push (adj
);
672 /* We have previously allocated one extra entry for the mask. Use
675 if (sc
->mask_mode
!= VOIDmode
)
676 base_type
= boolean_type_node
;
677 if (node
->definition
)
680 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
681 if (sc
->mask_mode
== VOIDmode
)
682 sc
->args
[i
].simd_array
683 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
684 else if (veclen
< sc
->simdlen
)
685 sc
->args
[i
].simd_array
686 = create_tmp_simd_array ("mask", adj
.type
, sc
->simdlen
/ veclen
);
688 sc
->args
[i
].simd_array
= NULL_TREE
;
690 sc
->args
[i
].orig_type
= base_type
;
691 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
694 if (node
->definition
)
695 ipa_modify_formal_parameters (node
->decl
, adjustments
);
698 tree new_arg_types
= NULL_TREE
, new_reversed
;
699 bool last_parm_void
= false;
700 if (args
.length () > 0 && args
.last () == void_type_node
)
701 last_parm_void
= true;
703 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
704 j
= adjustments
.length ();
705 for (i
= 0; i
< j
; i
++)
707 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
709 if (adj
->op
== IPA_PARM_OP_COPY
)
710 ptype
= args
[adj
->base_index
];
713 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
715 new_reversed
= nreverse (new_arg_types
);
719 TREE_CHAIN (new_arg_types
) = void_list_node
;
721 new_reversed
= void_list_node
;
724 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
725 TYPE_ARG_TYPES (new_type
) = new_reversed
;
726 TREE_TYPE (node
->decl
) = new_type
;
728 adjustments
.release ();
734 /* Initialize and copy the function arguments in NODE to their
735 corresponding local simd arrays. Returns a fresh gimple_seq with
736 the instruction sequence generated. */
739 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
740 ipa_parm_adjustment_vec adjustments
)
742 gimple_seq seq
= NULL
;
743 unsigned i
= 0, j
= 0, k
;
745 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
747 arg
= DECL_CHAIN (arg
), i
++, j
++)
749 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
750 || POINTER_TYPE_P (TREE_TYPE (arg
)))
753 node
->simdclone
->args
[i
].vector_arg
= arg
;
755 tree array
= node
->simdclone
->args
[i
].simd_array
;
756 if (node
->simdclone
->mask_mode
!= VOIDmode
757 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_MASK
)
759 if (array
== NULL_TREE
)
762 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array
))));
763 for (k
= 0; k
<= l
; k
++)
767 arg
= DECL_CHAIN (arg
);
770 tree t
= build4 (ARRAY_REF
, TREE_TYPE (TREE_TYPE (array
)),
771 array
, size_int (k
), NULL
, NULL
);
772 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
773 gimplify_and_add (t
, &seq
);
777 if (simd_clone_subparts (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
779 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
780 tree ptr
= build_fold_addr_expr (array
);
781 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
782 build_int_cst (ptype
, 0));
783 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
784 gimplify_and_add (t
, &seq
);
788 unsigned int simdlen
= simd_clone_subparts (TREE_TYPE (arg
));
789 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
790 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
792 tree ptr
= build_fold_addr_expr (array
);
796 arg
= DECL_CHAIN (arg
);
799 tree elemtype
= TREE_TYPE (TREE_TYPE (arg
));
800 elemsize
= GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype
));
801 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
802 build_int_cst (ptype
, k
* elemsize
));
803 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
804 gimplify_and_add (t
, &seq
);
811 /* Callback info for ipa_simd_modify_stmt_ops below. */
813 struct modify_stmt_info
{
814 ipa_parm_adjustment_vec adjustments
;
816 /* True if the parent statement was modified by
817 ipa_simd_modify_stmt_ops. */
821 /* Callback for walk_gimple_op.
823 Adjust operands from a given statement as specified in the
824 adjustments vector in the callback data. */
827 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
829 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
830 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
832 if (TREE_CODE (*tp
) == ADDR_EXPR
)
833 tp
= &TREE_OPERAND (*tp
, 0);
834 struct ipa_parm_adjustment
*cand
= NULL
;
835 if (TREE_CODE (*tp
) == PARM_DECL
)
836 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
837 else if (TYPE_P (*tp
))
840 tree repl
= NULL_TREE
;
842 repl
= unshare_expr (cand
->new_decl
);
848 bool modified
= info
->modified
;
849 info
->modified
= false;
850 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
853 info
->modified
= modified
;
856 info
->modified
= modified
;
865 repl
= build_fold_addr_expr (repl
);
867 if (is_gimple_debug (info
->stmt
))
869 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
870 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
871 DECL_ARTIFICIAL (vexpr
) = 1;
872 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
873 SET_DECL_MODE (vexpr
, TYPE_MODE (TREE_TYPE (repl
)));
878 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
879 repl
= gimple_assign_lhs (stmt
);
881 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
882 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
885 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
887 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
893 info
->modified
= true;
897 /* Traverse the function body and perform all modifications as
898 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
899 modified such that the replacement/reduction value will now be an
900 offset into the corresponding simd_array.
902 This function will replace all function argument uses with their
903 corresponding simd array elements, and ajust the return values
907 ipa_simd_modify_function_body (struct cgraph_node
*node
,
908 ipa_parm_adjustment_vec adjustments
,
909 tree retval_array
, tree iter
)
912 unsigned int i
, j
, l
;
914 /* Re-use the adjustments array, but this time use it to replace
915 every function argument use to an offset into the corresponding
917 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
919 if (!node
->simdclone
->args
[i
].vector_arg
)
922 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
923 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
924 adjustments
[j
].new_decl
927 node
->simdclone
->args
[i
].simd_array
,
929 NULL_TREE
, NULL_TREE
);
930 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
931 && simd_clone_subparts (vectype
) < node
->simdclone
->simdlen
)
932 j
+= node
->simdclone
->simdlen
/ simd_clone_subparts (vectype
) - 1;
935 l
= adjustments
.length ();
938 FOR_EACH_SSA_NAME (i
, name
, cfun
)
940 if (SSA_NAME_VAR (name
)
941 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
943 for (j
= 0; j
< l
; j
++)
944 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
945 && adjustments
[j
].new_decl
)
948 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
951 = copy_var_decl (adjustments
[j
].base
,
952 DECL_NAME (adjustments
[j
].base
),
953 TREE_TYPE (adjustments
[j
].base
));
954 adjustments
[j
].new_ssa_base
= base_var
;
957 base_var
= adjustments
[j
].new_ssa_base
;
958 if (SSA_NAME_IS_DEFAULT_DEF (name
))
960 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
961 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
962 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
963 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
964 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
965 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
966 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
967 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
970 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
975 struct modify_stmt_info info
;
976 info
.adjustments
= adjustments
;
978 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
980 gimple_stmt_iterator gsi
;
982 gsi
= gsi_start_bb (bb
);
983 while (!gsi_end_p (gsi
))
985 gimple
*stmt
= gsi_stmt (gsi
);
987 struct walk_stmt_info wi
;
989 memset (&wi
, 0, sizeof (wi
));
990 info
.modified
= false;
992 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
994 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
996 tree retval
= gimple_return_retval (return_stmt
);
999 gsi_remove (&gsi
, true);
1003 /* Replace `return foo' with `retval_array[iter] = foo'. */
1004 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
1005 retval_array
, iter
, NULL
, NULL
);
1006 stmt
= gimple_build_assign (ref
, retval
);
1007 gsi_replace (&gsi
, stmt
, true);
1008 info
.modified
= true;
1014 /* If the above changed the var of a debug bind into something
1015 different, remove the debug stmt. We could also for all the
1016 replaced parameters add VAR_DECLs for debug info purposes,
1017 add debug stmts for those to be the simd array accesses and
1018 replace debug stmt var operand with that var. Debugging of
1019 vectorized loops doesn't work too well, so don't bother for
1021 if ((gimple_debug_bind_p (stmt
)
1022 && !DECL_P (gimple_debug_bind_get_var (stmt
)))
1023 || (gimple_debug_source_bind_p (stmt
)
1024 && !DECL_P (gimple_debug_source_bind_get_var (stmt
))))
1026 gsi_remove (&gsi
, true);
1029 if (maybe_clean_eh_stmt (stmt
))
1030 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
1037 /* Helper function of simd_clone_adjust, return linear step addend
1041 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
1042 tree addtype
, basic_block entry_bb
)
1044 tree ptype
= NULL_TREE
;
1045 switch (node
->simdclone
->args
[i
].arg_type
)
1047 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
1048 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
1049 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
1050 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
1051 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
1052 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
1053 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
1054 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
1056 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
1057 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
1058 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
1064 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
1065 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
1066 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
1067 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1070 if (is_gimple_reg (arg
))
1071 ret
= get_or_create_ssa_default_def (cfun
, arg
);
1074 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
1075 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1076 ret
= gimple_assign_lhs (g
);
1078 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
1080 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
1081 build_simple_mem_ref (ret
));
1082 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1083 ret
= gimple_assign_lhs (g
);
1085 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
1087 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
1088 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1089 ret
= gimple_assign_lhs (g
);
1091 if (POINTER_TYPE_P (ptype
))
1093 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
1094 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
1096 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
1097 ret
, fold_convert (addtype
, size
));
1098 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1099 ret
= gimple_assign_lhs (g
);
1105 /* Adjust the argument types in NODE to their appropriate vector
1109 simd_clone_adjust (struct cgraph_node
*node
)
1111 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1113 targetm
.simd_clone
.adjust (node
);
1115 tree retval
= simd_clone_adjust_return_type (node
);
1116 ipa_parm_adjustment_vec adjustments
1117 = simd_clone_adjust_argument_types (node
);
1119 push_gimplify_context ();
1121 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
1123 /* Adjust all uses of vector arguments accordingly. Adjust all
1124 return values accordingly. */
1125 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
1126 tree iter1
= make_ssa_name (iter
);
1127 tree iter2
= NULL_TREE
;
1128 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
1129 adjustments
.release ();
1131 /* Initialize the iteration variable. */
1132 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1133 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
1134 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1135 /* Insert the SIMD array and iv initialization at function
1137 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
1139 pop_gimplify_context (NULL
);
1142 basic_block incr_bb
= NULL
;
1143 struct loop
*loop
= NULL
;
1145 /* Create a new BB right before the original exit BB, to hold the
1146 iteration increment and the condition/branch. */
1147 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
))
1149 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
1150 incr_bb
= create_empty_bb (orig_exit
);
1151 incr_bb
->count
= profile_count::zero ();
1152 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1153 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
1154 flag. Set it now to be a FALLTHRU_EDGE. */
1155 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
1156 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
1157 for (unsigned i
= 0;
1158 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
1160 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
1161 redirect_edge_succ (e
, incr_bb
);
1162 incr_bb
->count
+= e
->count ();
1165 else if (node
->simdclone
->inbranch
)
1167 incr_bb
= create_empty_bb (entry_bb
);
1168 incr_bb
->count
= profile_count::zero ();
1169 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1174 make_single_succ_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
1175 gsi
= gsi_last_bb (incr_bb
);
1176 iter2
= make_ssa_name (iter
);
1177 g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
1178 build_int_cst (unsigned_type_node
, 1));
1179 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1181 /* Mostly annotate the loop for the vectorizer (the rest is done
1183 loop
= alloc_loop ();
1184 cfun
->has_force_vectorize_loops
= true;
1185 loop
->safelen
= node
->simdclone
->simdlen
;
1186 loop
->force_vectorize
= true;
1187 loop
->header
= body_bb
;
1190 /* Branch around the body if the mask applies. */
1191 if (node
->simdclone
->inbranch
)
1193 gsi
= gsi_last_bb (loop
->header
);
1195 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
1197 if (node
->simdclone
->mask_mode
!= VOIDmode
)
1200 if (mask_array
== NULL_TREE
)
1202 tree arg
= node
->simdclone
->args
[node
->simdclone
->nargs
1204 mask
= get_or_create_ssa_default_def (cfun
, arg
);
1209 tree maskt
= TREE_TYPE (mask_array
);
1210 int c
= tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt
)));
1211 c
= node
->simdclone
->simdlen
/ (c
+ 1);
1212 int s
= exact_log2 (c
);
1215 tree idx
= make_ssa_name (TREE_TYPE (iter1
));
1216 g
= gimple_build_assign (idx
, RSHIFT_EXPR
, iter1
,
1217 build_int_cst (NULL_TREE
, s
));
1218 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1219 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1220 tree aref
= build4 (ARRAY_REF
,
1221 TREE_TYPE (TREE_TYPE (mask_array
)),
1222 mask_array
, idx
, NULL
, NULL
);
1223 g
= gimple_build_assign (mask
, aref
);
1224 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1225 shift_cnt
= make_ssa_name (TREE_TYPE (iter1
));
1226 g
= gimple_build_assign (shift_cnt
, BIT_AND_EXPR
, iter1
,
1227 build_int_cst (TREE_TYPE (iter1
), c
));
1228 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1230 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1231 RSHIFT_EXPR
, mask
, shift_cnt
);
1232 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1233 mask
= gimple_assign_lhs (g
);
1234 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1236 build_int_cst (TREE_TYPE (mask
), 1));
1237 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1238 mask
= gimple_assign_lhs (g
);
1242 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1243 tree aref
= build4 (ARRAY_REF
,
1244 TREE_TYPE (TREE_TYPE (mask_array
)),
1245 mask_array
, iter1
, NULL
, NULL
);
1246 g
= gimple_build_assign (mask
, aref
);
1247 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1248 int bitsize
= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref
)));
1249 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
1251 aref
= build1 (VIEW_CONVERT_EXPR
,
1252 build_nonstandard_integer_type (bitsize
, 0),
1254 mask
= make_ssa_name (TREE_TYPE (aref
));
1255 g
= gimple_build_assign (mask
, aref
);
1256 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1260 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
1262 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1263 edge e
= make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
1264 e
->probability
= profile_probability::unlikely ().guessed ();
1265 incr_bb
->count
+= e
->count ();
1266 edge fallthru
= FALLTHRU_EDGE (loop
->header
);
1267 fallthru
->flags
= EDGE_FALSE_VALUE
;
1268 fallthru
->probability
= profile_probability::likely ().guessed ();
1271 basic_block latch_bb
= NULL
;
1272 basic_block new_exit_bb
= NULL
;
1274 /* Generate the condition. */
1277 gsi
= gsi_last_bb (incr_bb
);
1278 g
= gimple_build_cond (LT_EXPR
, iter2
,
1279 build_int_cst (unsigned_type_node
,
1280 node
->simdclone
->simdlen
),
1282 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1283 edge e
= split_block (incr_bb
, gsi_stmt (gsi
));
1285 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
1286 loop
->latch
= latch_bb
;
1288 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
1290 edge new_e
= make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
1292 /* FIXME: Do we need to distribute probabilities for the conditional? */
1293 new_e
->probability
= profile_probability::guessed_never ();
1294 /* The successor of incr_bb is already pointing to latch_bb; just
1296 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1297 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
1300 gphi
*phi
= create_phi_node (iter1
, body_bb
);
1301 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
1302 edge latch_edge
= NULL
;
1303 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
1307 latch_edge
= single_succ_edge (latch_bb
);
1308 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1310 /* Generate the new return. */
1311 gsi
= gsi_last_bb (new_exit_bb
);
1313 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
1314 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
1315 retval
= TREE_OPERAND (retval
, 0);
1318 retval
= build1 (VIEW_CONVERT_EXPR
,
1319 TREE_TYPE (TREE_TYPE (node
->decl
)),
1321 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
1322 false, GSI_CONTINUE_LINKING
);
1324 g
= gimple_build_return (retval
);
1325 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1328 /* Handle aligned clauses by replacing default defs of the aligned
1329 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1330 lhs. Handle linear by adding PHIs. */
1331 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
1332 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1333 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
1334 || !is_gimple_reg_type
1335 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1337 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1338 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
1339 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1342 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
1343 gimple_add_tmp_var (iter1
);
1345 gsi
= gsi_after_labels (entry_bb
);
1346 g
= gimple_build_assign (iter1
, orig_arg
);
1347 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1348 gsi
= gsi_after_labels (body_bb
);
1349 g
= gimple_build_assign (orig_arg
, iter1
);
1350 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1352 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1353 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
1354 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1357 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1359 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1360 tree def
= ssa_default_def (cfun
, orig_arg
);
1361 if (def
&& !has_zero_uses (def
))
1363 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
1364 gimple_add_tmp_var (iter1
);
1365 gsi
= gsi_after_labels (entry_bb
);
1366 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
1367 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1368 gsi
= gsi_after_labels (body_bb
);
1369 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
1370 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1373 else if (node
->simdclone
->args
[i
].alignment
1374 && node
->simdclone
->args
[i
].arg_type
1375 == SIMD_CLONE_ARG_TYPE_UNIFORM
1376 && (node
->simdclone
->args
[i
].alignment
1377 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
1378 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1381 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
1382 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1383 tree def
= ssa_default_def (cfun
, orig_arg
);
1384 if (def
&& !has_zero_uses (def
))
1386 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
1387 gimple_seq seq
= NULL
;
1388 bool need_cvt
= false;
1390 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
1392 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
1395 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
1396 gimple_call_set_lhs (g
, t
);
1397 gimple_seq_add_stmt_without_update (&seq
, g
);
1400 t
= make_ssa_name (orig_arg
);
1401 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
1402 gimple_seq_add_stmt_without_update (&seq
, g
);
1404 gsi_insert_seq_on_edge_immediate
1405 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
1407 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1408 node
->create_edge (cgraph_node::get_create (fn
),
1409 call
, entry_bb
->count
);
1411 imm_use_iterator iter
;
1412 use_operand_p use_p
;
1414 tree repl
= gimple_get_lhs (g
);
1415 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1416 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
1419 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1420 SET_USE (use_p
, repl
);
1423 else if ((node
->simdclone
->args
[i
].arg_type
1424 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
1425 || (node
->simdclone
->args
[i
].arg_type
1426 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
1427 || (node
->simdclone
->args
[i
].arg_type
1428 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
1429 || (node
->simdclone
->args
[i
].arg_type
1430 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
1432 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1433 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1434 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
1435 tree def
= NULL_TREE
;
1436 if (TREE_ADDRESSABLE (orig_arg
))
1438 def
= make_ssa_name (TREE_TYPE (orig_arg
));
1439 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1441 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
1442 gsi
= gsi_after_labels (entry_bb
);
1443 g
= gimple_build_assign (def
, orig_arg
);
1444 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1448 def
= ssa_default_def (cfun
, orig_arg
);
1449 if (!def
|| has_zero_uses (def
))
1453 iter1
= make_ssa_name (orig_arg
);
1455 iter2
= make_ssa_name (orig_arg
);
1460 phi
= create_phi_node (iter1
, body_bb
);
1461 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
1464 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1465 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1466 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1467 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1468 ? TREE_TYPE (orig_arg
) : sizetype
;
1469 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1471 gsi
= gsi_last_bb (incr_bb
);
1472 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
1473 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1476 imm_use_iterator iter
;
1477 use_operand_p use_p
;
1479 if (TREE_ADDRESSABLE (orig_arg
))
1481 gsi
= gsi_after_labels (body_bb
);
1482 g
= gimple_build_assign (orig_arg
, iter1
);
1483 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1486 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1487 if (use_stmt
== phi
)
1490 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1491 SET_USE (use_p
, iter1
);
1494 else if (node
->simdclone
->args
[i
].arg_type
1495 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1496 || (node
->simdclone
->args
[i
].arg_type
1497 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
1499 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1500 tree def
= ssa_default_def (cfun
, orig_arg
);
1501 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
1502 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
1503 if (def
&& !has_zero_uses (def
))
1505 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
1506 iter1
= make_ssa_name (orig_arg
);
1508 iter2
= make_ssa_name (orig_arg
);
1509 tree iter3
= make_ssa_name (rtype
);
1510 tree iter4
= make_ssa_name (rtype
);
1511 tree iter5
= incr_bb
? make_ssa_name (rtype
) : NULL_TREE
;
1512 gsi
= gsi_after_labels (entry_bb
);
1514 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
1515 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
1517 tree array
= node
->simdclone
->args
[i
].simd_array
;
1518 TREE_ADDRESSABLE (array
) = 1;
1519 tree ptr
= build_fold_addr_expr (array
);
1520 phi
= create_phi_node (iter1
, body_bb
);
1521 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
1524 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1525 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
1526 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
1527 gsi
= gsi_last_bb (incr_bb
);
1528 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1531 phi
= create_phi_node (iter4
, body_bb
);
1532 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
1535 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
1536 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1537 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1538 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1539 ? TREE_TYPE (iter3
) : sizetype
;
1540 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1542 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
1543 gsi
= gsi_last_bb (incr_bb
);
1544 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1547 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
1548 gsi
= gsi_after_labels (body_bb
);
1549 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1551 imm_use_iterator iter
;
1552 use_operand_p use_p
;
1554 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1555 if (use_stmt
== load
)
1558 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1559 SET_USE (use_p
, iter1
);
1561 if (!TYPE_READONLY (rtype
) && incr_bb
)
1563 tree v
= make_ssa_name (rtype
);
1564 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
1565 size_zero_node
, NULL_TREE
,
1567 gsi
= gsi_after_labels (new_exit_bb
);
1568 g
= gimple_build_assign (v
, aref
);
1569 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1570 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
1571 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1576 calculate_dominance_info (CDI_DOMINATORS
);
1578 add_loop (loop
, loop
->header
->loop_father
);
1579 update_ssa (TODO_update_ssa
);
1584 /* If the function in NODE is tagged as an elemental SIMD function,
1585 create the appropriate SIMD clones. */
1588 expand_simd_clones (struct cgraph_node
*node
)
1590 tree attr
= lookup_attribute ("omp declare simd",
1591 DECL_ATTRIBUTES (node
->decl
));
1592 if (attr
== NULL_TREE
1593 || node
->global
.inlined_to
1594 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
1598 #pragma omp declare simd
1600 in C, there we don't know the argument types at all. */
1601 if (!node
->definition
1602 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
1605 /* Call this before creating clone_info, as it might ggc_collect. */
1606 if (node
->definition
&& node
->has_gimple_body_p ())
1611 /* Start with parsing the "omp declare simd" attribute(s). */
1612 bool inbranch_clause_specified
;
1613 struct cgraph_simd_clone
*clone_info
1614 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
1615 &inbranch_clause_specified
);
1616 if (clone_info
== NULL
)
1619 int orig_simdlen
= clone_info
->simdlen
;
1620 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
1621 /* The target can return 0 (no simd clones should be created),
1622 1 (just one ISA of simd clones should be created) or higher
1623 count of ISA variants. In that case, clone_info is initialized
1624 for the first ISA variant. */
1626 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
1631 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1632 also create one inbranch and one !inbranch clone of it. */
1633 for (int i
= 0; i
< count
* 2; i
++)
1635 struct cgraph_simd_clone
*clone
= clone_info
;
1636 if (inbranch_clause_specified
&& (i
& 1) != 0)
1641 clone
= simd_clone_struct_alloc (clone_info
->nargs
1643 simd_clone_struct_copy (clone
, clone_info
);
1644 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1645 and simd_clone_adjust_argument_types did to the first
1647 clone
->nargs
-= clone_info
->inbranch
;
1648 clone
->simdlen
= orig_simdlen
;
1649 /* And call the target hook again to get the right ISA. */
1650 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
1654 clone
->inbranch
= 1;
1657 /* simd_clone_mangle might fail if such a clone has been created
1659 tree id
= simd_clone_mangle (node
, clone
);
1660 if (id
== NULL_TREE
)
1663 /* Only when we are sure we want to create the clone actually
1664 clone the function (or definitions) or create another
1665 extern FUNCTION_DECL (for prototypes without definitions). */
1666 struct cgraph_node
*n
= simd_clone_create (node
);
1670 n
->simdclone
= clone
;
1671 clone
->origin
= node
;
1672 clone
->next_clone
= NULL
;
1673 if (node
->simd_clones
== NULL
)
1675 clone
->prev_clone
= n
;
1676 node
->simd_clones
= n
;
1680 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
1681 clone
->prev_clone
->simdclone
->next_clone
= n
;
1682 node
->simd_clones
->simdclone
->prev_clone
= n
;
1684 symtab
->change_decl_assembler_name (n
->decl
, id
);
1685 /* And finally adjust the return type, parameters and for
1686 definitions also function body. */
1687 if (node
->definition
)
1688 simd_clone_adjust (n
);
1691 simd_clone_adjust_return_type (n
);
1692 simd_clone_adjust_argument_types (n
);
1696 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
1699 /* Entry point for IPA simd clone creation pass. */
1702 ipa_omp_simd_clone (void)
1704 struct cgraph_node
*node
;
1705 FOR_EACH_FUNCTION (node
)
1706 expand_simd_clones (node
);
1712 const pass_data pass_data_omp_simd_clone
=
1714 SIMPLE_IPA_PASS
, /* type */
1715 "simdclone", /* name */
1716 OPTGROUP_OMP
, /* optinfo_flags */
1717 TV_NONE
, /* tv_id */
1718 ( PROP_ssa
| PROP_cfg
), /* properties_required */
1719 0, /* properties_provided */
1720 0, /* properties_destroyed */
1721 0, /* todo_flags_start */
1722 0, /* todo_flags_finish */
1725 class pass_omp_simd_clone
: public simple_ipa_opt_pass
1728 pass_omp_simd_clone(gcc::context
*ctxt
)
1729 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
1732 /* opt_pass methods: */
1733 virtual bool gate (function
*);
1734 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
1738 pass_omp_simd_clone::gate (function
*)
1740 return targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
;
1745 simple_ipa_opt_pass
*
1746 make_pass_omp_simd_clone (gcc::context
*ctxt
)
1748 return new pass_omp_simd_clone (ctxt
);