1 /* OMP constructs' SIMD clone supporting code.
3 Copyright (C) 2005-2016 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
44 #include "tree-into-ssa.h"
47 #include "symbol-summary.h"
53 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
54 of arguments to reserve space for. */
56 static struct cgraph_simd_clone
*
57 simd_clone_struct_alloc (int nargs
)
59 struct cgraph_simd_clone
*clone_info
;
60 size_t len
= (sizeof (struct cgraph_simd_clone
)
61 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
62 clone_info
= (struct cgraph_simd_clone
*)
63 ggc_internal_cleared_alloc (len
);
67 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
70 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
71 struct cgraph_simd_clone
*from
)
73 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
74 + ((from
->nargs
- from
->inbranch
)
75 * sizeof (struct cgraph_simd_clone_arg
))));
78 /* Return vector of parameter types of function FNDECL. This uses
79 TYPE_ARG_TYPES if available, otherwise falls back to types of
80 DECL_ARGUMENTS types. */
83 simd_clone_vector_of_formal_parm_types (tree fndecl
)
85 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
86 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
87 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
90 FOR_EACH_VEC_ELT (args
, i
, arg
)
91 args
[i
] = TREE_TYPE (args
[i
]);
95 /* Given a simd function in NODE, extract the simd specific
96 information from the OMP clauses passed in CLAUSES, and return
97 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
98 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
99 otherwise set to FALSE. */
101 static struct cgraph_simd_clone
*
102 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
103 bool *inbranch_specified
)
105 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
108 *inbranch_specified
= false;
111 if (n
> 0 && args
.last () == void_type_node
)
114 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
115 be cloned have a distinctive artificial label in addition to "omp
119 && lookup_attribute ("cilk simd function",
120 DECL_ATTRIBUTES (node
->decl
)));
122 /* Allocate one more than needed just in case this is an in-branch
123 clone which will require a mask argument. */
124 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
125 clone_info
->nargs
= n
;
126 clone_info
->cilk_elemental
= cilk_clone
;
133 clauses
= TREE_VALUE (clauses
);
134 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
137 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
139 switch (OMP_CLAUSE_CODE (t
))
141 case OMP_CLAUSE_INBRANCH
:
142 clone_info
->inbranch
= 1;
143 *inbranch_specified
= true;
145 case OMP_CLAUSE_NOTINBRANCH
:
146 clone_info
->inbranch
= 0;
147 *inbranch_specified
= true;
149 case OMP_CLAUSE_SIMDLEN
:
151 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
153 case OMP_CLAUSE_LINEAR
:
155 tree decl
= OMP_CLAUSE_DECL (t
);
156 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
157 int argno
= TREE_INT_CST_LOW (decl
);
158 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
160 enum cgraph_simd_clone_arg_type arg_type
;
161 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
162 switch (OMP_CLAUSE_LINEAR_KIND (t
))
164 case OMP_CLAUSE_LINEAR_REF
:
166 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
168 case OMP_CLAUSE_LINEAR_UVAL
:
170 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
172 case OMP_CLAUSE_LINEAR_VAL
:
173 case OMP_CLAUSE_LINEAR_DEFAULT
:
175 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
181 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
182 clone_info
->args
[argno
].arg_type
= arg_type
;
183 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
184 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
185 && clone_info
->args
[argno
].linear_step
< n
);
189 if (POINTER_TYPE_P (args
[argno
]))
190 step
= fold_convert (ssizetype
, step
);
191 if (!tree_fits_shwi_p (step
))
193 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
194 "ignoring large linear step");
198 else if (integer_zerop (step
))
200 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
201 "ignoring zero linear step");
207 enum cgraph_simd_clone_arg_type arg_type
;
208 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
209 switch (OMP_CLAUSE_LINEAR_KIND (t
))
211 case OMP_CLAUSE_LINEAR_REF
:
213 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
215 case OMP_CLAUSE_LINEAR_UVAL
:
217 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
219 case OMP_CLAUSE_LINEAR_VAL
:
220 case OMP_CLAUSE_LINEAR_DEFAULT
:
222 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
228 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
229 clone_info
->args
[argno
].arg_type
= arg_type
;
230 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
235 case OMP_CLAUSE_UNIFORM
:
237 tree decl
= OMP_CLAUSE_DECL (t
);
238 int argno
= tree_to_uhwi (decl
);
239 clone_info
->args
[argno
].arg_type
240 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
243 case OMP_CLAUSE_ALIGNED
:
245 tree decl
= OMP_CLAUSE_DECL (t
);
246 int argno
= tree_to_uhwi (decl
);
247 clone_info
->args
[argno
].alignment
248 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
259 /* Given a SIMD clone in NODE, calculate the characteristic data
260 type and return the coresponding type. The characteristic data
261 type is computed as described in the Intel Vector ABI. */
264 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
265 struct cgraph_simd_clone
*clone_info
)
267 tree type
= integer_type_node
;
268 tree fndecl
= node
->decl
;
270 /* a) For non-void function, the characteristic data type is the
272 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
273 type
= TREE_TYPE (TREE_TYPE (fndecl
));
275 /* b) If the function has any non-uniform, non-linear parameters,
276 then the characteristic data type is the type of the first
280 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
281 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
282 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
290 /* c) If the characteristic data type determined by a) or b) above
291 is struct, union, or class type which is pass-by-value (except
292 for the type that maps to the built-in complex data type), the
293 characteristic data type is int. */
294 if (RECORD_OR_UNION_TYPE_P (type
)
295 && !aggregate_value_p (type
, NULL
)
296 && TREE_CODE (type
) != COMPLEX_TYPE
)
297 return integer_type_node
;
299 /* d) If none of the above three classes is applicable, the
300 characteristic data type is int. */
304 /* e) For Intel Xeon Phi native and offload compilation, if the
305 resulting characteristic data type is 8-bit or 16-bit integer
306 data type, the characteristic data type is int. */
307 /* Well, we don't handle Xeon Phi yet. */
311 simd_clone_mangle (struct cgraph_node
*node
,
312 struct cgraph_simd_clone
*clone_info
)
314 char vecsize_mangle
= clone_info
->vecsize_mangle
;
315 char mask
= clone_info
->inbranch
? 'M' : 'N';
316 unsigned int simdlen
= clone_info
->simdlen
;
320 gcc_assert (vecsize_mangle
&& simdlen
);
322 pp_string (&pp
, "_ZGV");
323 pp_character (&pp
, vecsize_mangle
);
324 pp_character (&pp
, mask
);
325 pp_decimal_int (&pp
, simdlen
);
327 for (n
= 0; n
< clone_info
->nargs
; ++n
)
329 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
331 switch (arg
.arg_type
)
333 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
334 pp_character (&pp
, 'u');
336 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
337 pp_character (&pp
, 'l');
339 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
340 pp_character (&pp
, 'R');
342 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
343 pp_character (&pp
, 'L');
345 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
346 pp_character (&pp
, 'U');
349 gcc_assert (arg
.linear_step
!= 0);
350 if (arg
.linear_step
> 1)
351 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
352 else if (arg
.linear_step
< 0)
354 pp_character (&pp
, 'n');
355 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
359 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
360 pp_string (&pp
, "ls");
361 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
363 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
364 pp_string (&pp
, "Rs");
365 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
367 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
368 pp_string (&pp
, "Ls");
369 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
371 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
372 pp_string (&pp
, "Us");
373 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
376 pp_character (&pp
, 'v');
380 pp_character (&pp
, 'a');
381 pp_decimal_int (&pp
, arg
.alignment
);
386 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
389 pp_string (&pp
, str
);
390 str
= pp_formatted_text (&pp
);
392 /* If there already is a SIMD clone with the same mangled name, don't
393 add another one. This can happen e.g. for
394 #pragma omp declare simd
395 #pragma omp declare simd simdlen(8)
397 if the simdlen is assumed to be 8 for the first one, etc. */
398 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
399 clone
= clone
->simdclone
->next_clone
)
400 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
404 return get_identifier (str
);
407 /* Create a simd clone of OLD_NODE and return it. */
409 static struct cgraph_node
*
410 simd_clone_create (struct cgraph_node
*old_node
)
412 struct cgraph_node
*new_node
;
413 if (old_node
->definition
)
415 if (!old_node
->has_gimple_body_p ())
417 old_node
->get_body ();
418 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
424 tree old_decl
= old_node
->decl
;
425 tree new_decl
= copy_node (old_node
->decl
);
426 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
427 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
428 SET_DECL_RTL (new_decl
, NULL
);
429 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
430 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
431 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
432 if (old_node
->in_other_partition
)
433 new_node
->in_other_partition
= 1;
435 if (new_node
== NULL
)
438 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
439 DECL_COMDAT (new_node
->decl
) = DECL_COMDAT (old_node
->decl
);
440 DECL_WEAK (new_node
->decl
) = DECL_WEAK (old_node
->decl
);
441 DECL_EXTERNAL (new_node
->decl
) = DECL_EXTERNAL (old_node
->decl
);
442 DECL_VISIBILITY_SPECIFIED (new_node
->decl
)
443 = DECL_VISIBILITY_SPECIFIED (old_node
->decl
);
444 DECL_VISIBILITY (new_node
->decl
) = DECL_VISIBILITY (old_node
->decl
);
445 DECL_DLLIMPORT_P (new_node
->decl
) = DECL_DLLIMPORT_P (old_node
->decl
);
446 if (DECL_ONE_ONLY (old_node
->decl
))
447 make_decl_one_only (new_node
->decl
, DECL_ASSEMBLER_NAME (new_node
->decl
));
449 /* The method cgraph_version_clone_with_body () will force the new
450 symbol local. Undo this, and inherit external visibility from
452 new_node
->local
.local
= old_node
->local
.local
;
453 new_node
->externally_visible
= old_node
->externally_visible
;
458 /* Adjust the return type of the given function to its appropriate
459 vector counterpart. Returns a simd array to be used throughout the
460 function as a return value. */
463 simd_clone_adjust_return_type (struct cgraph_node
*node
)
465 tree fndecl
= node
->decl
;
466 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
470 /* Adjust the function return type. */
471 if (orig_rettype
== void_type_node
)
473 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
474 t
= TREE_TYPE (TREE_TYPE (fndecl
));
475 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
476 veclen
= node
->simdclone
->vecsize_int
;
478 veclen
= node
->simdclone
->vecsize_float
;
479 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
480 if (veclen
> node
->simdclone
->simdlen
)
481 veclen
= node
->simdclone
->simdlen
;
482 if (POINTER_TYPE_P (t
))
483 t
= pointer_sized_int_node
;
484 if (veclen
== node
->simdclone
->simdlen
)
485 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
488 t
= build_vector_type (t
, veclen
);
489 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
491 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
492 if (!node
->definition
)
495 t
= DECL_RESULT (fndecl
);
496 /* Adjust the DECL_RESULT. */
497 gcc_assert (TREE_TYPE (t
) != void_type_node
);
498 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
501 tree atype
= build_array_type_nelts (orig_rettype
,
502 node
->simdclone
->simdlen
);
503 if (veclen
!= node
->simdclone
->simdlen
)
504 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
506 /* Set up a SIMD array to use as the return value. */
507 tree retval
= create_tmp_var_raw (atype
, "retval");
508 gimple_add_tmp_var (retval
);
512 /* Each vector argument has a corresponding array to be used locally
513 as part of the eventual loop. Create such temporary array and
516 PREFIX is the prefix to be used for the temporary.
518 TYPE is the inner element type.
520 SIMDLEN is the number of elements. */
523 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
525 tree atype
= build_array_type_nelts (type
, simdlen
);
526 tree avar
= create_tmp_var_raw (atype
, prefix
);
527 gimple_add_tmp_var (avar
);
531 /* Modify the function argument types to their corresponding vector
532 counterparts if appropriate. Also, create one array for each simd
533 argument to be used locally when using the function arguments as
536 NODE is the function whose arguments are to be adjusted.
538 Returns an adjustment vector that will be filled describing how the
539 argument types will be adjusted. */
541 static ipa_parm_adjustment_vec
542 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
545 ipa_parm_adjustment_vec adjustments
;
547 if (node
->definition
)
548 args
= ipa_get_vector_of_formal_parms (node
->decl
);
550 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
551 adjustments
.create (args
.length ());
552 unsigned i
, j
, veclen
;
553 struct ipa_parm_adjustment adj
;
554 struct cgraph_simd_clone
*sc
= node
->simdclone
;
556 for (i
= 0; i
< sc
->nargs
; ++i
)
558 memset (&adj
, 0, sizeof (adj
));
560 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
564 sc
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
565 sc
->args
[i
].orig_type
= parm_type
;
567 switch (sc
->args
[i
].arg_type
)
570 /* No adjustment necessary for scalar arguments. */
571 adj
.op
= IPA_PARM_OP_COPY
;
573 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
574 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
575 if (node
->definition
)
576 sc
->args
[i
].simd_array
577 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
578 TREE_TYPE (parm_type
),
580 adj
.op
= IPA_PARM_OP_COPY
;
582 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
583 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
584 case SIMD_CLONE_ARG_TYPE_VECTOR
:
585 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
586 veclen
= sc
->vecsize_int
;
588 veclen
= sc
->vecsize_float
;
589 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
590 if (veclen
> sc
->simdlen
)
591 veclen
= sc
->simdlen
;
592 adj
.arg_prefix
= "simd";
593 if (POINTER_TYPE_P (parm_type
))
594 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
596 adj
.type
= build_vector_type (parm_type
, veclen
);
597 sc
->args
[i
].vector_type
= adj
.type
;
598 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
600 adjustments
.safe_push (adj
);
603 memset (&adj
, 0, sizeof (adj
));
604 adj
.op
= IPA_PARM_OP_NEW
;
605 adj
.arg_prefix
= "simd";
607 adj
.type
= sc
->args
[i
].vector_type
;
611 if (node
->definition
)
612 sc
->args
[i
].simd_array
613 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
614 parm_type
, sc
->simdlen
);
616 adjustments
.safe_push (adj
);
621 tree base_type
= simd_clone_compute_base_data_type (sc
->origin
, sc
);
623 memset (&adj
, 0, sizeof (adj
));
624 adj
.op
= IPA_PARM_OP_NEW
;
625 adj
.arg_prefix
= "mask";
628 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
629 veclen
= sc
->vecsize_int
;
631 veclen
= sc
->vecsize_float
;
632 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
633 if (veclen
> sc
->simdlen
)
634 veclen
= sc
->simdlen
;
635 if (sc
->mask_mode
!= VOIDmode
)
637 = lang_hooks
.types
.type_for_mode (sc
->mask_mode
, 1);
638 else if (POINTER_TYPE_P (base_type
))
639 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
641 adj
.type
= build_vector_type (base_type
, veclen
);
642 adjustments
.safe_push (adj
);
644 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
645 adjustments
.safe_push (adj
);
647 /* We have previously allocated one extra entry for the mask. Use
650 if (sc
->mask_mode
!= VOIDmode
)
651 base_type
= boolean_type_node
;
652 if (node
->definition
)
655 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
656 if (sc
->mask_mode
== VOIDmode
)
657 sc
->args
[i
].simd_array
658 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
659 else if (veclen
< sc
->simdlen
)
660 sc
->args
[i
].simd_array
661 = create_tmp_simd_array ("mask", adj
.type
, sc
->simdlen
/ veclen
);
663 sc
->args
[i
].simd_array
= NULL_TREE
;
665 sc
->args
[i
].orig_type
= base_type
;
666 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
669 if (node
->definition
)
670 ipa_modify_formal_parameters (node
->decl
, adjustments
);
673 tree new_arg_types
= NULL_TREE
, new_reversed
;
674 bool last_parm_void
= false;
675 if (args
.length () > 0 && args
.last () == void_type_node
)
676 last_parm_void
= true;
678 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
679 j
= adjustments
.length ();
680 for (i
= 0; i
< j
; i
++)
682 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
684 if (adj
->op
== IPA_PARM_OP_COPY
)
685 ptype
= args
[adj
->base_index
];
688 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
690 new_reversed
= nreverse (new_arg_types
);
694 TREE_CHAIN (new_arg_types
) = void_list_node
;
696 new_reversed
= void_list_node
;
699 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
700 TYPE_ARG_TYPES (new_type
) = new_reversed
;
701 TREE_TYPE (node
->decl
) = new_type
;
703 adjustments
.release ();
709 /* Initialize and copy the function arguments in NODE to their
710 corresponding local simd arrays. Returns a fresh gimple_seq with
711 the instruction sequence generated. */
714 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
715 ipa_parm_adjustment_vec adjustments
)
717 gimple_seq seq
= NULL
;
718 unsigned i
= 0, j
= 0, k
;
720 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
722 arg
= DECL_CHAIN (arg
), i
++, j
++)
724 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
725 || POINTER_TYPE_P (TREE_TYPE (arg
)))
728 node
->simdclone
->args
[i
].vector_arg
= arg
;
730 tree array
= node
->simdclone
->args
[i
].simd_array
;
731 if (node
->simdclone
->mask_mode
!= VOIDmode
732 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_MASK
)
734 if (array
== NULL_TREE
)
737 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array
))));
738 for (k
= 0; k
<= l
; k
++)
742 arg
= DECL_CHAIN (arg
);
745 tree t
= build4 (ARRAY_REF
, TREE_TYPE (TREE_TYPE (array
)),
746 array
, size_int (k
), NULL
, NULL
);
747 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
748 gimplify_and_add (t
, &seq
);
752 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
754 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
755 tree ptr
= build_fold_addr_expr (array
);
756 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
757 build_int_cst (ptype
, 0));
758 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
759 gimplify_and_add (t
, &seq
);
763 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
764 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
765 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
767 tree ptr
= build_fold_addr_expr (array
);
771 arg
= DECL_CHAIN (arg
);
775 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
776 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
777 build_int_cst (ptype
, k
* elemsize
));
778 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
779 gimplify_and_add (t
, &seq
);
786 /* Callback info for ipa_simd_modify_stmt_ops below. */
788 struct modify_stmt_info
{
789 ipa_parm_adjustment_vec adjustments
;
791 /* True if the parent statement was modified by
792 ipa_simd_modify_stmt_ops. */
796 /* Callback for walk_gimple_op.
798 Adjust operands from a given statement as specified in the
799 adjustments vector in the callback data. */
802 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
804 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
805 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
807 if (TREE_CODE (*tp
) == ADDR_EXPR
)
808 tp
= &TREE_OPERAND (*tp
, 0);
809 struct ipa_parm_adjustment
*cand
= NULL
;
810 if (TREE_CODE (*tp
) == PARM_DECL
)
811 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
818 tree repl
= NULL_TREE
;
820 repl
= unshare_expr (cand
->new_decl
);
826 bool modified
= info
->modified
;
827 info
->modified
= false;
828 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
831 info
->modified
= modified
;
834 info
->modified
= modified
;
843 repl
= build_fold_addr_expr (repl
);
845 if (is_gimple_debug (info
->stmt
))
847 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
848 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
849 DECL_ARTIFICIAL (vexpr
) = 1;
850 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
851 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
856 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
857 repl
= gimple_assign_lhs (stmt
);
859 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
860 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
863 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
865 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
871 info
->modified
= true;
875 /* Traverse the function body and perform all modifications as
876 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
877 modified such that the replacement/reduction value will now be an
878 offset into the corresponding simd_array.
880 This function will replace all function argument uses with their
881 corresponding simd array elements, and ajust the return values
885 ipa_simd_modify_function_body (struct cgraph_node
*node
,
886 ipa_parm_adjustment_vec adjustments
,
887 tree retval_array
, tree iter
)
890 unsigned int i
, j
, l
;
892 /* Re-use the adjustments array, but this time use it to replace
893 every function argument use to an offset into the corresponding
895 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
897 if (!node
->simdclone
->args
[i
].vector_arg
)
900 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
901 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
902 adjustments
[j
].new_decl
905 node
->simdclone
->args
[i
].simd_array
,
907 NULL_TREE
, NULL_TREE
);
908 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
909 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
910 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
913 l
= adjustments
.length ();
914 for (i
= 1; i
< num_ssa_names
; i
++)
916 tree name
= ssa_name (i
);
918 && SSA_NAME_VAR (name
)
919 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
921 for (j
= 0; j
< l
; j
++)
922 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
923 && adjustments
[j
].new_decl
)
926 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
929 = copy_var_decl (adjustments
[j
].base
,
930 DECL_NAME (adjustments
[j
].base
),
931 TREE_TYPE (adjustments
[j
].base
));
932 adjustments
[j
].new_ssa_base
= base_var
;
935 base_var
= adjustments
[j
].new_ssa_base
;
936 if (SSA_NAME_IS_DEFAULT_DEF (name
))
938 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
939 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
940 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
941 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
942 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
943 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
944 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
945 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
948 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
953 struct modify_stmt_info info
;
954 info
.adjustments
= adjustments
;
956 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
958 gimple_stmt_iterator gsi
;
960 gsi
= gsi_start_bb (bb
);
961 while (!gsi_end_p (gsi
))
963 gimple
*stmt
= gsi_stmt (gsi
);
965 struct walk_stmt_info wi
;
967 memset (&wi
, 0, sizeof (wi
));
968 info
.modified
= false;
970 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
972 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
974 tree retval
= gimple_return_retval (return_stmt
);
977 gsi_remove (&gsi
, true);
981 /* Replace `return foo' with `retval_array[iter] = foo'. */
982 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
983 retval_array
, iter
, NULL
, NULL
);
984 stmt
= gimple_build_assign (ref
, retval
);
985 gsi_replace (&gsi
, stmt
, true);
986 info
.modified
= true;
992 if (maybe_clean_eh_stmt (stmt
))
993 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
1000 /* Helper function of simd_clone_adjust, return linear step addend
1004 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
1005 tree addtype
, basic_block entry_bb
)
1007 tree ptype
= NULL_TREE
;
1008 switch (node
->simdclone
->args
[i
].arg_type
)
1010 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
1011 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
1012 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
1013 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
1014 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
1015 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
1016 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
1017 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
1019 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
1020 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
1021 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
1027 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
1028 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
1029 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
1030 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1033 if (is_gimple_reg (arg
))
1034 ret
= get_or_create_ssa_default_def (cfun
, arg
);
1037 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
1038 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1039 ret
= gimple_assign_lhs (g
);
1041 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
1043 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
1044 build_simple_mem_ref (ret
));
1045 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1046 ret
= gimple_assign_lhs (g
);
1048 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
1050 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
1051 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1052 ret
= gimple_assign_lhs (g
);
1054 if (POINTER_TYPE_P (ptype
))
1056 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
1057 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
1059 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
1060 ret
, fold_convert (addtype
, size
));
1061 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1062 ret
= gimple_assign_lhs (g
);
1068 /* Adjust the argument types in NODE to their appropriate vector
1072 simd_clone_adjust (struct cgraph_node
*node
)
1074 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1076 targetm
.simd_clone
.adjust (node
);
1078 tree retval
= simd_clone_adjust_return_type (node
);
1079 ipa_parm_adjustment_vec adjustments
1080 = simd_clone_adjust_argument_types (node
);
1082 push_gimplify_context ();
1084 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
1086 /* Adjust all uses of vector arguments accordingly. Adjust all
1087 return values accordingly. */
1088 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
1089 tree iter1
= make_ssa_name (iter
);
1090 tree iter2
= make_ssa_name (iter
);
1091 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
1092 adjustments
.release ();
1094 /* Initialize the iteration variable. */
1095 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1096 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
1097 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1098 /* Insert the SIMD array and iv initialization at function
1100 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
1102 pop_gimplify_context (NULL
);
1104 /* Create a new BB right before the original exit BB, to hold the
1105 iteration increment and the condition/branch. */
1106 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
1107 basic_block incr_bb
= create_empty_bb (orig_exit
);
1108 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1109 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
1110 flag. Set it now to be a FALLTHRU_EDGE. */
1111 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
1112 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
1113 for (unsigned i
= 0;
1114 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
1116 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
1117 redirect_edge_succ (e
, incr_bb
);
1119 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
1120 e
->probability
= REG_BR_PROB_BASE
;
1121 gsi
= gsi_last_bb (incr_bb
);
1122 gimple
*g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
1123 build_int_cst (unsigned_type_node
, 1));
1124 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1126 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
1127 struct loop
*loop
= alloc_loop ();
1128 cfun
->has_force_vectorize_loops
= true;
1129 loop
->safelen
= node
->simdclone
->simdlen
;
1130 loop
->force_vectorize
= true;
1131 loop
->header
= body_bb
;
1133 /* Branch around the body if the mask applies. */
1134 if (node
->simdclone
->inbranch
)
1136 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
1138 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
1140 if (node
->simdclone
->mask_mode
!= VOIDmode
)
1143 if (mask_array
== NULL_TREE
)
1145 tree arg
= node
->simdclone
->args
[node
->simdclone
->nargs
1147 mask
= get_or_create_ssa_default_def (cfun
, arg
);
1152 tree maskt
= TREE_TYPE (mask_array
);
1153 int c
= tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt
)));
1154 c
= node
->simdclone
->simdlen
/ (c
+ 1);
1155 int s
= exact_log2 (c
);
1158 tree idx
= make_ssa_name (TREE_TYPE (iter1
));
1159 g
= gimple_build_assign (idx
, RSHIFT_EXPR
, iter1
,
1160 build_int_cst (NULL_TREE
, s
));
1161 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1162 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1163 tree aref
= build4 (ARRAY_REF
,
1164 TREE_TYPE (TREE_TYPE (mask_array
)),
1165 mask_array
, idx
, NULL
, NULL
);
1166 g
= gimple_build_assign (mask
, aref
);
1167 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1168 shift_cnt
= make_ssa_name (TREE_TYPE (iter1
));
1169 g
= gimple_build_assign (shift_cnt
, BIT_AND_EXPR
, iter1
,
1170 build_int_cst (TREE_TYPE (iter1
), c
));
1171 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1173 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1174 RSHIFT_EXPR
, mask
, shift_cnt
);
1175 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1176 mask
= gimple_assign_lhs (g
);
1177 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1179 build_int_cst (TREE_TYPE (mask
), 1));
1180 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1181 mask
= gimple_assign_lhs (g
);
1185 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1186 tree aref
= build4 (ARRAY_REF
,
1187 TREE_TYPE (TREE_TYPE (mask_array
)),
1188 mask_array
, iter1
, NULL
, NULL
);
1189 g
= gimple_build_assign (mask
, aref
);
1190 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1191 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
1192 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
1194 aref
= build1 (VIEW_CONVERT_EXPR
,
1195 build_nonstandard_integer_type (bitsize
, 0),
1197 mask
= make_ssa_name (TREE_TYPE (aref
));
1198 g
= gimple_build_assign (mask
, aref
);
1199 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1203 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
1205 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1206 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
1207 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
1210 /* Generate the condition. */
1211 g
= gimple_build_cond (LT_EXPR
,
1213 build_int_cst (unsigned_type_node
,
1214 node
->simdclone
->simdlen
),
1216 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1217 e
= split_block (incr_bb
, gsi_stmt (gsi
));
1218 basic_block latch_bb
= e
->dest
;
1219 basic_block new_exit_bb
;
1220 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
1221 loop
->latch
= latch_bb
;
1223 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
1225 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
1226 /* The successor of incr_bb is already pointing to latch_bb; just
1228 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1229 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
1231 gphi
*phi
= create_phi_node (iter1
, body_bb
);
1232 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
1233 edge latch_edge
= single_succ_edge (latch_bb
);
1234 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
1236 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1238 /* Generate the new return. */
1239 gsi
= gsi_last_bb (new_exit_bb
);
1241 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
1242 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
1243 retval
= TREE_OPERAND (retval
, 0);
1246 retval
= build1 (VIEW_CONVERT_EXPR
,
1247 TREE_TYPE (TREE_TYPE (node
->decl
)),
1249 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
1250 false, GSI_CONTINUE_LINKING
);
1252 g
= gimple_build_return (retval
);
1253 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1255 /* Handle aligned clauses by replacing default defs of the aligned
1256 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1257 lhs. Handle linear by adding PHIs. */
1258 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
1259 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1260 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
1261 || !is_gimple_reg_type
1262 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1264 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1265 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
1266 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1269 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
1270 gimple_add_tmp_var (iter1
);
1272 gsi
= gsi_after_labels (entry_bb
);
1273 g
= gimple_build_assign (iter1
, orig_arg
);
1274 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1275 gsi
= gsi_after_labels (body_bb
);
1276 g
= gimple_build_assign (orig_arg
, iter1
);
1277 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1279 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1280 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
1281 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1284 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1286 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1287 tree def
= ssa_default_def (cfun
, orig_arg
);
1288 if (def
&& !has_zero_uses (def
))
1290 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
1291 gimple_add_tmp_var (iter1
);
1292 gsi
= gsi_after_labels (entry_bb
);
1293 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
1294 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1295 gsi
= gsi_after_labels (body_bb
);
1296 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
1297 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1300 else if (node
->simdclone
->args
[i
].alignment
1301 && node
->simdclone
->args
[i
].arg_type
1302 == SIMD_CLONE_ARG_TYPE_UNIFORM
1303 && (node
->simdclone
->args
[i
].alignment
1304 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
1305 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1308 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
1309 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1310 tree def
= ssa_default_def (cfun
, orig_arg
);
1311 if (def
&& !has_zero_uses (def
))
1313 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
1314 gimple_seq seq
= NULL
;
1315 bool need_cvt
= false;
1317 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
1319 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
1322 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
1323 gimple_call_set_lhs (g
, t
);
1324 gimple_seq_add_stmt_without_update (&seq
, g
);
1327 t
= make_ssa_name (orig_arg
);
1328 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
1329 gimple_seq_add_stmt_without_update (&seq
, g
);
1331 gsi_insert_seq_on_edge_immediate
1332 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
1334 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1335 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
1337 node
->create_edge (cgraph_node::get_create (fn
),
1338 call
, entry_bb
->count
, freq
);
1340 imm_use_iterator iter
;
1341 use_operand_p use_p
;
1343 tree repl
= gimple_get_lhs (g
);
1344 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1345 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
1348 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1349 SET_USE (use_p
, repl
);
1352 else if ((node
->simdclone
->args
[i
].arg_type
1353 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
1354 || (node
->simdclone
->args
[i
].arg_type
1355 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
1356 || (node
->simdclone
->args
[i
].arg_type
1357 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
1358 || (node
->simdclone
->args
[i
].arg_type
1359 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
1361 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1362 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1363 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
1364 tree def
= NULL_TREE
;
1365 if (TREE_ADDRESSABLE (orig_arg
))
1367 def
= make_ssa_name (TREE_TYPE (orig_arg
));
1368 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1369 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
1370 gsi
= gsi_after_labels (entry_bb
);
1371 g
= gimple_build_assign (def
, orig_arg
);
1372 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1376 def
= ssa_default_def (cfun
, orig_arg
);
1377 if (!def
|| has_zero_uses (def
))
1381 iter1
= make_ssa_name (orig_arg
);
1382 iter2
= make_ssa_name (orig_arg
);
1387 phi
= create_phi_node (iter1
, body_bb
);
1388 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
1389 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1390 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1391 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1392 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1393 ? TREE_TYPE (orig_arg
) : sizetype
;
1394 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1396 gsi
= gsi_last_bb (incr_bb
);
1397 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
1398 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1400 imm_use_iterator iter
;
1401 use_operand_p use_p
;
1403 if (TREE_ADDRESSABLE (orig_arg
))
1405 gsi
= gsi_after_labels (body_bb
);
1406 g
= gimple_build_assign (orig_arg
, iter1
);
1407 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1410 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1411 if (use_stmt
== phi
)
1414 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1415 SET_USE (use_p
, iter1
);
1418 else if (node
->simdclone
->args
[i
].arg_type
1419 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1420 || (node
->simdclone
->args
[i
].arg_type
1421 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
1423 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1424 tree def
= ssa_default_def (cfun
, orig_arg
);
1425 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
1426 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
1427 if (def
&& !has_zero_uses (def
))
1429 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
1430 iter1
= make_ssa_name (orig_arg
);
1431 iter2
= make_ssa_name (orig_arg
);
1432 tree iter3
= make_ssa_name (rtype
);
1433 tree iter4
= make_ssa_name (rtype
);
1434 tree iter5
= make_ssa_name (rtype
);
1435 gsi
= gsi_after_labels (entry_bb
);
1437 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
1438 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
1440 tree array
= node
->simdclone
->args
[i
].simd_array
;
1441 TREE_ADDRESSABLE (array
) = 1;
1442 tree ptr
= build_fold_addr_expr (array
);
1443 phi
= create_phi_node (iter1
, body_bb
);
1444 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
1445 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1446 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
1447 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
1448 gsi
= gsi_last_bb (incr_bb
);
1449 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1451 phi
= create_phi_node (iter4
, body_bb
);
1452 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
1453 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
1454 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1455 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1456 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1457 ? TREE_TYPE (iter3
) : sizetype
;
1458 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1460 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
1461 gsi
= gsi_last_bb (incr_bb
);
1462 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1464 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
1465 gsi
= gsi_after_labels (body_bb
);
1466 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1468 imm_use_iterator iter
;
1469 use_operand_p use_p
;
1471 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1472 if (use_stmt
== load
)
1475 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1476 SET_USE (use_p
, iter1
);
1478 if (!TYPE_READONLY (rtype
))
1480 tree v
= make_ssa_name (rtype
);
1481 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
1482 size_zero_node
, NULL_TREE
,
1484 gsi
= gsi_after_labels (new_exit_bb
);
1485 g
= gimple_build_assign (v
, aref
);
1486 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1487 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
1488 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1493 calculate_dominance_info (CDI_DOMINATORS
);
1494 add_loop (loop
, loop
->header
->loop_father
);
1495 update_ssa (TODO_update_ssa
);
1500 /* If the function in NODE is tagged as an elemental SIMD function,
1501 create the appropriate SIMD clones. */
1504 expand_simd_clones (struct cgraph_node
*node
)
1506 tree attr
= lookup_attribute ("omp declare simd",
1507 DECL_ATTRIBUTES (node
->decl
));
1508 if (attr
== NULL_TREE
1509 || node
->global
.inlined_to
1510 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
1514 #pragma omp declare simd
1516 in C, there we don't know the argument types at all. */
1517 if (!node
->definition
1518 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
1521 /* Call this before creating clone_info, as it might ggc_collect. */
1522 if (node
->definition
&& node
->has_gimple_body_p ())
1527 /* Start with parsing the "omp declare simd" attribute(s). */
1528 bool inbranch_clause_specified
;
1529 struct cgraph_simd_clone
*clone_info
1530 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
1531 &inbranch_clause_specified
);
1532 if (clone_info
== NULL
)
1535 int orig_simdlen
= clone_info
->simdlen
;
1536 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
1537 /* The target can return 0 (no simd clones should be created),
1538 1 (just one ISA of simd clones should be created) or higher
1539 count of ISA variants. In that case, clone_info is initialized
1540 for the first ISA variant. */
1542 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
1547 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1548 also create one inbranch and one !inbranch clone of it. */
1549 for (int i
= 0; i
< count
* 2; i
++)
1551 struct cgraph_simd_clone
*clone
= clone_info
;
1552 if (inbranch_clause_specified
&& (i
& 1) != 0)
1557 clone
= simd_clone_struct_alloc (clone_info
->nargs
1559 simd_clone_struct_copy (clone
, clone_info
);
1560 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1561 and simd_clone_adjust_argument_types did to the first
1563 clone
->nargs
-= clone_info
->inbranch
;
1564 clone
->simdlen
= orig_simdlen
;
1565 /* And call the target hook again to get the right ISA. */
1566 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
1570 clone
->inbranch
= 1;
1573 /* simd_clone_mangle might fail if such a clone has been created
1575 tree id
= simd_clone_mangle (node
, clone
);
1576 if (id
== NULL_TREE
)
1579 /* Only when we are sure we want to create the clone actually
1580 clone the function (or definitions) or create another
1581 extern FUNCTION_DECL (for prototypes without definitions). */
1582 struct cgraph_node
*n
= simd_clone_create (node
);
1586 n
->simdclone
= clone
;
1587 clone
->origin
= node
;
1588 clone
->next_clone
= NULL
;
1589 if (node
->simd_clones
== NULL
)
1591 clone
->prev_clone
= n
;
1592 node
->simd_clones
= n
;
1596 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
1597 clone
->prev_clone
->simdclone
->next_clone
= n
;
1598 node
->simd_clones
->simdclone
->prev_clone
= n
;
1600 symtab
->change_decl_assembler_name (n
->decl
, id
);
1601 /* And finally adjust the return type, parameters and for
1602 definitions also function body. */
1603 if (node
->definition
)
1604 simd_clone_adjust (n
);
1607 simd_clone_adjust_return_type (n
);
1608 simd_clone_adjust_argument_types (n
);
1612 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
1615 /* Entry point for IPA simd clone creation pass. */
1618 ipa_omp_simd_clone (void)
1620 struct cgraph_node
*node
;
1621 FOR_EACH_FUNCTION (node
)
1622 expand_simd_clones (node
);
1628 const pass_data pass_data_omp_simd_clone
=
1630 SIMPLE_IPA_PASS
, /* type */
1631 "simdclone", /* name */
1632 OPTGROUP_NONE
, /* optinfo_flags */
1633 TV_NONE
, /* tv_id */
1634 ( PROP_ssa
| PROP_cfg
), /* properties_required */
1635 0, /* properties_provided */
1636 0, /* properties_destroyed */
1637 0, /* todo_flags_start */
1638 0, /* todo_flags_finish */
1641 class pass_omp_simd_clone
: public simple_ipa_opt_pass
1644 pass_omp_simd_clone(gcc::context
*ctxt
)
1645 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
1648 /* opt_pass methods: */
1649 virtual bool gate (function
*);
1650 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
1654 pass_omp_simd_clone::gate (function
*)
1656 return targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
;
1661 simple_ipa_opt_pass
*
1662 make_pass_omp_simd_clone (gcc::context
*ctxt
)
1664 return new pass_omp_simd_clone (ctxt
);