1 /* OMP constructs' SIMD clone supporting code.
3 Copyright (C) 2005-2016 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
44 #include "tree-into-ssa.h"
47 #include "symbol-summary.h"
52 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
53 of arguments to reserve space for. */
55 static struct cgraph_simd_clone
*
56 simd_clone_struct_alloc (int nargs
)
58 struct cgraph_simd_clone
*clone_info
;
59 size_t len
= (sizeof (struct cgraph_simd_clone
)
60 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
61 clone_info
= (struct cgraph_simd_clone
*)
62 ggc_internal_cleared_alloc (len
);
66 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
69 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
70 struct cgraph_simd_clone
*from
)
72 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
73 + ((from
->nargs
- from
->inbranch
)
74 * sizeof (struct cgraph_simd_clone_arg
))));
77 /* Return vector of parameter types of function FNDECL. This uses
78 TYPE_ARG_TYPES if available, otherwise falls back to types of
79 DECL_ARGUMENTS types. */
82 simd_clone_vector_of_formal_parm_types (tree fndecl
)
84 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
85 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
86 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
89 FOR_EACH_VEC_ELT (args
, i
, arg
)
90 args
[i
] = TREE_TYPE (args
[i
]);
94 /* Given a simd function in NODE, extract the simd specific
95 information from the OMP clauses passed in CLAUSES, and return
96 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
97 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
98 otherwise set to FALSE. */
100 static struct cgraph_simd_clone
*
101 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
102 bool *inbranch_specified
)
104 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
107 *inbranch_specified
= false;
110 if (n
> 0 && args
.last () == void_type_node
)
113 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
114 be cloned have a distinctive artificial label in addition to "omp
118 && lookup_attribute ("cilk simd function",
119 DECL_ATTRIBUTES (node
->decl
)));
121 /* Allocate one more than needed just in case this is an in-branch
122 clone which will require a mask argument. */
123 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
124 clone_info
->nargs
= n
;
125 clone_info
->cilk_elemental
= cilk_clone
;
132 clauses
= TREE_VALUE (clauses
);
133 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
136 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
138 switch (OMP_CLAUSE_CODE (t
))
140 case OMP_CLAUSE_INBRANCH
:
141 clone_info
->inbranch
= 1;
142 *inbranch_specified
= true;
144 case OMP_CLAUSE_NOTINBRANCH
:
145 clone_info
->inbranch
= 0;
146 *inbranch_specified
= true;
148 case OMP_CLAUSE_SIMDLEN
:
150 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
152 case OMP_CLAUSE_LINEAR
:
154 tree decl
= OMP_CLAUSE_DECL (t
);
155 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
156 int argno
= TREE_INT_CST_LOW (decl
);
157 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
159 enum cgraph_simd_clone_arg_type arg_type
;
160 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
161 switch (OMP_CLAUSE_LINEAR_KIND (t
))
163 case OMP_CLAUSE_LINEAR_REF
:
165 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
167 case OMP_CLAUSE_LINEAR_UVAL
:
169 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
171 case OMP_CLAUSE_LINEAR_VAL
:
172 case OMP_CLAUSE_LINEAR_DEFAULT
:
174 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
180 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
181 clone_info
->args
[argno
].arg_type
= arg_type
;
182 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
183 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
184 && clone_info
->args
[argno
].linear_step
< n
);
188 if (POINTER_TYPE_P (args
[argno
]))
189 step
= fold_convert (ssizetype
, step
);
190 if (!tree_fits_shwi_p (step
))
192 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
193 "ignoring large linear step");
197 else if (integer_zerop (step
))
199 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
200 "ignoring zero linear step");
206 enum cgraph_simd_clone_arg_type arg_type
;
207 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
208 switch (OMP_CLAUSE_LINEAR_KIND (t
))
210 case OMP_CLAUSE_LINEAR_REF
:
212 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
214 case OMP_CLAUSE_LINEAR_UVAL
:
216 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
218 case OMP_CLAUSE_LINEAR_VAL
:
219 case OMP_CLAUSE_LINEAR_DEFAULT
:
221 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
227 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
228 clone_info
->args
[argno
].arg_type
= arg_type
;
229 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
234 case OMP_CLAUSE_UNIFORM
:
236 tree decl
= OMP_CLAUSE_DECL (t
);
237 int argno
= tree_to_uhwi (decl
);
238 clone_info
->args
[argno
].arg_type
239 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
242 case OMP_CLAUSE_ALIGNED
:
244 tree decl
= OMP_CLAUSE_DECL (t
);
245 int argno
= tree_to_uhwi (decl
);
246 clone_info
->args
[argno
].alignment
247 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
258 /* Given a SIMD clone in NODE, calculate the characteristic data
259 type and return the coresponding type. The characteristic data
260 type is computed as described in the Intel Vector ABI. */
263 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
264 struct cgraph_simd_clone
*clone_info
)
266 tree type
= integer_type_node
;
267 tree fndecl
= node
->decl
;
269 /* a) For non-void function, the characteristic data type is the
271 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
272 type
= TREE_TYPE (TREE_TYPE (fndecl
));
274 /* b) If the function has any non-uniform, non-linear parameters,
275 then the characteristic data type is the type of the first
279 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
280 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
281 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
289 /* c) If the characteristic data type determined by a) or b) above
290 is struct, union, or class type which is pass-by-value (except
291 for the type that maps to the built-in complex data type), the
292 characteristic data type is int. */
293 if (RECORD_OR_UNION_TYPE_P (type
)
294 && !aggregate_value_p (type
, NULL
)
295 && TREE_CODE (type
) != COMPLEX_TYPE
)
296 return integer_type_node
;
298 /* d) If none of the above three classes is applicable, the
299 characteristic data type is int. */
303 /* e) For Intel Xeon Phi native and offload compilation, if the
304 resulting characteristic data type is 8-bit or 16-bit integer
305 data type, the characteristic data type is int. */
306 /* Well, we don't handle Xeon Phi yet. */
310 simd_clone_mangle (struct cgraph_node
*node
,
311 struct cgraph_simd_clone
*clone_info
)
313 char vecsize_mangle
= clone_info
->vecsize_mangle
;
314 char mask
= clone_info
->inbranch
? 'M' : 'N';
315 unsigned int simdlen
= clone_info
->simdlen
;
319 gcc_assert (vecsize_mangle
&& simdlen
);
321 pp_string (&pp
, "_ZGV");
322 pp_character (&pp
, vecsize_mangle
);
323 pp_character (&pp
, mask
);
324 pp_decimal_int (&pp
, simdlen
);
326 for (n
= 0; n
< clone_info
->nargs
; ++n
)
328 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
330 switch (arg
.arg_type
)
332 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
333 pp_character (&pp
, 'u');
335 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
336 pp_character (&pp
, 'l');
338 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
339 pp_character (&pp
, 'R');
341 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
342 pp_character (&pp
, 'L');
344 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
345 pp_character (&pp
, 'U');
348 gcc_assert (arg
.linear_step
!= 0);
349 if (arg
.linear_step
> 1)
350 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
351 else if (arg
.linear_step
< 0)
353 pp_character (&pp
, 'n');
354 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
358 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
359 pp_string (&pp
, "ls");
360 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
362 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
363 pp_string (&pp
, "Rs");
364 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
366 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
367 pp_string (&pp
, "Ls");
368 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
370 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
371 pp_string (&pp
, "Us");
372 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
375 pp_character (&pp
, 'v');
379 pp_character (&pp
, 'a');
380 pp_decimal_int (&pp
, arg
.alignment
);
385 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
388 pp_string (&pp
, str
);
389 str
= pp_formatted_text (&pp
);
391 /* If there already is a SIMD clone with the same mangled name, don't
392 add another one. This can happen e.g. for
393 #pragma omp declare simd
394 #pragma omp declare simd simdlen(8)
396 if the simdlen is assumed to be 8 for the first one, etc. */
397 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
398 clone
= clone
->simdclone
->next_clone
)
399 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
403 return get_identifier (str
);
406 /* Create a simd clone of OLD_NODE and return it. */
408 static struct cgraph_node
*
409 simd_clone_create (struct cgraph_node
*old_node
)
411 struct cgraph_node
*new_node
;
412 if (old_node
->definition
)
414 if (!old_node
->has_gimple_body_p ())
416 old_node
->get_body ();
417 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
423 tree old_decl
= old_node
->decl
;
424 tree new_decl
= copy_node (old_node
->decl
);
425 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
426 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
427 SET_DECL_RTL (new_decl
, NULL
);
428 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
429 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
430 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
431 if (old_node
->in_other_partition
)
432 new_node
->in_other_partition
= 1;
434 if (new_node
== NULL
)
437 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
439 /* The function cgraph_function_versioning () will force the new
440 symbol local. Undo this, and inherit external visability from
442 new_node
->local
.local
= old_node
->local
.local
;
443 new_node
->externally_visible
= old_node
->externally_visible
;
448 /* Adjust the return type of the given function to its appropriate
449 vector counterpart. Returns a simd array to be used throughout the
450 function as a return value. */
453 simd_clone_adjust_return_type (struct cgraph_node
*node
)
455 tree fndecl
= node
->decl
;
456 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
460 /* Adjust the function return type. */
461 if (orig_rettype
== void_type_node
)
463 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
464 t
= TREE_TYPE (TREE_TYPE (fndecl
));
465 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
466 veclen
= node
->simdclone
->vecsize_int
;
468 veclen
= node
->simdclone
->vecsize_float
;
469 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
470 if (veclen
> node
->simdclone
->simdlen
)
471 veclen
= node
->simdclone
->simdlen
;
472 if (POINTER_TYPE_P (t
))
473 t
= pointer_sized_int_node
;
474 if (veclen
== node
->simdclone
->simdlen
)
475 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
478 t
= build_vector_type (t
, veclen
);
479 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
481 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
482 if (!node
->definition
)
485 t
= DECL_RESULT (fndecl
);
486 /* Adjust the DECL_RESULT. */
487 gcc_assert (TREE_TYPE (t
) != void_type_node
);
488 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
491 tree atype
= build_array_type_nelts (orig_rettype
,
492 node
->simdclone
->simdlen
);
493 if (veclen
!= node
->simdclone
->simdlen
)
494 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
496 /* Set up a SIMD array to use as the return value. */
497 tree retval
= create_tmp_var_raw (atype
, "retval");
498 gimple_add_tmp_var (retval
);
502 /* Each vector argument has a corresponding array to be used locally
503 as part of the eventual loop. Create such temporary array and
506 PREFIX is the prefix to be used for the temporary.
508 TYPE is the inner element type.
510 SIMDLEN is the number of elements. */
513 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
515 tree atype
= build_array_type_nelts (type
, simdlen
);
516 tree avar
= create_tmp_var_raw (atype
, prefix
);
517 gimple_add_tmp_var (avar
);
521 /* Modify the function argument types to their corresponding vector
522 counterparts if appropriate. Also, create one array for each simd
523 argument to be used locally when using the function arguments as
526 NODE is the function whose arguments are to be adjusted.
528 Returns an adjustment vector that will be filled describing how the
529 argument types will be adjusted. */
531 static ipa_parm_adjustment_vec
532 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
535 ipa_parm_adjustment_vec adjustments
;
537 if (node
->definition
)
538 args
= ipa_get_vector_of_formal_parms (node
->decl
);
540 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
541 adjustments
.create (args
.length ());
542 unsigned i
, j
, veclen
;
543 struct ipa_parm_adjustment adj
;
544 struct cgraph_simd_clone
*sc
= node
->simdclone
;
546 for (i
= 0; i
< sc
->nargs
; ++i
)
548 memset (&adj
, 0, sizeof (adj
));
550 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
554 sc
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
555 sc
->args
[i
].orig_type
= parm_type
;
557 switch (sc
->args
[i
].arg_type
)
560 /* No adjustment necessary for scalar arguments. */
561 adj
.op
= IPA_PARM_OP_COPY
;
563 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
564 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
565 if (node
->definition
)
566 sc
->args
[i
].simd_array
567 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
568 TREE_TYPE (parm_type
),
570 adj
.op
= IPA_PARM_OP_COPY
;
572 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
573 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
574 case SIMD_CLONE_ARG_TYPE_VECTOR
:
575 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
576 veclen
= sc
->vecsize_int
;
578 veclen
= sc
->vecsize_float
;
579 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
580 if (veclen
> sc
->simdlen
)
581 veclen
= sc
->simdlen
;
582 adj
.arg_prefix
= "simd";
583 if (POINTER_TYPE_P (parm_type
))
584 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
586 adj
.type
= build_vector_type (parm_type
, veclen
);
587 sc
->args
[i
].vector_type
= adj
.type
;
588 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
590 adjustments
.safe_push (adj
);
593 memset (&adj
, 0, sizeof (adj
));
594 adj
.op
= IPA_PARM_OP_NEW
;
595 adj
.arg_prefix
= "simd";
597 adj
.type
= sc
->args
[i
].vector_type
;
601 if (node
->definition
)
602 sc
->args
[i
].simd_array
603 = create_tmp_simd_array (DECL_NAME (parm
)
604 ? IDENTIFIER_POINTER (DECL_NAME (parm
))
605 : NULL
, parm_type
, sc
->simdlen
);
607 adjustments
.safe_push (adj
);
612 tree base_type
= simd_clone_compute_base_data_type (sc
->origin
, sc
);
614 memset (&adj
, 0, sizeof (adj
));
615 adj
.op
= IPA_PARM_OP_NEW
;
616 adj
.arg_prefix
= "mask";
619 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
620 veclen
= sc
->vecsize_int
;
622 veclen
= sc
->vecsize_float
;
623 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
624 if (veclen
> sc
->simdlen
)
625 veclen
= sc
->simdlen
;
626 if (sc
->mask_mode
!= VOIDmode
)
628 = lang_hooks
.types
.type_for_mode (sc
->mask_mode
, 1);
629 else if (POINTER_TYPE_P (base_type
))
630 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
632 adj
.type
= build_vector_type (base_type
, veclen
);
633 adjustments
.safe_push (adj
);
635 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
636 adjustments
.safe_push (adj
);
638 /* We have previously allocated one extra entry for the mask. Use
641 if (sc
->mask_mode
!= VOIDmode
)
642 base_type
= boolean_type_node
;
643 if (node
->definition
)
646 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
647 if (sc
->mask_mode
== VOIDmode
)
648 sc
->args
[i
].simd_array
649 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
650 else if (veclen
< sc
->simdlen
)
651 sc
->args
[i
].simd_array
652 = create_tmp_simd_array ("mask", adj
.type
, sc
->simdlen
/ veclen
);
654 sc
->args
[i
].simd_array
= NULL_TREE
;
656 sc
->args
[i
].orig_type
= base_type
;
657 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
660 if (node
->definition
)
661 ipa_modify_formal_parameters (node
->decl
, adjustments
);
664 tree new_arg_types
= NULL_TREE
, new_reversed
;
665 bool last_parm_void
= false;
666 if (args
.length () > 0 && args
.last () == void_type_node
)
667 last_parm_void
= true;
669 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
670 j
= adjustments
.length ();
671 for (i
= 0; i
< j
; i
++)
673 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
675 if (adj
->op
== IPA_PARM_OP_COPY
)
676 ptype
= args
[adj
->base_index
];
679 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
681 new_reversed
= nreverse (new_arg_types
);
685 TREE_CHAIN (new_arg_types
) = void_list_node
;
687 new_reversed
= void_list_node
;
690 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
691 TYPE_ARG_TYPES (new_type
) = new_reversed
;
692 TREE_TYPE (node
->decl
) = new_type
;
694 adjustments
.release ();
700 /* Initialize and copy the function arguments in NODE to their
701 corresponding local simd arrays. Returns a fresh gimple_seq with
702 the instruction sequence generated. */
705 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
706 ipa_parm_adjustment_vec adjustments
)
708 gimple_seq seq
= NULL
;
709 unsigned i
= 0, j
= 0, k
;
711 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
713 arg
= DECL_CHAIN (arg
), i
++, j
++)
715 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
716 || POINTER_TYPE_P (TREE_TYPE (arg
)))
719 node
->simdclone
->args
[i
].vector_arg
= arg
;
721 tree array
= node
->simdclone
->args
[i
].simd_array
;
722 if (node
->simdclone
->mask_mode
!= VOIDmode
723 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_MASK
)
725 if (array
== NULL_TREE
)
728 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array
))));
729 for (k
= 0; k
<= l
; k
++)
733 arg
= DECL_CHAIN (arg
);
736 tree t
= build4 (ARRAY_REF
, TREE_TYPE (TREE_TYPE (array
)),
737 array
, size_int (k
), NULL
, NULL
);
738 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
739 gimplify_and_add (t
, &seq
);
743 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
745 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
746 tree ptr
= build_fold_addr_expr (array
);
747 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
748 build_int_cst (ptype
, 0));
749 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
750 gimplify_and_add (t
, &seq
);
754 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
755 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
756 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
758 tree ptr
= build_fold_addr_expr (array
);
762 arg
= DECL_CHAIN (arg
);
766 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
767 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
768 build_int_cst (ptype
, k
* elemsize
));
769 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
770 gimplify_and_add (t
, &seq
);
777 /* Callback info for ipa_simd_modify_stmt_ops below. */
779 struct modify_stmt_info
{
780 ipa_parm_adjustment_vec adjustments
;
782 /* True if the parent statement was modified by
783 ipa_simd_modify_stmt_ops. */
787 /* Callback for walk_gimple_op.
789 Adjust operands from a given statement as specified in the
790 adjustments vector in the callback data. */
793 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
795 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
796 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
798 if (TREE_CODE (*tp
) == ADDR_EXPR
)
799 tp
= &TREE_OPERAND (*tp
, 0);
800 struct ipa_parm_adjustment
*cand
= NULL
;
801 if (TREE_CODE (*tp
) == PARM_DECL
)
802 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
809 tree repl
= NULL_TREE
;
811 repl
= unshare_expr (cand
->new_decl
);
817 bool modified
= info
->modified
;
818 info
->modified
= false;
819 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
822 info
->modified
= modified
;
825 info
->modified
= modified
;
834 repl
= build_fold_addr_expr (repl
);
836 if (is_gimple_debug (info
->stmt
))
838 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
839 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
840 DECL_ARTIFICIAL (vexpr
) = 1;
841 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
842 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
847 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
848 repl
= gimple_assign_lhs (stmt
);
850 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
851 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
854 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
856 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
862 info
->modified
= true;
866 /* Traverse the function body and perform all modifications as
867 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
868 modified such that the replacement/reduction value will now be an
869 offset into the corresponding simd_array.
871 This function will replace all function argument uses with their
872 corresponding simd array elements, and ajust the return values
876 ipa_simd_modify_function_body (struct cgraph_node
*node
,
877 ipa_parm_adjustment_vec adjustments
,
878 tree retval_array
, tree iter
)
881 unsigned int i
, j
, l
;
883 /* Re-use the adjustments array, but this time use it to replace
884 every function argument use to an offset into the corresponding
886 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
888 if (!node
->simdclone
->args
[i
].vector_arg
)
891 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
892 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
893 adjustments
[j
].new_decl
896 node
->simdclone
->args
[i
].simd_array
,
898 NULL_TREE
, NULL_TREE
);
899 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
900 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
901 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
904 l
= adjustments
.length ();
905 for (i
= 1; i
< num_ssa_names
; i
++)
907 tree name
= ssa_name (i
);
909 && SSA_NAME_VAR (name
)
910 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
912 for (j
= 0; j
< l
; j
++)
913 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
914 && adjustments
[j
].new_decl
)
917 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
920 = copy_var_decl (adjustments
[j
].base
,
921 DECL_NAME (adjustments
[j
].base
),
922 TREE_TYPE (adjustments
[j
].base
));
923 adjustments
[j
].new_ssa_base
= base_var
;
926 base_var
= adjustments
[j
].new_ssa_base
;
927 if (SSA_NAME_IS_DEFAULT_DEF (name
))
929 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
930 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
931 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
932 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
933 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
934 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
935 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
936 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
939 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
944 struct modify_stmt_info info
;
945 info
.adjustments
= adjustments
;
947 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
949 gimple_stmt_iterator gsi
;
951 gsi
= gsi_start_bb (bb
);
952 while (!gsi_end_p (gsi
))
954 gimple
*stmt
= gsi_stmt (gsi
);
956 struct walk_stmt_info wi
;
958 memset (&wi
, 0, sizeof (wi
));
959 info
.modified
= false;
961 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
963 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
965 tree retval
= gimple_return_retval (return_stmt
);
968 gsi_remove (&gsi
, true);
972 /* Replace `return foo' with `retval_array[iter] = foo'. */
973 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
974 retval_array
, iter
, NULL
, NULL
);
975 stmt
= gimple_build_assign (ref
, retval
);
976 gsi_replace (&gsi
, stmt
, true);
977 info
.modified
= true;
983 if (maybe_clean_eh_stmt (stmt
))
984 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
991 /* Helper function of simd_clone_adjust, return linear step addend
995 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
996 tree addtype
, basic_block entry_bb
)
998 tree ptype
= NULL_TREE
;
999 switch (node
->simdclone
->args
[i
].arg_type
)
1001 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
1002 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
1003 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
1004 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
1005 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
1006 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
1007 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
1008 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
1010 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
1011 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
1012 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
1018 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
1019 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
1020 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
1021 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1024 if (is_gimple_reg (arg
))
1025 ret
= get_or_create_ssa_default_def (cfun
, arg
);
1028 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
1029 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1030 ret
= gimple_assign_lhs (g
);
1032 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
1034 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
1035 build_simple_mem_ref (ret
));
1036 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1037 ret
= gimple_assign_lhs (g
);
1039 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
1041 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
1042 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1043 ret
= gimple_assign_lhs (g
);
1045 if (POINTER_TYPE_P (ptype
))
1047 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
1048 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
1050 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
1051 ret
, fold_convert (addtype
, size
));
1052 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1053 ret
= gimple_assign_lhs (g
);
1059 /* Adjust the argument types in NODE to their appropriate vector
1063 simd_clone_adjust (struct cgraph_node
*node
)
1065 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1067 targetm
.simd_clone
.adjust (node
);
1069 tree retval
= simd_clone_adjust_return_type (node
);
1070 ipa_parm_adjustment_vec adjustments
1071 = simd_clone_adjust_argument_types (node
);
1073 push_gimplify_context ();
1075 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
1077 /* Adjust all uses of vector arguments accordingly. Adjust all
1078 return values accordingly. */
1079 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
1080 tree iter1
= make_ssa_name (iter
);
1081 tree iter2
= NULL_TREE
;
1082 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
1084 /* Initialize the iteration variable. */
1085 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1086 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
1087 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1088 /* Insert the SIMD array and iv initialization at function
1090 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
1092 pop_gimplify_context (NULL
);
1095 basic_block incr_bb
= NULL
;
1096 struct loop
*loop
= NULL
;
1098 /* Create a new BB right before the original exit BB, to hold the
1099 iteration increment and the condition/branch. */
1100 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
))
1102 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
1103 incr_bb
= create_empty_bb (orig_exit
);
1104 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1105 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
1106 flag. Set it now to be a FALLTHRU_EDGE. */
1107 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
1108 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
1109 for (unsigned i
= 0;
1110 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
1112 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
1113 redirect_edge_succ (e
, incr_bb
);
1116 else if (node
->simdclone
->inbranch
)
1118 incr_bb
= create_empty_bb (entry_bb
);
1119 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1124 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
1125 e
->probability
= REG_BR_PROB_BASE
;
1126 gsi
= gsi_last_bb (incr_bb
);
1127 iter2
= make_ssa_name (iter
);
1128 g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
1129 build_int_cst (unsigned_type_node
, 1));
1130 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1132 /* Mostly annotate the loop for the vectorizer (the rest is done
1134 loop
= alloc_loop ();
1135 cfun
->has_force_vectorize_loops
= true;
1136 loop
->safelen
= node
->simdclone
->simdlen
;
1137 loop
->force_vectorize
= true;
1138 loop
->header
= body_bb
;
1141 /* Branch around the body if the mask applies. */
1142 if (node
->simdclone
->inbranch
)
1144 gsi
= gsi_last_bb (loop
->header
);
1146 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
1148 if (node
->simdclone
->mask_mode
!= VOIDmode
)
1151 if (mask_array
== NULL_TREE
)
1153 tree arg
= node
->simdclone
->args
[node
->simdclone
->nargs
1155 mask
= get_or_create_ssa_default_def (cfun
, arg
);
1160 tree maskt
= TREE_TYPE (mask_array
);
1161 int c
= tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt
)));
1162 c
= node
->simdclone
->simdlen
/ (c
+ 1);
1163 int s
= exact_log2 (c
);
1166 tree idx
= make_ssa_name (TREE_TYPE (iter1
));
1167 g
= gimple_build_assign (idx
, RSHIFT_EXPR
, iter1
,
1168 build_int_cst (NULL_TREE
, s
));
1169 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1170 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1171 tree aref
= build4 (ARRAY_REF
,
1172 TREE_TYPE (TREE_TYPE (mask_array
)),
1173 mask_array
, idx
, NULL
, NULL
);
1174 g
= gimple_build_assign (mask
, aref
);
1175 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1176 shift_cnt
= make_ssa_name (TREE_TYPE (iter1
));
1177 g
= gimple_build_assign (shift_cnt
, BIT_AND_EXPR
, iter1
,
1178 build_int_cst (TREE_TYPE (iter1
), c
));
1179 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1181 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1182 RSHIFT_EXPR
, mask
, shift_cnt
);
1183 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1184 mask
= gimple_assign_lhs (g
);
1185 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1187 build_int_cst (TREE_TYPE (mask
), 1));
1188 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1189 mask
= gimple_assign_lhs (g
);
1193 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1194 tree aref
= build4 (ARRAY_REF
,
1195 TREE_TYPE (TREE_TYPE (mask_array
)),
1196 mask_array
, iter1
, NULL
, NULL
);
1197 g
= gimple_build_assign (mask
, aref
);
1198 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1199 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
1200 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
1202 aref
= build1 (VIEW_CONVERT_EXPR
,
1203 build_nonstandard_integer_type (bitsize
, 0),
1205 mask
= make_ssa_name (TREE_TYPE (aref
));
1206 g
= gimple_build_assign (mask
, aref
);
1207 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1211 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
1213 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1214 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
1215 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
1218 basic_block latch_bb
= NULL
;
1219 basic_block new_exit_bb
= NULL
;
1221 /* Generate the condition. */
1224 gsi
= gsi_last_bb (incr_bb
);
1225 g
= gimple_build_cond (LT_EXPR
, iter2
,
1226 build_int_cst (unsigned_type_node
,
1227 node
->simdclone
->simdlen
),
1229 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1230 edge e
= split_block (incr_bb
, gsi_stmt (gsi
));
1232 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
1233 loop
->latch
= latch_bb
;
1235 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
1237 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
1238 /* The successor of incr_bb is already pointing to latch_bb; just
1240 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1241 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
1244 gphi
*phi
= create_phi_node (iter1
, body_bb
);
1245 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
1246 edge latch_edge
= NULL
;
1247 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
1251 latch_edge
= single_succ_edge (latch_bb
);
1252 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1254 /* Generate the new return. */
1255 gsi
= gsi_last_bb (new_exit_bb
);
1257 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
1258 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
1259 retval
= TREE_OPERAND (retval
, 0);
1262 retval
= build1 (VIEW_CONVERT_EXPR
,
1263 TREE_TYPE (TREE_TYPE (node
->decl
)),
1265 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
1266 false, GSI_CONTINUE_LINKING
);
1268 g
= gimple_build_return (retval
);
1269 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1272 /* Handle aligned clauses by replacing default defs of the aligned
1273 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1274 lhs. Handle linear by adding PHIs. */
1275 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
1276 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1277 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
1278 || !is_gimple_reg_type
1279 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1281 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1282 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
1283 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1286 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
1287 gimple_add_tmp_var (iter1
);
1289 gsi
= gsi_after_labels (entry_bb
);
1290 g
= gimple_build_assign (iter1
, orig_arg
);
1291 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1292 gsi
= gsi_after_labels (body_bb
);
1293 g
= gimple_build_assign (orig_arg
, iter1
);
1294 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1296 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1297 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
1298 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1301 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1303 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1304 tree def
= ssa_default_def (cfun
, orig_arg
);
1305 if (def
&& !has_zero_uses (def
))
1307 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
1308 gimple_add_tmp_var (iter1
);
1309 gsi
= gsi_after_labels (entry_bb
);
1310 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
1311 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1312 gsi
= gsi_after_labels (body_bb
);
1313 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
1314 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1317 else if (node
->simdclone
->args
[i
].alignment
1318 && node
->simdclone
->args
[i
].arg_type
1319 == SIMD_CLONE_ARG_TYPE_UNIFORM
1320 && (node
->simdclone
->args
[i
].alignment
1321 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
1322 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1325 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
1326 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1327 tree def
= ssa_default_def (cfun
, orig_arg
);
1328 if (def
&& !has_zero_uses (def
))
1330 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
1331 gimple_seq seq
= NULL
;
1332 bool need_cvt
= false;
1334 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
1336 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
1339 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
1340 gimple_call_set_lhs (g
, t
);
1341 gimple_seq_add_stmt_without_update (&seq
, g
);
1344 t
= make_ssa_name (orig_arg
);
1345 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
1346 gimple_seq_add_stmt_without_update (&seq
, g
);
1348 gsi_insert_seq_on_edge_immediate
1349 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
1351 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1352 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
1354 node
->create_edge (cgraph_node::get_create (fn
),
1355 call
, entry_bb
->count
, freq
);
1357 imm_use_iterator iter
;
1358 use_operand_p use_p
;
1360 tree repl
= gimple_get_lhs (g
);
1361 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1362 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
1365 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1366 SET_USE (use_p
, repl
);
1369 else if ((node
->simdclone
->args
[i
].arg_type
1370 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
1371 || (node
->simdclone
->args
[i
].arg_type
1372 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
1373 || (node
->simdclone
->args
[i
].arg_type
1374 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
1375 || (node
->simdclone
->args
[i
].arg_type
1376 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
1378 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1379 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1380 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
1381 tree def
= NULL_TREE
;
1382 if (TREE_ADDRESSABLE (orig_arg
))
1384 def
= make_ssa_name (TREE_TYPE (orig_arg
));
1385 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1387 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
1388 gsi
= gsi_after_labels (entry_bb
);
1389 g
= gimple_build_assign (def
, orig_arg
);
1390 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1394 def
= ssa_default_def (cfun
, orig_arg
);
1395 if (!def
|| has_zero_uses (def
))
1399 iter1
= make_ssa_name (orig_arg
);
1401 iter2
= make_ssa_name (orig_arg
);
1406 phi
= create_phi_node (iter1
, body_bb
);
1407 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
1410 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1411 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1412 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1413 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1414 ? TREE_TYPE (orig_arg
) : sizetype
;
1415 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1417 gsi
= gsi_last_bb (incr_bb
);
1418 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
1419 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1422 imm_use_iterator iter
;
1423 use_operand_p use_p
;
1425 if (TREE_ADDRESSABLE (orig_arg
))
1427 gsi
= gsi_after_labels (body_bb
);
1428 g
= gimple_build_assign (orig_arg
, iter1
);
1429 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1432 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1433 if (use_stmt
== phi
)
1436 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1437 SET_USE (use_p
, iter1
);
1440 else if (node
->simdclone
->args
[i
].arg_type
1441 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1442 || (node
->simdclone
->args
[i
].arg_type
1443 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
1445 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1446 tree def
= ssa_default_def (cfun
, orig_arg
);
1447 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
1448 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
1449 if (def
&& !has_zero_uses (def
))
1451 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
1452 iter1
= make_ssa_name (orig_arg
);
1454 iter2
= make_ssa_name (orig_arg
);
1455 tree iter3
= make_ssa_name (rtype
);
1456 tree iter4
= make_ssa_name (rtype
);
1457 tree iter5
= incr_bb
? make_ssa_name (rtype
) : NULL_TREE
;
1458 gsi
= gsi_after_labels (entry_bb
);
1460 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
1461 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
1463 tree array
= node
->simdclone
->args
[i
].simd_array
;
1464 TREE_ADDRESSABLE (array
) = 1;
1465 tree ptr
= build_fold_addr_expr (array
);
1466 phi
= create_phi_node (iter1
, body_bb
);
1467 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
1470 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1471 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
1472 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
1473 gsi
= gsi_last_bb (incr_bb
);
1474 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1477 phi
= create_phi_node (iter4
, body_bb
);
1478 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
1481 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
1482 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1483 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1484 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1485 ? TREE_TYPE (iter3
) : sizetype
;
1486 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1488 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
1489 gsi
= gsi_last_bb (incr_bb
);
1490 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1493 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
1494 gsi
= gsi_after_labels (body_bb
);
1495 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1497 imm_use_iterator iter
;
1498 use_operand_p use_p
;
1500 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1501 if (use_stmt
== load
)
1504 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1505 SET_USE (use_p
, iter1
);
1507 if (!TYPE_READONLY (rtype
) && incr_bb
)
1509 tree v
= make_ssa_name (rtype
);
1510 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
1511 size_zero_node
, NULL_TREE
,
1513 gsi
= gsi_after_labels (new_exit_bb
);
1514 g
= gimple_build_assign (v
, aref
);
1515 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1516 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
1517 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1522 calculate_dominance_info (CDI_DOMINATORS
);
1524 add_loop (loop
, loop
->header
->loop_father
);
1525 update_ssa (TODO_update_ssa
);
1530 /* If the function in NODE is tagged as an elemental SIMD function,
1531 create the appropriate SIMD clones. */
1534 expand_simd_clones (struct cgraph_node
*node
)
1536 tree attr
= lookup_attribute ("omp declare simd",
1537 DECL_ATTRIBUTES (node
->decl
));
1538 if (attr
== NULL_TREE
1539 || node
->global
.inlined_to
1540 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
1544 #pragma omp declare simd
1546 in C, there we don't know the argument types at all. */
1547 if (!node
->definition
1548 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
1551 /* Call this before creating clone_info, as it might ggc_collect. */
1552 if (node
->definition
&& node
->has_gimple_body_p ())
1557 /* Start with parsing the "omp declare simd" attribute(s). */
1558 bool inbranch_clause_specified
;
1559 struct cgraph_simd_clone
*clone_info
1560 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
1561 &inbranch_clause_specified
);
1562 if (clone_info
== NULL
)
1565 int orig_simdlen
= clone_info
->simdlen
;
1566 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
1567 /* The target can return 0 (no simd clones should be created),
1568 1 (just one ISA of simd clones should be created) or higher
1569 count of ISA variants. In that case, clone_info is initialized
1570 for the first ISA variant. */
1572 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
1577 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1578 also create one inbranch and one !inbranch clone of it. */
1579 for (int i
= 0; i
< count
* 2; i
++)
1581 struct cgraph_simd_clone
*clone
= clone_info
;
1582 if (inbranch_clause_specified
&& (i
& 1) != 0)
1587 clone
= simd_clone_struct_alloc (clone_info
->nargs
1589 simd_clone_struct_copy (clone
, clone_info
);
1590 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1591 and simd_clone_adjust_argument_types did to the first
1593 clone
->nargs
-= clone_info
->inbranch
;
1594 clone
->simdlen
= orig_simdlen
;
1595 /* And call the target hook again to get the right ISA. */
1596 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
1600 clone
->inbranch
= 1;
1603 /* simd_clone_mangle might fail if such a clone has been created
1605 tree id
= simd_clone_mangle (node
, clone
);
1606 if (id
== NULL_TREE
)
1609 /* Only when we are sure we want to create the clone actually
1610 clone the function (or definitions) or create another
1611 extern FUNCTION_DECL (for prototypes without definitions). */
1612 struct cgraph_node
*n
= simd_clone_create (node
);
1616 n
->simdclone
= clone
;
1617 clone
->origin
= node
;
1618 clone
->next_clone
= NULL
;
1619 if (node
->simd_clones
== NULL
)
1621 clone
->prev_clone
= n
;
1622 node
->simd_clones
= n
;
1626 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
1627 clone
->prev_clone
->simdclone
->next_clone
= n
;
1628 node
->simd_clones
->simdclone
->prev_clone
= n
;
1630 symtab
->change_decl_assembler_name (n
->decl
, id
);
1631 /* And finally adjust the return type, parameters and for
1632 definitions also function body. */
1633 if (node
->definition
)
1634 simd_clone_adjust (n
);
1637 simd_clone_adjust_return_type (n
);
1638 simd_clone_adjust_argument_types (n
);
1642 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
1645 /* Entry point for IPA simd clone creation pass. */
1648 ipa_omp_simd_clone (void)
1650 struct cgraph_node
*node
;
1651 FOR_EACH_FUNCTION (node
)
1652 expand_simd_clones (node
);
1658 const pass_data pass_data_omp_simd_clone
=
1660 SIMPLE_IPA_PASS
, /* type */
1661 "simdclone", /* name */
1662 OPTGROUP_NONE
, /* optinfo_flags */
1663 TV_NONE
, /* tv_id */
1664 ( PROP_ssa
| PROP_cfg
), /* properties_required */
1665 0, /* properties_provided */
1666 0, /* properties_destroyed */
1667 0, /* todo_flags_start */
1668 0, /* todo_flags_finish */
1671 class pass_omp_simd_clone
: public simple_ipa_opt_pass
1674 pass_omp_simd_clone(gcc::context
*ctxt
)
1675 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
1678 /* opt_pass methods: */
1679 virtual bool gate (function
*);
1680 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
1684 pass_omp_simd_clone::gate (function
*)
1686 return targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
;
1691 simple_ipa_opt_pass
*
1692 make_pass_omp_simd_clone (gcc::context
*ctxt
)
1694 return new pass_omp_simd_clone (ctxt
);