1 /* OMP constructs' SIMD clone supporting code.
3 Copyright (C) 2005-2016 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
44 #include "tree-into-ssa.h"
47 #include "symbol-summary.h"
53 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
54 of arguments to reserve space for. */
56 static struct cgraph_simd_clone
*
57 simd_clone_struct_alloc (int nargs
)
59 struct cgraph_simd_clone
*clone_info
;
60 size_t len
= (sizeof (struct cgraph_simd_clone
)
61 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
62 clone_info
= (struct cgraph_simd_clone
*)
63 ggc_internal_cleared_alloc (len
);
67 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
70 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
71 struct cgraph_simd_clone
*from
)
73 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
74 + ((from
->nargs
- from
->inbranch
)
75 * sizeof (struct cgraph_simd_clone_arg
))));
78 /* Return vector of parameter types of function FNDECL. This uses
79 TYPE_ARG_TYPES if available, otherwise falls back to types of
80 DECL_ARGUMENTS types. */
83 simd_clone_vector_of_formal_parm_types (tree fndecl
)
85 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
86 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
87 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
90 FOR_EACH_VEC_ELT (args
, i
, arg
)
91 args
[i
] = TREE_TYPE (args
[i
]);
95 /* Given a simd function in NODE, extract the simd specific
96 information from the OMP clauses passed in CLAUSES, and return
97 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
98 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
99 otherwise set to FALSE. */
101 static struct cgraph_simd_clone
*
102 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
103 bool *inbranch_specified
)
105 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
108 *inbranch_specified
= false;
111 if (n
> 0 && args
.last () == void_type_node
)
114 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
115 be cloned have a distinctive artificial label in addition to "omp
119 && lookup_attribute ("cilk simd function",
120 DECL_ATTRIBUTES (node
->decl
)));
122 /* Allocate one more than needed just in case this is an in-branch
123 clone which will require a mask argument. */
124 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
125 clone_info
->nargs
= n
;
126 clone_info
->cilk_elemental
= cilk_clone
;
131 clauses
= TREE_VALUE (clauses
);
132 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
135 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
137 switch (OMP_CLAUSE_CODE (t
))
139 case OMP_CLAUSE_INBRANCH
:
140 clone_info
->inbranch
= 1;
141 *inbranch_specified
= true;
143 case OMP_CLAUSE_NOTINBRANCH
:
144 clone_info
->inbranch
= 0;
145 *inbranch_specified
= true;
147 case OMP_CLAUSE_SIMDLEN
:
149 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
151 case OMP_CLAUSE_LINEAR
:
153 tree decl
= OMP_CLAUSE_DECL (t
);
154 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
155 int argno
= TREE_INT_CST_LOW (decl
);
156 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
158 enum cgraph_simd_clone_arg_type arg_type
;
159 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
160 switch (OMP_CLAUSE_LINEAR_KIND (t
))
162 case OMP_CLAUSE_LINEAR_REF
:
164 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
166 case OMP_CLAUSE_LINEAR_UVAL
:
168 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
170 case OMP_CLAUSE_LINEAR_VAL
:
171 case OMP_CLAUSE_LINEAR_DEFAULT
:
173 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
179 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
180 clone_info
->args
[argno
].arg_type
= arg_type
;
181 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
182 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
183 && clone_info
->args
[argno
].linear_step
< n
);
187 if (POINTER_TYPE_P (args
[argno
]))
188 step
= fold_convert (ssizetype
, step
);
189 if (!tree_fits_shwi_p (step
))
191 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
192 "ignoring large linear step");
196 else if (integer_zerop (step
))
198 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
199 "ignoring zero linear step");
205 enum cgraph_simd_clone_arg_type arg_type
;
206 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
207 switch (OMP_CLAUSE_LINEAR_KIND (t
))
209 case OMP_CLAUSE_LINEAR_REF
:
211 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
213 case OMP_CLAUSE_LINEAR_UVAL
:
215 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
217 case OMP_CLAUSE_LINEAR_VAL
:
218 case OMP_CLAUSE_LINEAR_DEFAULT
:
220 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
226 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
227 clone_info
->args
[argno
].arg_type
= arg_type
;
228 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
233 case OMP_CLAUSE_UNIFORM
:
235 tree decl
= OMP_CLAUSE_DECL (t
);
236 int argno
= tree_to_uhwi (decl
);
237 clone_info
->args
[argno
].arg_type
238 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
241 case OMP_CLAUSE_ALIGNED
:
243 tree decl
= OMP_CLAUSE_DECL (t
);
244 int argno
= tree_to_uhwi (decl
);
245 clone_info
->args
[argno
].alignment
246 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
255 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node
->decl
))))
257 warning_at (DECL_SOURCE_LOCATION (node
->decl
), 0,
258 "ignoring %<#pragma omp declare simd%> on function "
259 "with %<_Atomic%> qualified return type");
264 for (unsigned int argno
= 0; argno
< clone_info
->nargs
; argno
++)
265 if (TYPE_ATOMIC (args
[argno
])
266 && clone_info
->args
[argno
].arg_type
!= SIMD_CLONE_ARG_TYPE_UNIFORM
)
268 warning_at (DECL_SOURCE_LOCATION (node
->decl
), 0,
269 "ignoring %<#pragma omp declare simd%> on function "
270 "with %<_Atomic%> qualified non-%<uniform%> argument");
279 /* Given a SIMD clone in NODE, calculate the characteristic data
280 type and return the coresponding type. The characteristic data
281 type is computed as described in the Intel Vector ABI. */
284 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
285 struct cgraph_simd_clone
*clone_info
)
287 tree type
= integer_type_node
;
288 tree fndecl
= node
->decl
;
290 /* a) For non-void function, the characteristic data type is the
292 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
293 type
= TREE_TYPE (TREE_TYPE (fndecl
));
295 /* b) If the function has any non-uniform, non-linear parameters,
296 then the characteristic data type is the type of the first
300 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
301 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
302 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
310 /* c) If the characteristic data type determined by a) or b) above
311 is struct, union, or class type which is pass-by-value (except
312 for the type that maps to the built-in complex data type), the
313 characteristic data type is int. */
314 if (RECORD_OR_UNION_TYPE_P (type
)
315 && !aggregate_value_p (type
, NULL
)
316 && TREE_CODE (type
) != COMPLEX_TYPE
)
317 return integer_type_node
;
319 /* d) If none of the above three classes is applicable, the
320 characteristic data type is int. */
324 /* e) For Intel Xeon Phi native and offload compilation, if the
325 resulting characteristic data type is 8-bit or 16-bit integer
326 data type, the characteristic data type is int. */
327 /* Well, we don't handle Xeon Phi yet. */
331 simd_clone_mangle (struct cgraph_node
*node
,
332 struct cgraph_simd_clone
*clone_info
)
334 char vecsize_mangle
= clone_info
->vecsize_mangle
;
335 char mask
= clone_info
->inbranch
? 'M' : 'N';
336 unsigned int simdlen
= clone_info
->simdlen
;
340 gcc_assert (vecsize_mangle
&& simdlen
);
342 pp_string (&pp
, "_ZGV");
343 pp_character (&pp
, vecsize_mangle
);
344 pp_character (&pp
, mask
);
345 pp_decimal_int (&pp
, simdlen
);
347 for (n
= 0; n
< clone_info
->nargs
; ++n
)
349 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
351 switch (arg
.arg_type
)
353 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
354 pp_character (&pp
, 'u');
356 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
357 pp_character (&pp
, 'l');
359 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
360 pp_character (&pp
, 'R');
362 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
363 pp_character (&pp
, 'L');
365 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
366 pp_character (&pp
, 'U');
369 gcc_assert (arg
.linear_step
!= 0);
370 if (arg
.linear_step
> 1)
371 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
372 else if (arg
.linear_step
< 0)
374 pp_character (&pp
, 'n');
375 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
379 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
380 pp_string (&pp
, "ls");
381 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
383 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
384 pp_string (&pp
, "Rs");
385 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
387 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
388 pp_string (&pp
, "Ls");
389 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
391 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
392 pp_string (&pp
, "Us");
393 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
396 pp_character (&pp
, 'v');
400 pp_character (&pp
, 'a');
401 pp_decimal_int (&pp
, arg
.alignment
);
406 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
409 pp_string (&pp
, str
);
410 str
= pp_formatted_text (&pp
);
412 /* If there already is a SIMD clone with the same mangled name, don't
413 add another one. This can happen e.g. for
414 #pragma omp declare simd
415 #pragma omp declare simd simdlen(8)
417 if the simdlen is assumed to be 8 for the first one, etc. */
418 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
419 clone
= clone
->simdclone
->next_clone
)
420 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
424 return get_identifier (str
);
427 /* Create a simd clone of OLD_NODE and return it. */
429 static struct cgraph_node
*
430 simd_clone_create (struct cgraph_node
*old_node
)
432 struct cgraph_node
*new_node
;
433 if (old_node
->definition
)
435 if (!old_node
->has_gimple_body_p ())
437 old_node
->get_body ();
438 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
444 tree old_decl
= old_node
->decl
;
445 tree new_decl
= copy_node (old_node
->decl
);
446 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
447 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
448 SET_DECL_RTL (new_decl
, NULL
);
449 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
450 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
451 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
452 if (old_node
->in_other_partition
)
453 new_node
->in_other_partition
= 1;
455 if (new_node
== NULL
)
458 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
459 DECL_COMDAT (new_node
->decl
) = DECL_COMDAT (old_node
->decl
);
460 DECL_WEAK (new_node
->decl
) = DECL_WEAK (old_node
->decl
);
461 DECL_EXTERNAL (new_node
->decl
) = DECL_EXTERNAL (old_node
->decl
);
462 DECL_VISIBILITY_SPECIFIED (new_node
->decl
)
463 = DECL_VISIBILITY_SPECIFIED (old_node
->decl
);
464 DECL_VISIBILITY (new_node
->decl
) = DECL_VISIBILITY (old_node
->decl
);
465 DECL_DLLIMPORT_P (new_node
->decl
) = DECL_DLLIMPORT_P (old_node
->decl
);
466 if (DECL_ONE_ONLY (old_node
->decl
))
467 make_decl_one_only (new_node
->decl
, DECL_ASSEMBLER_NAME (new_node
->decl
));
469 /* The method cgraph_version_clone_with_body () will force the new
470 symbol local. Undo this, and inherit external visibility from
472 new_node
->local
.local
= old_node
->local
.local
;
473 new_node
->externally_visible
= old_node
->externally_visible
;
478 /* Adjust the return type of the given function to its appropriate
479 vector counterpart. Returns a simd array to be used throughout the
480 function as a return value. */
483 simd_clone_adjust_return_type (struct cgraph_node
*node
)
485 tree fndecl
= node
->decl
;
486 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
490 /* Adjust the function return type. */
491 if (orig_rettype
== void_type_node
)
493 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
494 t
= TREE_TYPE (TREE_TYPE (fndecl
));
495 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
496 veclen
= node
->simdclone
->vecsize_int
;
498 veclen
= node
->simdclone
->vecsize_float
;
499 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
500 if (veclen
> node
->simdclone
->simdlen
)
501 veclen
= node
->simdclone
->simdlen
;
502 if (POINTER_TYPE_P (t
))
503 t
= pointer_sized_int_node
;
504 if (veclen
== node
->simdclone
->simdlen
)
505 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
508 t
= build_vector_type (t
, veclen
);
509 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
511 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
512 if (!node
->definition
)
515 t
= DECL_RESULT (fndecl
);
516 /* Adjust the DECL_RESULT. */
517 gcc_assert (TREE_TYPE (t
) != void_type_node
);
518 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
521 tree atype
= build_array_type_nelts (orig_rettype
,
522 node
->simdclone
->simdlen
);
523 if (veclen
!= node
->simdclone
->simdlen
)
524 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
526 /* Set up a SIMD array to use as the return value. */
527 tree retval
= create_tmp_var_raw (atype
, "retval");
528 gimple_add_tmp_var (retval
);
532 /* Each vector argument has a corresponding array to be used locally
533 as part of the eventual loop. Create such temporary array and
536 PREFIX is the prefix to be used for the temporary.
538 TYPE is the inner element type.
540 SIMDLEN is the number of elements. */
543 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
545 tree atype
= build_array_type_nelts (type
, simdlen
);
546 tree avar
= create_tmp_var_raw (atype
, prefix
);
547 gimple_add_tmp_var (avar
);
551 /* Modify the function argument types to their corresponding vector
552 counterparts if appropriate. Also, create one array for each simd
553 argument to be used locally when using the function arguments as
556 NODE is the function whose arguments are to be adjusted.
558 Returns an adjustment vector that will be filled describing how the
559 argument types will be adjusted. */
561 static ipa_parm_adjustment_vec
562 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
565 ipa_parm_adjustment_vec adjustments
;
567 if (node
->definition
)
568 args
= ipa_get_vector_of_formal_parms (node
->decl
);
570 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
571 adjustments
.create (args
.length ());
572 unsigned i
, j
, veclen
;
573 struct ipa_parm_adjustment adj
;
574 struct cgraph_simd_clone
*sc
= node
->simdclone
;
576 for (i
= 0; i
< sc
->nargs
; ++i
)
578 memset (&adj
, 0, sizeof (adj
));
580 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
584 sc
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
585 sc
->args
[i
].orig_type
= parm_type
;
587 switch (sc
->args
[i
].arg_type
)
590 /* No adjustment necessary for scalar arguments. */
591 adj
.op
= IPA_PARM_OP_COPY
;
593 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
594 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
595 if (node
->definition
)
596 sc
->args
[i
].simd_array
597 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
598 TREE_TYPE (parm_type
),
600 adj
.op
= IPA_PARM_OP_COPY
;
602 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
603 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
604 case SIMD_CLONE_ARG_TYPE_VECTOR
:
605 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
606 veclen
= sc
->vecsize_int
;
608 veclen
= sc
->vecsize_float
;
609 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
610 if (veclen
> sc
->simdlen
)
611 veclen
= sc
->simdlen
;
612 adj
.arg_prefix
= "simd";
613 if (POINTER_TYPE_P (parm_type
))
614 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
616 adj
.type
= build_vector_type (parm_type
, veclen
);
617 sc
->args
[i
].vector_type
= adj
.type
;
618 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
620 adjustments
.safe_push (adj
);
623 memset (&adj
, 0, sizeof (adj
));
624 adj
.op
= IPA_PARM_OP_NEW
;
625 adj
.arg_prefix
= "simd";
627 adj
.type
= sc
->args
[i
].vector_type
;
631 if (node
->definition
)
632 sc
->args
[i
].simd_array
633 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
634 parm_type
, sc
->simdlen
);
636 adjustments
.safe_push (adj
);
641 tree base_type
= simd_clone_compute_base_data_type (sc
->origin
, sc
);
643 memset (&adj
, 0, sizeof (adj
));
644 adj
.op
= IPA_PARM_OP_NEW
;
645 adj
.arg_prefix
= "mask";
648 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
649 veclen
= sc
->vecsize_int
;
651 veclen
= sc
->vecsize_float
;
652 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
653 if (veclen
> sc
->simdlen
)
654 veclen
= sc
->simdlen
;
655 if (sc
->mask_mode
!= VOIDmode
)
657 = lang_hooks
.types
.type_for_mode (sc
->mask_mode
, 1);
658 else if (POINTER_TYPE_P (base_type
))
659 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
661 adj
.type
= build_vector_type (base_type
, veclen
);
662 adjustments
.safe_push (adj
);
664 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
665 adjustments
.safe_push (adj
);
667 /* We have previously allocated one extra entry for the mask. Use
670 if (sc
->mask_mode
!= VOIDmode
)
671 base_type
= boolean_type_node
;
672 if (node
->definition
)
675 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
676 if (sc
->mask_mode
== VOIDmode
)
677 sc
->args
[i
].simd_array
678 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
679 else if (veclen
< sc
->simdlen
)
680 sc
->args
[i
].simd_array
681 = create_tmp_simd_array ("mask", adj
.type
, sc
->simdlen
/ veclen
);
683 sc
->args
[i
].simd_array
= NULL_TREE
;
685 sc
->args
[i
].orig_type
= base_type
;
686 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
689 if (node
->definition
)
690 ipa_modify_formal_parameters (node
->decl
, adjustments
);
693 tree new_arg_types
= NULL_TREE
, new_reversed
;
694 bool last_parm_void
= false;
695 if (args
.length () > 0 && args
.last () == void_type_node
)
696 last_parm_void
= true;
698 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
699 j
= adjustments
.length ();
700 for (i
= 0; i
< j
; i
++)
702 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
704 if (adj
->op
== IPA_PARM_OP_COPY
)
705 ptype
= args
[adj
->base_index
];
708 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
710 new_reversed
= nreverse (new_arg_types
);
714 TREE_CHAIN (new_arg_types
) = void_list_node
;
716 new_reversed
= void_list_node
;
719 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
720 TYPE_ARG_TYPES (new_type
) = new_reversed
;
721 TREE_TYPE (node
->decl
) = new_type
;
723 adjustments
.release ();
729 /* Initialize and copy the function arguments in NODE to their
730 corresponding local simd arrays. Returns a fresh gimple_seq with
731 the instruction sequence generated. */
734 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
735 ipa_parm_adjustment_vec adjustments
)
737 gimple_seq seq
= NULL
;
738 unsigned i
= 0, j
= 0, k
;
740 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
742 arg
= DECL_CHAIN (arg
), i
++, j
++)
744 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
745 || POINTER_TYPE_P (TREE_TYPE (arg
)))
748 node
->simdclone
->args
[i
].vector_arg
= arg
;
750 tree array
= node
->simdclone
->args
[i
].simd_array
;
751 if (node
->simdclone
->mask_mode
!= VOIDmode
752 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_MASK
)
754 if (array
== NULL_TREE
)
757 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array
))));
758 for (k
= 0; k
<= l
; k
++)
762 arg
= DECL_CHAIN (arg
);
765 tree t
= build4 (ARRAY_REF
, TREE_TYPE (TREE_TYPE (array
)),
766 array
, size_int (k
), NULL
, NULL
);
767 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
768 gimplify_and_add (t
, &seq
);
772 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
774 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
775 tree ptr
= build_fold_addr_expr (array
);
776 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
777 build_int_cst (ptype
, 0));
778 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
779 gimplify_and_add (t
, &seq
);
783 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
784 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
785 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
787 tree ptr
= build_fold_addr_expr (array
);
791 arg
= DECL_CHAIN (arg
);
795 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
796 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
797 build_int_cst (ptype
, k
* elemsize
));
798 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
799 gimplify_and_add (t
, &seq
);
806 /* Callback info for ipa_simd_modify_stmt_ops below. */
808 struct modify_stmt_info
{
809 ipa_parm_adjustment_vec adjustments
;
811 /* True if the parent statement was modified by
812 ipa_simd_modify_stmt_ops. */
816 /* Callback for walk_gimple_op.
818 Adjust operands from a given statement as specified in the
819 adjustments vector in the callback data. */
822 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
824 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
825 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
827 if (TREE_CODE (*tp
) == ADDR_EXPR
)
828 tp
= &TREE_OPERAND (*tp
, 0);
829 struct ipa_parm_adjustment
*cand
= NULL
;
830 if (TREE_CODE (*tp
) == PARM_DECL
)
831 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
838 tree repl
= NULL_TREE
;
840 repl
= unshare_expr (cand
->new_decl
);
846 bool modified
= info
->modified
;
847 info
->modified
= false;
848 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
851 info
->modified
= modified
;
854 info
->modified
= modified
;
863 repl
= build_fold_addr_expr (repl
);
865 if (is_gimple_debug (info
->stmt
))
867 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
868 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
869 DECL_ARTIFICIAL (vexpr
) = 1;
870 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
871 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
876 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
877 repl
= gimple_assign_lhs (stmt
);
879 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
880 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
883 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
885 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
891 info
->modified
= true;
895 /* Traverse the function body and perform all modifications as
896 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
897 modified such that the replacement/reduction value will now be an
898 offset into the corresponding simd_array.
900 This function will replace all function argument uses with their
901 corresponding simd array elements, and ajust the return values
905 ipa_simd_modify_function_body (struct cgraph_node
*node
,
906 ipa_parm_adjustment_vec adjustments
,
907 tree retval_array
, tree iter
)
910 unsigned int i
, j
, l
;
912 /* Re-use the adjustments array, but this time use it to replace
913 every function argument use to an offset into the corresponding
915 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
917 if (!node
->simdclone
->args
[i
].vector_arg
)
920 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
921 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
922 adjustments
[j
].new_decl
925 node
->simdclone
->args
[i
].simd_array
,
927 NULL_TREE
, NULL_TREE
);
928 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
929 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
930 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
933 l
= adjustments
.length ();
936 FOR_EACH_SSA_NAME (i
, name
, cfun
)
938 if (SSA_NAME_VAR (name
)
939 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
941 for (j
= 0; j
< l
; j
++)
942 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
943 && adjustments
[j
].new_decl
)
946 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
949 = copy_var_decl (adjustments
[j
].base
,
950 DECL_NAME (adjustments
[j
].base
),
951 TREE_TYPE (adjustments
[j
].base
));
952 adjustments
[j
].new_ssa_base
= base_var
;
955 base_var
= adjustments
[j
].new_ssa_base
;
956 if (SSA_NAME_IS_DEFAULT_DEF (name
))
958 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
959 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
960 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
961 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
962 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
963 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
964 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
965 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
968 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
973 struct modify_stmt_info info
;
974 info
.adjustments
= adjustments
;
976 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
978 gimple_stmt_iterator gsi
;
980 gsi
= gsi_start_bb (bb
);
981 while (!gsi_end_p (gsi
))
983 gimple
*stmt
= gsi_stmt (gsi
);
985 struct walk_stmt_info wi
;
987 memset (&wi
, 0, sizeof (wi
));
988 info
.modified
= false;
990 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
992 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
994 tree retval
= gimple_return_retval (return_stmt
);
997 gsi_remove (&gsi
, true);
1001 /* Replace `return foo' with `retval_array[iter] = foo'. */
1002 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
1003 retval_array
, iter
, NULL
, NULL
);
1004 stmt
= gimple_build_assign (ref
, retval
);
1005 gsi_replace (&gsi
, stmt
, true);
1006 info
.modified
= true;
1012 if (maybe_clean_eh_stmt (stmt
))
1013 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
1020 /* Helper function of simd_clone_adjust, return linear step addend
1024 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
1025 tree addtype
, basic_block entry_bb
)
1027 tree ptype
= NULL_TREE
;
1028 switch (node
->simdclone
->args
[i
].arg_type
)
1030 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
1031 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
1032 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
1033 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
1034 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
1035 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
1036 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
1037 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
1039 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
1040 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
1041 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
1047 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
1048 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
1049 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
1050 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1053 if (is_gimple_reg (arg
))
1054 ret
= get_or_create_ssa_default_def (cfun
, arg
);
1057 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
1058 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1059 ret
= gimple_assign_lhs (g
);
1061 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
1063 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
1064 build_simple_mem_ref (ret
));
1065 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1066 ret
= gimple_assign_lhs (g
);
1068 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
1070 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
1071 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1072 ret
= gimple_assign_lhs (g
);
1074 if (POINTER_TYPE_P (ptype
))
1076 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
1077 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
1079 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
1080 ret
, fold_convert (addtype
, size
));
1081 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1082 ret
= gimple_assign_lhs (g
);
1088 /* Adjust the argument types in NODE to their appropriate vector
1092 simd_clone_adjust (struct cgraph_node
*node
)
1094 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1096 targetm
.simd_clone
.adjust (node
);
1098 tree retval
= simd_clone_adjust_return_type (node
);
1099 ipa_parm_adjustment_vec adjustments
1100 = simd_clone_adjust_argument_types (node
);
1102 push_gimplify_context ();
1104 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
1106 /* Adjust all uses of vector arguments accordingly. Adjust all
1107 return values accordingly. */
1108 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
1109 tree iter1
= make_ssa_name (iter
);
1110 tree iter2
= make_ssa_name (iter
);
1111 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
1112 adjustments
.release ();
1114 /* Initialize the iteration variable. */
1115 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1116 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
1117 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1118 /* Insert the SIMD array and iv initialization at function
1120 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
1122 pop_gimplify_context (NULL
);
1124 /* Create a new BB right before the original exit BB, to hold the
1125 iteration increment and the condition/branch. */
1126 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
1127 basic_block incr_bb
= create_empty_bb (orig_exit
);
1128 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1129 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
1130 flag. Set it now to be a FALLTHRU_EDGE. */
1131 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
1132 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
1133 for (unsigned i
= 0;
1134 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
1136 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
1137 redirect_edge_succ (e
, incr_bb
);
1139 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
1140 e
->probability
= REG_BR_PROB_BASE
;
1141 gsi
= gsi_last_bb (incr_bb
);
1142 gimple
*g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
1143 build_int_cst (unsigned_type_node
, 1));
1144 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1146 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
1147 struct loop
*loop
= alloc_loop ();
1148 cfun
->has_force_vectorize_loops
= true;
1149 loop
->safelen
= node
->simdclone
->simdlen
;
1150 loop
->force_vectorize
= true;
1151 loop
->header
= body_bb
;
1153 /* Branch around the body if the mask applies. */
1154 if (node
->simdclone
->inbranch
)
1156 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
1158 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
1160 if (node
->simdclone
->mask_mode
!= VOIDmode
)
1163 if (mask_array
== NULL_TREE
)
1165 tree arg
= node
->simdclone
->args
[node
->simdclone
->nargs
1167 mask
= get_or_create_ssa_default_def (cfun
, arg
);
1172 tree maskt
= TREE_TYPE (mask_array
);
1173 int c
= tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt
)));
1174 c
= node
->simdclone
->simdlen
/ (c
+ 1);
1175 int s
= exact_log2 (c
);
1178 tree idx
= make_ssa_name (TREE_TYPE (iter1
));
1179 g
= gimple_build_assign (idx
, RSHIFT_EXPR
, iter1
,
1180 build_int_cst (NULL_TREE
, s
));
1181 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1182 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1183 tree aref
= build4 (ARRAY_REF
,
1184 TREE_TYPE (TREE_TYPE (mask_array
)),
1185 mask_array
, idx
, NULL
, NULL
);
1186 g
= gimple_build_assign (mask
, aref
);
1187 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1188 shift_cnt
= make_ssa_name (TREE_TYPE (iter1
));
1189 g
= gimple_build_assign (shift_cnt
, BIT_AND_EXPR
, iter1
,
1190 build_int_cst (TREE_TYPE (iter1
), c
));
1191 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1193 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1194 RSHIFT_EXPR
, mask
, shift_cnt
);
1195 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1196 mask
= gimple_assign_lhs (g
);
1197 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1199 build_int_cst (TREE_TYPE (mask
), 1));
1200 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1201 mask
= gimple_assign_lhs (g
);
1205 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1206 tree aref
= build4 (ARRAY_REF
,
1207 TREE_TYPE (TREE_TYPE (mask_array
)),
1208 mask_array
, iter1
, NULL
, NULL
);
1209 g
= gimple_build_assign (mask
, aref
);
1210 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1211 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
1212 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
1214 aref
= build1 (VIEW_CONVERT_EXPR
,
1215 build_nonstandard_integer_type (bitsize
, 0),
1217 mask
= make_ssa_name (TREE_TYPE (aref
));
1218 g
= gimple_build_assign (mask
, aref
);
1219 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1223 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
1225 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1226 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
1227 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
1230 /* Generate the condition. */
1231 g
= gimple_build_cond (LT_EXPR
,
1233 build_int_cst (unsigned_type_node
,
1234 node
->simdclone
->simdlen
),
1236 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1237 e
= split_block (incr_bb
, gsi_stmt (gsi
));
1238 basic_block latch_bb
= e
->dest
;
1239 basic_block new_exit_bb
;
1240 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
1241 loop
->latch
= latch_bb
;
1243 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
1245 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
1246 /* The successor of incr_bb is already pointing to latch_bb; just
1248 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1249 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
1251 gphi
*phi
= create_phi_node (iter1
, body_bb
);
1252 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
1253 edge latch_edge
= single_succ_edge (latch_bb
);
1254 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
1256 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1258 /* Generate the new return. */
1259 gsi
= gsi_last_bb (new_exit_bb
);
1261 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
1262 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
1263 retval
= TREE_OPERAND (retval
, 0);
1266 retval
= build1 (VIEW_CONVERT_EXPR
,
1267 TREE_TYPE (TREE_TYPE (node
->decl
)),
1269 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
1270 false, GSI_CONTINUE_LINKING
);
1272 g
= gimple_build_return (retval
);
1273 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1275 /* Handle aligned clauses by replacing default defs of the aligned
1276 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1277 lhs. Handle linear by adding PHIs. */
1278 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
1279 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1280 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
1281 || !is_gimple_reg_type
1282 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1284 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1285 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
1286 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1289 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
1290 gimple_add_tmp_var (iter1
);
1292 gsi
= gsi_after_labels (entry_bb
);
1293 g
= gimple_build_assign (iter1
, orig_arg
);
1294 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1295 gsi
= gsi_after_labels (body_bb
);
1296 g
= gimple_build_assign (orig_arg
, iter1
);
1297 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1299 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1300 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
1301 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1304 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1306 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1307 tree def
= ssa_default_def (cfun
, orig_arg
);
1308 if (def
&& !has_zero_uses (def
))
1310 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
1311 gimple_add_tmp_var (iter1
);
1312 gsi
= gsi_after_labels (entry_bb
);
1313 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
1314 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1315 gsi
= gsi_after_labels (body_bb
);
1316 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
1317 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1320 else if (node
->simdclone
->args
[i
].alignment
1321 && node
->simdclone
->args
[i
].arg_type
1322 == SIMD_CLONE_ARG_TYPE_UNIFORM
1323 && (node
->simdclone
->args
[i
].alignment
1324 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
1325 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1328 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
1329 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1330 tree def
= ssa_default_def (cfun
, orig_arg
);
1331 if (def
&& !has_zero_uses (def
))
1333 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
1334 gimple_seq seq
= NULL
;
1335 bool need_cvt
= false;
1337 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
1339 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
1342 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
1343 gimple_call_set_lhs (g
, t
);
1344 gimple_seq_add_stmt_without_update (&seq
, g
);
1347 t
= make_ssa_name (orig_arg
);
1348 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
1349 gimple_seq_add_stmt_without_update (&seq
, g
);
1351 gsi_insert_seq_on_edge_immediate
1352 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
1354 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1355 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
1357 node
->create_edge (cgraph_node::get_create (fn
),
1358 call
, entry_bb
->count
, freq
);
1360 imm_use_iterator iter
;
1361 use_operand_p use_p
;
1363 tree repl
= gimple_get_lhs (g
);
1364 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1365 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
1368 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1369 SET_USE (use_p
, repl
);
1372 else if ((node
->simdclone
->args
[i
].arg_type
1373 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
1374 || (node
->simdclone
->args
[i
].arg_type
1375 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
1376 || (node
->simdclone
->args
[i
].arg_type
1377 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
1378 || (node
->simdclone
->args
[i
].arg_type
1379 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
1381 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1382 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1383 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
1384 tree def
= NULL_TREE
;
1385 if (TREE_ADDRESSABLE (orig_arg
))
1387 def
= make_ssa_name (TREE_TYPE (orig_arg
));
1388 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1389 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
1390 gsi
= gsi_after_labels (entry_bb
);
1391 g
= gimple_build_assign (def
, orig_arg
);
1392 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1396 def
= ssa_default_def (cfun
, orig_arg
);
1397 if (!def
|| has_zero_uses (def
))
1401 iter1
= make_ssa_name (orig_arg
);
1402 iter2
= make_ssa_name (orig_arg
);
1407 phi
= create_phi_node (iter1
, body_bb
);
1408 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
1409 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1410 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1411 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1412 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1413 ? TREE_TYPE (orig_arg
) : sizetype
;
1414 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1416 gsi
= gsi_last_bb (incr_bb
);
1417 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
1418 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1420 imm_use_iterator iter
;
1421 use_operand_p use_p
;
1423 if (TREE_ADDRESSABLE (orig_arg
))
1425 gsi
= gsi_after_labels (body_bb
);
1426 g
= gimple_build_assign (orig_arg
, iter1
);
1427 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1430 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1431 if (use_stmt
== phi
)
1434 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1435 SET_USE (use_p
, iter1
);
1438 else if (node
->simdclone
->args
[i
].arg_type
1439 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1440 || (node
->simdclone
->args
[i
].arg_type
1441 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
1443 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1444 tree def
= ssa_default_def (cfun
, orig_arg
);
1445 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
1446 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
1447 if (def
&& !has_zero_uses (def
))
1449 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
1450 iter1
= make_ssa_name (orig_arg
);
1451 iter2
= make_ssa_name (orig_arg
);
1452 tree iter3
= make_ssa_name (rtype
);
1453 tree iter4
= make_ssa_name (rtype
);
1454 tree iter5
= make_ssa_name (rtype
);
1455 gsi
= gsi_after_labels (entry_bb
);
1457 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
1458 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
1460 tree array
= node
->simdclone
->args
[i
].simd_array
;
1461 TREE_ADDRESSABLE (array
) = 1;
1462 tree ptr
= build_fold_addr_expr (array
);
1463 phi
= create_phi_node (iter1
, body_bb
);
1464 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
1465 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1466 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
1467 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
1468 gsi
= gsi_last_bb (incr_bb
);
1469 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1471 phi
= create_phi_node (iter4
, body_bb
);
1472 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
1473 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
1474 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1475 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1476 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1477 ? TREE_TYPE (iter3
) : sizetype
;
1478 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1480 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
1481 gsi
= gsi_last_bb (incr_bb
);
1482 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1484 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
1485 gsi
= gsi_after_labels (body_bb
);
1486 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1488 imm_use_iterator iter
;
1489 use_operand_p use_p
;
1491 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1492 if (use_stmt
== load
)
1495 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1496 SET_USE (use_p
, iter1
);
1498 if (!TYPE_READONLY (rtype
))
1500 tree v
= make_ssa_name (rtype
);
1501 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
1502 size_zero_node
, NULL_TREE
,
1504 gsi
= gsi_after_labels (new_exit_bb
);
1505 g
= gimple_build_assign (v
, aref
);
1506 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1507 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
1508 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1513 calculate_dominance_info (CDI_DOMINATORS
);
1514 add_loop (loop
, loop
->header
->loop_father
);
1515 update_ssa (TODO_update_ssa
);
1520 /* If the function in NODE is tagged as an elemental SIMD function,
1521 create the appropriate SIMD clones. */
1524 expand_simd_clones (struct cgraph_node
*node
)
1526 tree attr
= lookup_attribute ("omp declare simd",
1527 DECL_ATTRIBUTES (node
->decl
));
1528 if (attr
== NULL_TREE
1529 || node
->global
.inlined_to
1530 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
1534 #pragma omp declare simd
1536 in C, there we don't know the argument types at all. */
1537 if (!node
->definition
1538 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
1541 /* Call this before creating clone_info, as it might ggc_collect. */
1542 if (node
->definition
&& node
->has_gimple_body_p ())
1547 /* Start with parsing the "omp declare simd" attribute(s). */
1548 bool inbranch_clause_specified
;
1549 struct cgraph_simd_clone
*clone_info
1550 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
1551 &inbranch_clause_specified
);
1552 if (clone_info
== NULL
)
1555 int orig_simdlen
= clone_info
->simdlen
;
1556 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
1557 /* The target can return 0 (no simd clones should be created),
1558 1 (just one ISA of simd clones should be created) or higher
1559 count of ISA variants. In that case, clone_info is initialized
1560 for the first ISA variant. */
1562 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
1567 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1568 also create one inbranch and one !inbranch clone of it. */
1569 for (int i
= 0; i
< count
* 2; i
++)
1571 struct cgraph_simd_clone
*clone
= clone_info
;
1572 if (inbranch_clause_specified
&& (i
& 1) != 0)
1577 clone
= simd_clone_struct_alloc (clone_info
->nargs
1579 simd_clone_struct_copy (clone
, clone_info
);
1580 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1581 and simd_clone_adjust_argument_types did to the first
1583 clone
->nargs
-= clone_info
->inbranch
;
1584 clone
->simdlen
= orig_simdlen
;
1585 /* And call the target hook again to get the right ISA. */
1586 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
1590 clone
->inbranch
= 1;
1593 /* simd_clone_mangle might fail if such a clone has been created
1595 tree id
= simd_clone_mangle (node
, clone
);
1596 if (id
== NULL_TREE
)
1599 /* Only when we are sure we want to create the clone actually
1600 clone the function (or definitions) or create another
1601 extern FUNCTION_DECL (for prototypes without definitions). */
1602 struct cgraph_node
*n
= simd_clone_create (node
);
1606 n
->simdclone
= clone
;
1607 clone
->origin
= node
;
1608 clone
->next_clone
= NULL
;
1609 if (node
->simd_clones
== NULL
)
1611 clone
->prev_clone
= n
;
1612 node
->simd_clones
= n
;
1616 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
1617 clone
->prev_clone
->simdclone
->next_clone
= n
;
1618 node
->simd_clones
->simdclone
->prev_clone
= n
;
1620 symtab
->change_decl_assembler_name (n
->decl
, id
);
1621 /* And finally adjust the return type, parameters and for
1622 definitions also function body. */
1623 if (node
->definition
)
1624 simd_clone_adjust (n
);
1627 simd_clone_adjust_return_type (n
);
1628 simd_clone_adjust_argument_types (n
);
1632 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
1635 /* Entry point for IPA simd clone creation pass. */
1638 ipa_omp_simd_clone (void)
1640 struct cgraph_node
*node
;
1641 FOR_EACH_FUNCTION (node
)
1642 expand_simd_clones (node
);
1648 const pass_data pass_data_omp_simd_clone
=
1650 SIMPLE_IPA_PASS
, /* type */
1651 "simdclone", /* name */
1652 OPTGROUP_NONE
, /* optinfo_flags */
1653 TV_NONE
, /* tv_id */
1654 ( PROP_ssa
| PROP_cfg
), /* properties_required */
1655 0, /* properties_provided */
1656 0, /* properties_destroyed */
1657 0, /* todo_flags_start */
1658 0, /* todo_flags_finish */
1661 class pass_omp_simd_clone
: public simple_ipa_opt_pass
1664 pass_omp_simd_clone(gcc::context
*ctxt
)
1665 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
1668 /* opt_pass methods: */
1669 virtual bool gate (function
*);
1670 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
1674 pass_omp_simd_clone::gate (function
*)
1676 return targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
;
1681 simple_ipa_opt_pass
*
1682 make_pass_omp_simd_clone (gcc::context
*ctxt
)
1684 return new pass_omp_simd_clone (ctxt
);