Daily bump.
[official-gcc.git] / gcc / omp-simd-clone.c
blob99589d44664609cb5ff59ca3e0d61c9759d92400
1 /* OMP constructs' SIMD clone supporting code.
3 Copyright (C) 2005-2017 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "cfghooks.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
31 #include "ssa.h"
32 #include "cgraph.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
37 #include "cfganal.h"
38 #include "gimplify.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
43 #include "tree-cfg.h"
44 #include "tree-into-ssa.h"
45 #include "tree-dfa.h"
46 #include "cfgloop.h"
47 #include "symbol-summary.h"
48 #include "ipa-prop.h"
49 #include "tree-eh.h"
50 #include "varasm.h"
53 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
54 of arguments to reserve space for. */
56 static struct cgraph_simd_clone *
57 simd_clone_struct_alloc (int nargs)
59 struct cgraph_simd_clone *clone_info;
60 size_t len = (sizeof (struct cgraph_simd_clone)
61 + nargs * sizeof (struct cgraph_simd_clone_arg));
62 clone_info = (struct cgraph_simd_clone *)
63 ggc_internal_cleared_alloc (len);
64 return clone_info;
67 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
69 static inline void
70 simd_clone_struct_copy (struct cgraph_simd_clone *to,
71 struct cgraph_simd_clone *from)
73 memcpy (to, from, (sizeof (struct cgraph_simd_clone)
74 + ((from->nargs - from->inbranch)
75 * sizeof (struct cgraph_simd_clone_arg))));
78 /* Return vector of parameter types of function FNDECL. This uses
79 TYPE_ARG_TYPES if available, otherwise falls back to types of
80 DECL_ARGUMENTS types. */
82 static vec<tree>
83 simd_clone_vector_of_formal_parm_types (tree fndecl)
85 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
86 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl));
87 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl);
88 unsigned int i;
89 tree arg;
90 FOR_EACH_VEC_ELT (args, i, arg)
91 args[i] = TREE_TYPE (args[i]);
92 return args;
95 /* Given a simd function in NODE, extract the simd specific
96 information from the OMP clauses passed in CLAUSES, and return
97 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
98 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
99 otherwise set to FALSE. */
101 static struct cgraph_simd_clone *
102 simd_clone_clauses_extract (struct cgraph_node *node, tree clauses,
103 bool *inbranch_specified)
105 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl);
106 tree t;
107 int n;
108 *inbranch_specified = false;
110 n = args.length ();
111 if (n > 0 && args.last () == void_type_node)
112 n--;
114 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
115 be cloned have a distinctive artificial label in addition to "omp
116 declare simd". */
117 bool cilk_clone
118 = (flag_cilkplus
119 && lookup_attribute ("cilk simd function",
120 DECL_ATTRIBUTES (node->decl)));
122 /* Allocate one more than needed just in case this is an in-branch
123 clone which will require a mask argument. */
124 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1);
125 clone_info->nargs = n;
126 clone_info->cilk_elemental = cilk_clone;
128 if (!clauses)
129 goto out;
131 clauses = TREE_VALUE (clauses);
132 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE)
133 goto out;
135 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t))
137 switch (OMP_CLAUSE_CODE (t))
139 case OMP_CLAUSE_INBRANCH:
140 clone_info->inbranch = 1;
141 *inbranch_specified = true;
142 break;
143 case OMP_CLAUSE_NOTINBRANCH:
144 clone_info->inbranch = 0;
145 *inbranch_specified = true;
146 break;
147 case OMP_CLAUSE_SIMDLEN:
148 clone_info->simdlen
149 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t));
150 break;
151 case OMP_CLAUSE_LINEAR:
153 tree decl = OMP_CLAUSE_DECL (t);
154 tree step = OMP_CLAUSE_LINEAR_STEP (t);
155 int argno = TREE_INT_CST_LOW (decl);
156 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t))
158 enum cgraph_simd_clone_arg_type arg_type;
159 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
160 switch (OMP_CLAUSE_LINEAR_KIND (t))
162 case OMP_CLAUSE_LINEAR_REF:
163 arg_type
164 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP;
165 break;
166 case OMP_CLAUSE_LINEAR_UVAL:
167 arg_type
168 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP;
169 break;
170 case OMP_CLAUSE_LINEAR_VAL:
171 case OMP_CLAUSE_LINEAR_DEFAULT:
172 arg_type
173 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP;
174 break;
175 default:
176 gcc_unreachable ();
178 else
179 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP;
180 clone_info->args[argno].arg_type = arg_type;
181 clone_info->args[argno].linear_step = tree_to_shwi (step);
182 gcc_assert (clone_info->args[argno].linear_step >= 0
183 && clone_info->args[argno].linear_step < n);
185 else
187 if (POINTER_TYPE_P (args[argno]))
188 step = fold_convert (ssizetype, step);
189 if (!tree_fits_shwi_p (step))
191 warning_at (OMP_CLAUSE_LOCATION (t), 0,
192 "ignoring large linear step");
193 args.release ();
194 return NULL;
196 else if (integer_zerop (step))
198 warning_at (OMP_CLAUSE_LOCATION (t), 0,
199 "ignoring zero linear step");
200 args.release ();
201 return NULL;
203 else
205 enum cgraph_simd_clone_arg_type arg_type;
206 if (TREE_CODE (args[argno]) == REFERENCE_TYPE)
207 switch (OMP_CLAUSE_LINEAR_KIND (t))
209 case OMP_CLAUSE_LINEAR_REF:
210 arg_type
211 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP;
212 break;
213 case OMP_CLAUSE_LINEAR_UVAL:
214 arg_type
215 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP;
216 break;
217 case OMP_CLAUSE_LINEAR_VAL:
218 case OMP_CLAUSE_LINEAR_DEFAULT:
219 arg_type
220 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP;
221 break;
222 default:
223 gcc_unreachable ();
225 else
226 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP;
227 clone_info->args[argno].arg_type = arg_type;
228 clone_info->args[argno].linear_step = tree_to_shwi (step);
231 break;
233 case OMP_CLAUSE_UNIFORM:
235 tree decl = OMP_CLAUSE_DECL (t);
236 int argno = tree_to_uhwi (decl);
237 clone_info->args[argno].arg_type
238 = SIMD_CLONE_ARG_TYPE_UNIFORM;
239 break;
241 case OMP_CLAUSE_ALIGNED:
243 tree decl = OMP_CLAUSE_DECL (t);
244 int argno = tree_to_uhwi (decl);
245 clone_info->args[argno].alignment
246 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t));
247 break;
249 default:
250 break;
254 out:
255 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl))))
257 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
258 "ignoring %<#pragma omp declare simd%> on function "
259 "with %<_Atomic%> qualified return type");
260 args.release ();
261 return NULL;
264 for (unsigned int argno = 0; argno < clone_info->nargs; argno++)
265 if (TYPE_ATOMIC (args[argno])
266 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM)
268 warning_at (DECL_SOURCE_LOCATION (node->decl), 0,
269 "ignoring %<#pragma omp declare simd%> on function "
270 "with %<_Atomic%> qualified non-%<uniform%> argument");
271 args.release ();
272 return NULL;
275 args.release ();
276 return clone_info;
279 /* Given a SIMD clone in NODE, calculate the characteristic data
280 type and return the coresponding type. The characteristic data
281 type is computed as described in the Intel Vector ABI. */
283 static tree
284 simd_clone_compute_base_data_type (struct cgraph_node *node,
285 struct cgraph_simd_clone *clone_info)
287 tree type = integer_type_node;
288 tree fndecl = node->decl;
290 /* a) For non-void function, the characteristic data type is the
291 return type. */
292 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE)
293 type = TREE_TYPE (TREE_TYPE (fndecl));
295 /* b) If the function has any non-uniform, non-linear parameters,
296 then the characteristic data type is the type of the first
297 such parameter. */
298 else
300 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl);
301 for (unsigned int i = 0; i < clone_info->nargs; ++i)
302 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
304 type = map[i];
305 break;
307 map.release ();
310 /* c) If the characteristic data type determined by a) or b) above
311 is struct, union, or class type which is pass-by-value (except
312 for the type that maps to the built-in complex data type), the
313 characteristic data type is int. */
314 if (RECORD_OR_UNION_TYPE_P (type)
315 && !aggregate_value_p (type, NULL)
316 && TREE_CODE (type) != COMPLEX_TYPE)
317 return integer_type_node;
319 /* d) If none of the above three classes is applicable, the
320 characteristic data type is int. */
322 return type;
324 /* e) For Intel Xeon Phi native and offload compilation, if the
325 resulting characteristic data type is 8-bit or 16-bit integer
326 data type, the characteristic data type is int. */
327 /* Well, we don't handle Xeon Phi yet. */
330 static tree
331 simd_clone_mangle (struct cgraph_node *node,
332 struct cgraph_simd_clone *clone_info)
334 char vecsize_mangle = clone_info->vecsize_mangle;
335 char mask = clone_info->inbranch ? 'M' : 'N';
336 unsigned int simdlen = clone_info->simdlen;
337 unsigned int n;
338 pretty_printer pp;
340 gcc_assert (vecsize_mangle && simdlen);
342 pp_string (&pp, "_ZGV");
343 pp_character (&pp, vecsize_mangle);
344 pp_character (&pp, mask);
345 pp_decimal_int (&pp, simdlen);
347 for (n = 0; n < clone_info->nargs; ++n)
349 struct cgraph_simd_clone_arg arg = clone_info->args[n];
351 switch (arg.arg_type)
353 case SIMD_CLONE_ARG_TYPE_UNIFORM:
354 pp_character (&pp, 'u');
355 break;
356 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
357 pp_character (&pp, 'l');
358 goto mangle_linear;
359 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
360 pp_character (&pp, 'R');
361 goto mangle_linear;
362 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
363 pp_character (&pp, 'L');
364 goto mangle_linear;
365 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
366 pp_character (&pp, 'U');
367 goto mangle_linear;
368 mangle_linear:
369 gcc_assert (arg.linear_step != 0);
370 if (arg.linear_step > 1)
371 pp_unsigned_wide_integer (&pp, arg.linear_step);
372 else if (arg.linear_step < 0)
374 pp_character (&pp, 'n');
375 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT)
376 arg.linear_step));
378 break;
379 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
380 pp_string (&pp, "ls");
381 pp_unsigned_wide_integer (&pp, arg.linear_step);
382 break;
383 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
384 pp_string (&pp, "Rs");
385 pp_unsigned_wide_integer (&pp, arg.linear_step);
386 break;
387 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
388 pp_string (&pp, "Ls");
389 pp_unsigned_wide_integer (&pp, arg.linear_step);
390 break;
391 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
392 pp_string (&pp, "Us");
393 pp_unsigned_wide_integer (&pp, arg.linear_step);
394 break;
395 default:
396 pp_character (&pp, 'v');
398 if (arg.alignment)
400 pp_character (&pp, 'a');
401 pp_decimal_int (&pp, arg.alignment);
405 pp_underscore (&pp);
406 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl));
407 if (*str == '*')
408 ++str;
409 pp_string (&pp, str);
410 str = pp_formatted_text (&pp);
412 /* If there already is a SIMD clone with the same mangled name, don't
413 add another one. This can happen e.g. for
414 #pragma omp declare simd
415 #pragma omp declare simd simdlen(8)
416 int foo (int, int);
417 if the simdlen is assumed to be 8 for the first one, etc. */
418 for (struct cgraph_node *clone = node->simd_clones; clone;
419 clone = clone->simdclone->next_clone)
420 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)),
421 str) == 0)
422 return NULL_TREE;
424 return get_identifier (str);
427 /* Create a simd clone of OLD_NODE and return it. */
429 static struct cgraph_node *
430 simd_clone_create (struct cgraph_node *old_node)
432 struct cgraph_node *new_node;
433 if (old_node->definition)
435 if (!old_node->has_gimple_body_p ())
436 return NULL;
437 old_node->get_body ();
438 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL,
439 false, NULL, NULL,
440 "simdclone");
442 else
444 tree old_decl = old_node->decl;
445 tree new_decl = copy_node (old_node->decl);
446 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone");
447 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl));
448 SET_DECL_RTL (new_decl, NULL);
449 DECL_STATIC_CONSTRUCTOR (new_decl) = 0;
450 DECL_STATIC_DESTRUCTOR (new_decl) = 0;
451 new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
452 if (old_node->in_other_partition)
453 new_node->in_other_partition = 1;
455 if (new_node == NULL)
456 return new_node;
458 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl);
459 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl);
460 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl);
461 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl);
462 DECL_VISIBILITY_SPECIFIED (new_node->decl)
463 = DECL_VISIBILITY_SPECIFIED (old_node->decl);
464 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl);
465 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl);
466 if (DECL_ONE_ONLY (old_node->decl))
467 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl));
469 /* The method cgraph_version_clone_with_body () will force the new
470 symbol local. Undo this, and inherit external visibility from
471 the old node. */
472 new_node->local.local = old_node->local.local;
473 new_node->externally_visible = old_node->externally_visible;
475 return new_node;
478 /* Adjust the return type of the given function to its appropriate
479 vector counterpart. Returns a simd array to be used throughout the
480 function as a return value. */
482 static tree
483 simd_clone_adjust_return_type (struct cgraph_node *node)
485 tree fndecl = node->decl;
486 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl));
487 unsigned int veclen;
488 tree t;
490 /* Adjust the function return type. */
491 if (orig_rettype == void_type_node)
492 return NULL_TREE;
493 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl));
494 t = TREE_TYPE (TREE_TYPE (fndecl));
495 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t))
496 veclen = node->simdclone->vecsize_int;
497 else
498 veclen = node->simdclone->vecsize_float;
499 veclen /= GET_MODE_BITSIZE (TYPE_MODE (t));
500 if (veclen > node->simdclone->simdlen)
501 veclen = node->simdclone->simdlen;
502 if (POINTER_TYPE_P (t))
503 t = pointer_sized_int_node;
504 if (veclen == node->simdclone->simdlen)
505 t = build_vector_type (t, node->simdclone->simdlen);
506 else
508 t = build_vector_type (t, veclen);
509 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen);
511 TREE_TYPE (TREE_TYPE (fndecl)) = t;
512 if (!node->definition)
513 return NULL_TREE;
515 t = DECL_RESULT (fndecl);
516 /* Adjust the DECL_RESULT. */
517 gcc_assert (TREE_TYPE (t) != void_type_node);
518 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl));
519 relayout_decl (t);
521 tree atype = build_array_type_nelts (orig_rettype,
522 node->simdclone->simdlen);
523 if (veclen != node->simdclone->simdlen)
524 return build1 (VIEW_CONVERT_EXPR, atype, t);
526 /* Set up a SIMD array to use as the return value. */
527 tree retval = create_tmp_var_raw (atype, "retval");
528 gimple_add_tmp_var (retval);
529 return retval;
532 /* Each vector argument has a corresponding array to be used locally
533 as part of the eventual loop. Create such temporary array and
534 return it.
536 PREFIX is the prefix to be used for the temporary.
538 TYPE is the inner element type.
540 SIMDLEN is the number of elements. */
542 static tree
543 create_tmp_simd_array (const char *prefix, tree type, int simdlen)
545 tree atype = build_array_type_nelts (type, simdlen);
546 tree avar = create_tmp_var_raw (atype, prefix);
547 gimple_add_tmp_var (avar);
548 return avar;
551 /* Modify the function argument types to their corresponding vector
552 counterparts if appropriate. Also, create one array for each simd
553 argument to be used locally when using the function arguments as
554 part of the loop.
556 NODE is the function whose arguments are to be adjusted.
558 Returns an adjustment vector that will be filled describing how the
559 argument types will be adjusted. */
561 static ipa_parm_adjustment_vec
562 simd_clone_adjust_argument_types (struct cgraph_node *node)
564 vec<tree> args;
565 ipa_parm_adjustment_vec adjustments;
567 if (node->definition)
568 args = ipa_get_vector_of_formal_parms (node->decl);
569 else
570 args = simd_clone_vector_of_formal_parm_types (node->decl);
571 adjustments.create (args.length ());
572 unsigned i, j, veclen;
573 struct ipa_parm_adjustment adj;
574 struct cgraph_simd_clone *sc = node->simdclone;
576 for (i = 0; i < sc->nargs; ++i)
578 memset (&adj, 0, sizeof (adj));
579 tree parm = args[i];
580 tree parm_type = node->definition ? TREE_TYPE (parm) : parm;
581 adj.base_index = i;
582 adj.base = parm;
584 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE;
585 sc->args[i].orig_type = parm_type;
587 switch (sc->args[i].arg_type)
589 default:
590 /* No adjustment necessary for scalar arguments. */
591 adj.op = IPA_PARM_OP_COPY;
592 break;
593 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
594 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
595 if (node->definition)
596 sc->args[i].simd_array
597 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)),
598 TREE_TYPE (parm_type),
599 sc->simdlen);
600 adj.op = IPA_PARM_OP_COPY;
601 break;
602 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
603 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
604 case SIMD_CLONE_ARG_TYPE_VECTOR:
605 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type))
606 veclen = sc->vecsize_int;
607 else
608 veclen = sc->vecsize_float;
609 veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type));
610 if (veclen > sc->simdlen)
611 veclen = sc->simdlen;
612 adj.arg_prefix = "simd";
613 if (POINTER_TYPE_P (parm_type))
614 adj.type = build_vector_type (pointer_sized_int_node, veclen);
615 else
616 adj.type = build_vector_type (parm_type, veclen);
617 sc->args[i].vector_type = adj.type;
618 for (j = veclen; j < sc->simdlen; j += veclen)
620 adjustments.safe_push (adj);
621 if (j == veclen)
623 memset (&adj, 0, sizeof (adj));
624 adj.op = IPA_PARM_OP_NEW;
625 adj.arg_prefix = "simd";
626 adj.base_index = i;
627 adj.type = sc->args[i].vector_type;
631 if (node->definition)
632 sc->args[i].simd_array
633 = create_tmp_simd_array (DECL_NAME (parm)
634 ? IDENTIFIER_POINTER (DECL_NAME (parm))
635 : NULL, parm_type, sc->simdlen);
637 adjustments.safe_push (adj);
640 if (sc->inbranch)
642 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc);
644 memset (&adj, 0, sizeof (adj));
645 adj.op = IPA_PARM_OP_NEW;
646 adj.arg_prefix = "mask";
648 adj.base_index = i;
649 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type))
650 veclen = sc->vecsize_int;
651 else
652 veclen = sc->vecsize_float;
653 veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type));
654 if (veclen > sc->simdlen)
655 veclen = sc->simdlen;
656 if (sc->mask_mode != VOIDmode)
657 adj.type
658 = lang_hooks.types.type_for_mode (sc->mask_mode, 1);
659 else if (POINTER_TYPE_P (base_type))
660 adj.type = build_vector_type (pointer_sized_int_node, veclen);
661 else
662 adj.type = build_vector_type (base_type, veclen);
663 adjustments.safe_push (adj);
665 for (j = veclen; j < sc->simdlen; j += veclen)
666 adjustments.safe_push (adj);
668 /* We have previously allocated one extra entry for the mask. Use
669 it and fill it. */
670 sc->nargs++;
671 if (sc->mask_mode != VOIDmode)
672 base_type = boolean_type_node;
673 if (node->definition)
675 sc->args[i].orig_arg
676 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type);
677 if (sc->mask_mode == VOIDmode)
678 sc->args[i].simd_array
679 = create_tmp_simd_array ("mask", base_type, sc->simdlen);
680 else if (veclen < sc->simdlen)
681 sc->args[i].simd_array
682 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen);
683 else
684 sc->args[i].simd_array = NULL_TREE;
686 sc->args[i].orig_type = base_type;
687 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK;
690 if (node->definition)
691 ipa_modify_formal_parameters (node->decl, adjustments);
692 else
694 tree new_arg_types = NULL_TREE, new_reversed;
695 bool last_parm_void = false;
696 if (args.length () > 0 && args.last () == void_type_node)
697 last_parm_void = true;
699 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl)));
700 j = adjustments.length ();
701 for (i = 0; i < j; i++)
703 struct ipa_parm_adjustment *adj = &adjustments[i];
704 tree ptype;
705 if (adj->op == IPA_PARM_OP_COPY)
706 ptype = args[adj->base_index];
707 else
708 ptype = adj->type;
709 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types);
711 new_reversed = nreverse (new_arg_types);
712 if (last_parm_void)
714 if (new_reversed)
715 TREE_CHAIN (new_arg_types) = void_list_node;
716 else
717 new_reversed = void_list_node;
720 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl));
721 TYPE_ARG_TYPES (new_type) = new_reversed;
722 TREE_TYPE (node->decl) = new_type;
724 adjustments.release ();
726 args.release ();
727 return adjustments;
730 /* Initialize and copy the function arguments in NODE to their
731 corresponding local simd arrays. Returns a fresh gimple_seq with
732 the instruction sequence generated. */
734 static gimple_seq
735 simd_clone_init_simd_arrays (struct cgraph_node *node,
736 ipa_parm_adjustment_vec adjustments)
738 gimple_seq seq = NULL;
739 unsigned i = 0, j = 0, k;
741 for (tree arg = DECL_ARGUMENTS (node->decl);
742 arg;
743 arg = DECL_CHAIN (arg), i++, j++)
745 if (adjustments[j].op == IPA_PARM_OP_COPY
746 || POINTER_TYPE_P (TREE_TYPE (arg)))
747 continue;
749 node->simdclone->args[i].vector_arg = arg;
751 tree array = node->simdclone->args[i].simd_array;
752 if (node->simdclone->mask_mode != VOIDmode
753 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK)
755 if (array == NULL_TREE)
756 continue;
757 unsigned int l
758 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array))));
759 for (k = 0; k <= l; k++)
761 if (k)
763 arg = DECL_CHAIN (arg);
764 j++;
766 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)),
767 array, size_int (k), NULL, NULL);
768 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
769 gimplify_and_add (t, &seq);
771 continue;
773 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen)
775 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
776 tree ptr = build_fold_addr_expr (array);
777 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
778 build_int_cst (ptype, 0));
779 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
780 gimplify_and_add (t, &seq);
782 else
784 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg));
785 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array)));
786 for (k = 0; k < node->simdclone->simdlen; k += simdlen)
788 tree ptr = build_fold_addr_expr (array);
789 int elemsize;
790 if (k)
792 arg = DECL_CHAIN (arg);
793 j++;
795 elemsize
796 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg))));
797 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr,
798 build_int_cst (ptype, k * elemsize));
799 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg);
800 gimplify_and_add (t, &seq);
804 return seq;
807 /* Callback info for ipa_simd_modify_stmt_ops below. */
809 struct modify_stmt_info {
810 ipa_parm_adjustment_vec adjustments;
811 gimple *stmt;
812 /* True if the parent statement was modified by
813 ipa_simd_modify_stmt_ops. */
814 bool modified;
817 /* Callback for walk_gimple_op.
819 Adjust operands from a given statement as specified in the
820 adjustments vector in the callback data. */
822 static tree
823 ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data)
825 struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
826 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info;
827 tree *orig_tp = tp;
828 if (TREE_CODE (*tp) == ADDR_EXPR)
829 tp = &TREE_OPERAND (*tp, 0);
830 struct ipa_parm_adjustment *cand = NULL;
831 if (TREE_CODE (*tp) == PARM_DECL)
832 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true);
833 else
835 if (TYPE_P (*tp))
836 *walk_subtrees = 0;
839 tree repl = NULL_TREE;
840 if (cand)
841 repl = unshare_expr (cand->new_decl);
842 else
844 if (tp != orig_tp)
846 *walk_subtrees = 0;
847 bool modified = info->modified;
848 info->modified = false;
849 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset);
850 if (!info->modified)
852 info->modified = modified;
853 return NULL_TREE;
855 info->modified = modified;
856 repl = *tp;
858 else
859 return NULL_TREE;
862 if (tp != orig_tp)
864 repl = build_fold_addr_expr (repl);
865 gimple *stmt;
866 if (is_gimple_debug (info->stmt))
868 tree vexpr = make_node (DEBUG_EXPR_DECL);
869 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL);
870 DECL_ARTIFICIAL (vexpr) = 1;
871 TREE_TYPE (vexpr) = TREE_TYPE (repl);
872 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl)));
873 repl = vexpr;
875 else
877 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl);
878 repl = gimple_assign_lhs (stmt);
880 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt);
881 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
882 *orig_tp = repl;
884 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl)))
886 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl);
887 *tp = vce;
889 else
890 *tp = repl;
892 info->modified = true;
893 return NULL_TREE;
896 /* Traverse the function body and perform all modifications as
897 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
898 modified such that the replacement/reduction value will now be an
899 offset into the corresponding simd_array.
901 This function will replace all function argument uses with their
902 corresponding simd array elements, and ajust the return values
903 accordingly. */
905 static void
906 ipa_simd_modify_function_body (struct cgraph_node *node,
907 ipa_parm_adjustment_vec adjustments,
908 tree retval_array, tree iter)
910 basic_block bb;
911 unsigned int i, j, l;
913 /* Re-use the adjustments array, but this time use it to replace
914 every function argument use to an offset into the corresponding
915 simd_array. */
916 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j)
918 if (!node->simdclone->args[i].vector_arg)
919 continue;
921 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg);
922 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg);
923 adjustments[j].new_decl
924 = build4 (ARRAY_REF,
925 basetype,
926 node->simdclone->args[i].simd_array,
927 iter,
928 NULL_TREE, NULL_TREE);
929 if (adjustments[j].op == IPA_PARM_OP_NONE
930 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen)
931 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1;
934 l = adjustments.length ();
935 tree name;
937 FOR_EACH_SSA_NAME (i, name, cfun)
939 if (SSA_NAME_VAR (name)
940 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL)
942 for (j = 0; j < l; j++)
943 if (SSA_NAME_VAR (name) == adjustments[j].base
944 && adjustments[j].new_decl)
946 tree base_var;
947 if (adjustments[j].new_ssa_base == NULL_TREE)
949 base_var
950 = copy_var_decl (adjustments[j].base,
951 DECL_NAME (adjustments[j].base),
952 TREE_TYPE (adjustments[j].base));
953 adjustments[j].new_ssa_base = base_var;
955 else
956 base_var = adjustments[j].new_ssa_base;
957 if (SSA_NAME_IS_DEFAULT_DEF (name))
959 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
960 gimple_stmt_iterator gsi = gsi_after_labels (bb);
961 tree new_decl = unshare_expr (adjustments[j].new_decl);
962 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE);
963 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
964 SSA_NAME_IS_DEFAULT_DEF (name) = 0;
965 gimple *stmt = gimple_build_assign (name, new_decl);
966 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
968 else
969 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var);
974 struct modify_stmt_info info;
975 info.adjustments = adjustments;
977 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
979 gimple_stmt_iterator gsi;
981 gsi = gsi_start_bb (bb);
982 while (!gsi_end_p (gsi))
984 gimple *stmt = gsi_stmt (gsi);
985 info.stmt = stmt;
986 struct walk_stmt_info wi;
988 memset (&wi, 0, sizeof (wi));
989 info.modified = false;
990 wi.info = &info;
991 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi);
993 if (greturn *return_stmt = dyn_cast <greturn *> (stmt))
995 tree retval = gimple_return_retval (return_stmt);
996 if (!retval)
998 gsi_remove (&gsi, true);
999 continue;
1002 /* Replace `return foo' with `retval_array[iter] = foo'. */
1003 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval),
1004 retval_array, iter, NULL, NULL);
1005 stmt = gimple_build_assign (ref, retval);
1006 gsi_replace (&gsi, stmt, true);
1007 info.modified = true;
1010 if (info.modified)
1012 update_stmt (stmt);
1013 if (maybe_clean_eh_stmt (stmt))
1014 gimple_purge_dead_eh_edges (gimple_bb (stmt));
1016 gsi_next (&gsi);
1021 /* Helper function of simd_clone_adjust, return linear step addend
1022 of Ith argument. */
1024 static tree
1025 simd_clone_linear_addend (struct cgraph_node *node, unsigned int i,
1026 tree addtype, basic_block entry_bb)
1028 tree ptype = NULL_TREE;
1029 switch (node->simdclone->args[i].arg_type)
1031 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP:
1032 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP:
1033 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP:
1034 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP:
1035 return build_int_cst (addtype, node->simdclone->args[i].linear_step);
1036 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP:
1037 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP:
1038 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg);
1039 break;
1040 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP:
1041 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP:
1042 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg));
1043 break;
1044 default:
1045 gcc_unreachable ();
1048 unsigned int idx = node->simdclone->args[i].linear_step;
1049 tree arg = node->simdclone->args[idx].orig_arg;
1050 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg)));
1051 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1052 gimple *g;
1053 tree ret;
1054 if (is_gimple_reg (arg))
1055 ret = get_or_create_ssa_default_def (cfun, arg);
1056 else
1058 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg);
1059 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1060 ret = gimple_assign_lhs (g);
1062 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE)
1064 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))),
1065 build_simple_mem_ref (ret));
1066 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1067 ret = gimple_assign_lhs (g);
1069 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret)))
1071 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret);
1072 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1073 ret = gimple_assign_lhs (g);
1075 if (POINTER_TYPE_P (ptype))
1077 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype));
1078 if (size && TREE_CODE (size) == INTEGER_CST)
1080 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR,
1081 ret, fold_convert (addtype, size));
1082 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1083 ret = gimple_assign_lhs (g);
1086 return ret;
1089 /* Adjust the argument types in NODE to their appropriate vector
1090 counterparts. */
1092 static void
1093 simd_clone_adjust (struct cgraph_node *node)
1095 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
1097 targetm.simd_clone.adjust (node);
1099 tree retval = simd_clone_adjust_return_type (node);
1100 ipa_parm_adjustment_vec adjustments
1101 = simd_clone_adjust_argument_types (node);
1103 push_gimplify_context ();
1105 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments);
1107 /* Adjust all uses of vector arguments accordingly. Adjust all
1108 return values accordingly. */
1109 tree iter = create_tmp_var (unsigned_type_node, "iter");
1110 tree iter1 = make_ssa_name (iter);
1111 tree iter2 = NULL_TREE;
1112 ipa_simd_modify_function_body (node, adjustments, retval, iter1);
1113 adjustments.release ();
1115 /* Initialize the iteration variable. */
1116 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1117 basic_block body_bb = split_block_after_labels (entry_bb)->dest;
1118 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb);
1119 /* Insert the SIMD array and iv initialization at function
1120 entry. */
1121 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT);
1123 pop_gimplify_context (NULL);
1125 gimple *g;
1126 basic_block incr_bb = NULL;
1127 struct loop *loop = NULL;
1129 /* Create a new BB right before the original exit BB, to hold the
1130 iteration increment and the condition/branch. */
1131 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
1133 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src;
1134 incr_bb = create_empty_bb (orig_exit);
1135 add_bb_to_loop (incr_bb, body_bb->loop_father);
1136 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
1137 flag. Set it now to be a FALLTHRU_EDGE. */
1138 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1);
1139 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU;
1140 for (unsigned i = 0;
1141 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i)
1143 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i);
1144 redirect_edge_succ (e, incr_bb);
1147 else if (node->simdclone->inbranch)
1149 incr_bb = create_empty_bb (entry_bb);
1150 add_bb_to_loop (incr_bb, body_bb->loop_father);
1153 if (incr_bb)
1155 edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
1156 e->probability = REG_BR_PROB_BASE;
1157 gsi = gsi_last_bb (incr_bb);
1158 iter2 = make_ssa_name (iter);
1159 g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
1160 build_int_cst (unsigned_type_node, 1));
1161 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1163 /* Mostly annotate the loop for the vectorizer (the rest is done
1164 below). */
1165 loop = alloc_loop ();
1166 cfun->has_force_vectorize_loops = true;
1167 loop->safelen = node->simdclone->simdlen;
1168 loop->force_vectorize = true;
1169 loop->header = body_bb;
1172 /* Branch around the body if the mask applies. */
1173 if (node->simdclone->inbranch)
1175 gsi = gsi_last_bb (loop->header);
1176 tree mask_array
1177 = node->simdclone->args[node->simdclone->nargs - 1].simd_array;
1178 tree mask;
1179 if (node->simdclone->mask_mode != VOIDmode)
1181 tree shift_cnt;
1182 if (mask_array == NULL_TREE)
1184 tree arg = node->simdclone->args[node->simdclone->nargs
1185 - 1].vector_arg;
1186 mask = get_or_create_ssa_default_def (cfun, arg);
1187 shift_cnt = iter1;
1189 else
1191 tree maskt = TREE_TYPE (mask_array);
1192 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt)));
1193 c = node->simdclone->simdlen / (c + 1);
1194 int s = exact_log2 (c);
1195 gcc_assert (s > 0);
1196 c--;
1197 tree idx = make_ssa_name (TREE_TYPE (iter1));
1198 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1,
1199 build_int_cst (NULL_TREE, s));
1200 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1201 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1202 tree aref = build4 (ARRAY_REF,
1203 TREE_TYPE (TREE_TYPE (mask_array)),
1204 mask_array, idx, NULL, NULL);
1205 g = gimple_build_assign (mask, aref);
1206 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1207 shift_cnt = make_ssa_name (TREE_TYPE (iter1));
1208 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1,
1209 build_int_cst (TREE_TYPE (iter1), c));
1210 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1212 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1213 RSHIFT_EXPR, mask, shift_cnt);
1214 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1215 mask = gimple_assign_lhs (g);
1216 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)),
1217 BIT_AND_EXPR, mask,
1218 build_int_cst (TREE_TYPE (mask), 1));
1219 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1220 mask = gimple_assign_lhs (g);
1222 else
1224 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array)));
1225 tree aref = build4 (ARRAY_REF,
1226 TREE_TYPE (TREE_TYPE (mask_array)),
1227 mask_array, iter1, NULL, NULL);
1228 g = gimple_build_assign (mask, aref);
1229 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1230 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref)));
1231 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref)))
1233 aref = build1 (VIEW_CONVERT_EXPR,
1234 build_nonstandard_integer_type (bitsize, 0),
1235 mask);
1236 mask = make_ssa_name (TREE_TYPE (aref));
1237 g = gimple_build_assign (mask, aref);
1238 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1242 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)),
1243 NULL, NULL);
1244 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1245 make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE);
1246 FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE;
1249 basic_block latch_bb = NULL;
1250 basic_block new_exit_bb = NULL;
1252 /* Generate the condition. */
1253 if (incr_bb)
1255 gsi = gsi_last_bb (incr_bb);
1256 g = gimple_build_cond (LT_EXPR, iter2,
1257 build_int_cst (unsigned_type_node,
1258 node->simdclone->simdlen),
1259 NULL, NULL);
1260 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1261 edge e = split_block (incr_bb, gsi_stmt (gsi));
1262 latch_bb = e->dest;
1263 new_exit_bb = split_block_after_labels (latch_bb)->dest;
1264 loop->latch = latch_bb;
1266 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
1268 make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
1269 /* The successor of incr_bb is already pointing to latch_bb; just
1270 change the flags.
1271 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1272 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE;
1275 gphi *phi = create_phi_node (iter1, body_bb);
1276 edge preheader_edge = find_edge (entry_bb, body_bb);
1277 edge latch_edge = NULL;
1278 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge,
1279 UNKNOWN_LOCATION);
1280 if (incr_bb)
1282 latch_edge = single_succ_edge (latch_bb);
1283 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1285 /* Generate the new return. */
1286 gsi = gsi_last_bb (new_exit_bb);
1287 if (retval
1288 && TREE_CODE (retval) == VIEW_CONVERT_EXPR
1289 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
1290 retval = TREE_OPERAND (retval, 0);
1291 else if (retval)
1293 retval = build1 (VIEW_CONVERT_EXPR,
1294 TREE_TYPE (TREE_TYPE (node->decl)),
1295 retval);
1296 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL,
1297 false, GSI_CONTINUE_LINKING);
1299 g = gimple_build_return (retval);
1300 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING);
1303 /* Handle aligned clauses by replacing default defs of the aligned
1304 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1305 lhs. Handle linear by adding PHIs. */
1306 for (unsigned i = 0; i < node->simdclone->nargs; i++)
1307 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1308 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg)
1309 || !is_gimple_reg_type
1310 (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1312 tree orig_arg = node->simdclone->args[i].orig_arg;
1313 if (is_gimple_reg_type (TREE_TYPE (orig_arg)))
1314 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1315 else
1317 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg));
1318 gimple_add_tmp_var (iter1);
1320 gsi = gsi_after_labels (entry_bb);
1321 g = gimple_build_assign (iter1, orig_arg);
1322 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1323 gsi = gsi_after_labels (body_bb);
1324 g = gimple_build_assign (orig_arg, iter1);
1325 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1327 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM
1328 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg)
1329 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1330 == REFERENCE_TYPE
1331 && TREE_ADDRESSABLE
1332 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg))))
1334 tree orig_arg = node->simdclone->args[i].orig_arg;
1335 tree def = ssa_default_def (cfun, orig_arg);
1336 if (def && !has_zero_uses (def))
1338 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg)));
1339 gimple_add_tmp_var (iter1);
1340 gsi = gsi_after_labels (entry_bb);
1341 g = gimple_build_assign (iter1, build_simple_mem_ref (def));
1342 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1343 gsi = gsi_after_labels (body_bb);
1344 g = gimple_build_assign (build_simple_mem_ref (def), iter1);
1345 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1348 else if (node->simdclone->args[i].alignment
1349 && node->simdclone->args[i].arg_type
1350 == SIMD_CLONE_ARG_TYPE_UNIFORM
1351 && (node->simdclone->args[i].alignment
1352 & (node->simdclone->args[i].alignment - 1)) == 0
1353 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg))
1354 == POINTER_TYPE)
1356 unsigned int alignment = node->simdclone->args[i].alignment;
1357 tree orig_arg = node->simdclone->args[i].orig_arg;
1358 tree def = ssa_default_def (cfun, orig_arg);
1359 if (def && !has_zero_uses (def))
1361 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED);
1362 gimple_seq seq = NULL;
1363 bool need_cvt = false;
1364 gcall *call
1365 = gimple_build_call (fn, 2, def, size_int (alignment));
1366 g = call;
1367 if (!useless_type_conversion_p (TREE_TYPE (orig_arg),
1368 ptr_type_node))
1369 need_cvt = true;
1370 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg);
1371 gimple_call_set_lhs (g, t);
1372 gimple_seq_add_stmt_without_update (&seq, g);
1373 if (need_cvt)
1375 t = make_ssa_name (orig_arg);
1376 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g));
1377 gimple_seq_add_stmt_without_update (&seq, g);
1379 gsi_insert_seq_on_edge_immediate
1380 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq);
1382 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1383 int freq = compute_call_stmt_bb_frequency (current_function_decl,
1384 entry_bb);
1385 node->create_edge (cgraph_node::get_create (fn),
1386 call, entry_bb->count, freq);
1388 imm_use_iterator iter;
1389 use_operand_p use_p;
1390 gimple *use_stmt;
1391 tree repl = gimple_get_lhs (g);
1392 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1393 if (is_gimple_debug (use_stmt) || use_stmt == call)
1394 continue;
1395 else
1396 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1397 SET_USE (use_p, repl);
1400 else if ((node->simdclone->args[i].arg_type
1401 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP)
1402 || (node->simdclone->args[i].arg_type
1403 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP)
1404 || (node->simdclone->args[i].arg_type
1405 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP)
1406 || (node->simdclone->args[i].arg_type
1407 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP))
1409 tree orig_arg = node->simdclone->args[i].orig_arg;
1410 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1411 || POINTER_TYPE_P (TREE_TYPE (orig_arg)));
1412 tree def = NULL_TREE;
1413 if (TREE_ADDRESSABLE (orig_arg))
1415 def = make_ssa_name (TREE_TYPE (orig_arg));
1416 iter1 = make_ssa_name (TREE_TYPE (orig_arg));
1417 if (incr_bb)
1418 iter2 = make_ssa_name (TREE_TYPE (orig_arg));
1419 gsi = gsi_after_labels (entry_bb);
1420 g = gimple_build_assign (def, orig_arg);
1421 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1423 else
1425 def = ssa_default_def (cfun, orig_arg);
1426 if (!def || has_zero_uses (def))
1427 def = NULL_TREE;
1428 else
1430 iter1 = make_ssa_name (orig_arg);
1431 if (incr_bb)
1432 iter2 = make_ssa_name (orig_arg);
1435 if (def)
1437 phi = create_phi_node (iter1, body_bb);
1438 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION);
1439 if (incr_bb)
1441 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1442 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1443 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1444 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg))
1445 ? TREE_TYPE (orig_arg) : sizetype;
1446 tree addcst = simd_clone_linear_addend (node, i, addtype,
1447 entry_bb);
1448 gsi = gsi_last_bb (incr_bb);
1449 g = gimple_build_assign (iter2, code, iter1, addcst);
1450 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1453 imm_use_iterator iter;
1454 use_operand_p use_p;
1455 gimple *use_stmt;
1456 if (TREE_ADDRESSABLE (orig_arg))
1458 gsi = gsi_after_labels (body_bb);
1459 g = gimple_build_assign (orig_arg, iter1);
1460 gsi_insert_before (&gsi, g, GSI_NEW_STMT);
1462 else
1463 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1464 if (use_stmt == phi)
1465 continue;
1466 else
1467 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1468 SET_USE (use_p, iter1);
1471 else if (node->simdclone->args[i].arg_type
1472 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1473 || (node->simdclone->args[i].arg_type
1474 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP))
1476 tree orig_arg = node->simdclone->args[i].orig_arg;
1477 tree def = ssa_default_def (cfun, orig_arg);
1478 gcc_assert (!TREE_ADDRESSABLE (orig_arg)
1479 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE);
1480 if (def && !has_zero_uses (def))
1482 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg));
1483 iter1 = make_ssa_name (orig_arg);
1484 if (incr_bb)
1485 iter2 = make_ssa_name (orig_arg);
1486 tree iter3 = make_ssa_name (rtype);
1487 tree iter4 = make_ssa_name (rtype);
1488 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE;
1489 gsi = gsi_after_labels (entry_bb);
1490 gimple *load
1491 = gimple_build_assign (iter3, build_simple_mem_ref (def));
1492 gsi_insert_before (&gsi, load, GSI_NEW_STMT);
1494 tree array = node->simdclone->args[i].simd_array;
1495 TREE_ADDRESSABLE (array) = 1;
1496 tree ptr = build_fold_addr_expr (array);
1497 phi = create_phi_node (iter1, body_bb);
1498 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION);
1499 if (incr_bb)
1501 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION);
1502 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1,
1503 TYPE_SIZE_UNIT (TREE_TYPE (iter3)));
1504 gsi = gsi_last_bb (incr_bb);
1505 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1508 phi = create_phi_node (iter4, body_bb);
1509 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION);
1510 if (incr_bb)
1512 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION);
1513 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1514 ? PLUS_EXPR : POINTER_PLUS_EXPR;
1515 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3))
1516 ? TREE_TYPE (iter3) : sizetype;
1517 tree addcst = simd_clone_linear_addend (node, i, addtype,
1518 entry_bb);
1519 g = gimple_build_assign (iter5, code, iter4, addcst);
1520 gsi = gsi_last_bb (incr_bb);
1521 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1524 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4);
1525 gsi = gsi_after_labels (body_bb);
1526 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1528 imm_use_iterator iter;
1529 use_operand_p use_p;
1530 gimple *use_stmt;
1531 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def)
1532 if (use_stmt == load)
1533 continue;
1534 else
1535 FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
1536 SET_USE (use_p, iter1);
1538 if (!TYPE_READONLY (rtype) && incr_bb)
1540 tree v = make_ssa_name (rtype);
1541 tree aref = build4 (ARRAY_REF, rtype, array,
1542 size_zero_node, NULL_TREE,
1543 NULL_TREE);
1544 gsi = gsi_after_labels (new_exit_bb);
1545 g = gimple_build_assign (v, aref);
1546 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1547 g = gimple_build_assign (build_simple_mem_ref (def), v);
1548 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
1553 calculate_dominance_info (CDI_DOMINATORS);
1554 if (loop)
1555 add_loop (loop, loop->header->loop_father);
1556 update_ssa (TODO_update_ssa);
1558 pop_cfun ();
1561 /* If the function in NODE is tagged as an elemental SIMD function,
1562 create the appropriate SIMD clones. */
1564 static void
1565 expand_simd_clones (struct cgraph_node *node)
1567 tree attr = lookup_attribute ("omp declare simd",
1568 DECL_ATTRIBUTES (node->decl));
1569 if (attr == NULL_TREE
1570 || node->global.inlined_to
1571 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
1572 return;
1574 /* Ignore
1575 #pragma omp declare simd
1576 extern int foo ();
1577 in C, there we don't know the argument types at all. */
1578 if (!node->definition
1579 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE)
1580 return;
1582 /* Call this before creating clone_info, as it might ggc_collect. */
1583 if (node->definition && node->has_gimple_body_p ())
1584 node->get_body ();
1588 /* Start with parsing the "omp declare simd" attribute(s). */
1589 bool inbranch_clause_specified;
1590 struct cgraph_simd_clone *clone_info
1591 = simd_clone_clauses_extract (node, TREE_VALUE (attr),
1592 &inbranch_clause_specified);
1593 if (clone_info == NULL)
1594 continue;
1596 int orig_simdlen = clone_info->simdlen;
1597 tree base_type = simd_clone_compute_base_data_type (node, clone_info);
1598 /* The target can return 0 (no simd clones should be created),
1599 1 (just one ISA of simd clones should be created) or higher
1600 count of ISA variants. In that case, clone_info is initialized
1601 for the first ISA variant. */
1602 int count
1603 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info,
1604 base_type, 0);
1605 if (count == 0)
1606 continue;
1608 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1609 also create one inbranch and one !inbranch clone of it. */
1610 for (int i = 0; i < count * 2; i++)
1612 struct cgraph_simd_clone *clone = clone_info;
1613 if (inbranch_clause_specified && (i & 1) != 0)
1614 continue;
1616 if (i != 0)
1618 clone = simd_clone_struct_alloc (clone_info->nargs
1619 + ((i & 1) != 0));
1620 simd_clone_struct_copy (clone, clone_info);
1621 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1622 and simd_clone_adjust_argument_types did to the first
1623 clone's info. */
1624 clone->nargs -= clone_info->inbranch;
1625 clone->simdlen = orig_simdlen;
1626 /* And call the target hook again to get the right ISA. */
1627 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone,
1628 base_type,
1629 i / 2);
1630 if ((i & 1) != 0)
1631 clone->inbranch = 1;
1634 /* simd_clone_mangle might fail if such a clone has been created
1635 already. */
1636 tree id = simd_clone_mangle (node, clone);
1637 if (id == NULL_TREE)
1638 continue;
1640 /* Only when we are sure we want to create the clone actually
1641 clone the function (or definitions) or create another
1642 extern FUNCTION_DECL (for prototypes without definitions). */
1643 struct cgraph_node *n = simd_clone_create (node);
1644 if (n == NULL)
1645 continue;
1647 n->simdclone = clone;
1648 clone->origin = node;
1649 clone->next_clone = NULL;
1650 if (node->simd_clones == NULL)
1652 clone->prev_clone = n;
1653 node->simd_clones = n;
1655 else
1657 clone->prev_clone = node->simd_clones->simdclone->prev_clone;
1658 clone->prev_clone->simdclone->next_clone = n;
1659 node->simd_clones->simdclone->prev_clone = n;
1661 symtab->change_decl_assembler_name (n->decl, id);
1662 /* And finally adjust the return type, parameters and for
1663 definitions also function body. */
1664 if (node->definition)
1665 simd_clone_adjust (n);
1666 else
1668 simd_clone_adjust_return_type (n);
1669 simd_clone_adjust_argument_types (n);
1673 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr))));
1676 /* Entry point for IPA simd clone creation pass. */
1678 static unsigned int
1679 ipa_omp_simd_clone (void)
1681 struct cgraph_node *node;
1682 FOR_EACH_FUNCTION (node)
1683 expand_simd_clones (node);
1684 return 0;
1687 namespace {
1689 const pass_data pass_data_omp_simd_clone =
1691 SIMPLE_IPA_PASS, /* type */
1692 "simdclone", /* name */
1693 OPTGROUP_OMP, /* optinfo_flags */
1694 TV_NONE, /* tv_id */
1695 ( PROP_ssa | PROP_cfg ), /* properties_required */
1696 0, /* properties_provided */
1697 0, /* properties_destroyed */
1698 0, /* todo_flags_start */
1699 0, /* todo_flags_finish */
1702 class pass_omp_simd_clone : public simple_ipa_opt_pass
1704 public:
1705 pass_omp_simd_clone(gcc::context *ctxt)
1706 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt)
1709 /* opt_pass methods: */
1710 virtual bool gate (function *);
1711 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); }
1714 bool
1715 pass_omp_simd_clone::gate (function *)
1717 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL;
1720 } // anon namespace
1722 simple_ipa_opt_pass *
1723 make_pass_omp_simd_clone (gcc::context *ctxt)
1725 return new pass_omp_simd_clone (ctxt);