1 /* Lower vector operations to scalar operations.
2 Copyright (C) 2004-2013 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 GCC is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
25 #include "langhooks.h"
27 #include "gimple-ssa.h"
29 #include "tree-ssanames.h"
30 #include "tree-iterator.h"
31 #include "tree-pass.h"
34 #include "diagnostic.h"
37 /* Need to include rtl.h, expr.h, etc. for optabs. */
42 static void expand_vector_operations_1 (gimple_stmt_iterator
*);
45 /* Build a constant of type TYPE, made of VALUE's bits replicated
46 every TYPE_SIZE (INNER_TYPE) bits to fit TYPE's precision. */
48 build_replicated_const (tree type
, tree inner_type
, HOST_WIDE_INT value
)
50 int width
= tree_low_cst (TYPE_SIZE (inner_type
), 1);
51 int n
= HOST_BITS_PER_WIDE_INT
/ width
;
52 unsigned HOST_WIDE_INT low
, high
, mask
;
57 if (width
== HOST_BITS_PER_WIDE_INT
)
61 mask
= ((HOST_WIDE_INT
)1 << width
) - 1;
62 low
= (unsigned HOST_WIDE_INT
) ~0 / mask
* (value
& mask
);
65 if (TYPE_PRECISION (type
) < HOST_BITS_PER_WIDE_INT
)
66 low
&= ((HOST_WIDE_INT
)1 << TYPE_PRECISION (type
)) - 1, high
= 0;
67 else if (TYPE_PRECISION (type
) == HOST_BITS_PER_WIDE_INT
)
69 else if (TYPE_PRECISION (type
) == HOST_BITS_PER_DOUBLE_INT
)
74 ret
= build_int_cst_wide (type
, low
, high
);
78 static GTY(()) tree vector_inner_type
;
79 static GTY(()) tree vector_last_type
;
80 static GTY(()) int vector_last_nunits
;
82 /* Return a suitable vector types made of SUBPARTS units each of mode
83 "word_mode" (the global variable). */
85 build_word_mode_vector_type (int nunits
)
87 if (!vector_inner_type
)
88 vector_inner_type
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
89 else if (vector_last_nunits
== nunits
)
91 gcc_assert (TREE_CODE (vector_last_type
) == VECTOR_TYPE
);
92 return vector_last_type
;
95 /* We build a new type, but we canonicalize it nevertheless,
96 because it still saves some memory. */
97 vector_last_nunits
= nunits
;
98 vector_last_type
= type_hash_canon (nunits
,
99 build_vector_type (vector_inner_type
,
101 return vector_last_type
;
104 typedef tree (*elem_op_func
) (gimple_stmt_iterator
*,
105 tree
, tree
, tree
, tree
, tree
, enum tree_code
);
108 tree_vec_extract (gimple_stmt_iterator
*gsi
, tree type
,
109 tree t
, tree bitsize
, tree bitpos
)
112 return gimplify_build3 (gsi
, BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
114 return gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, type
, t
);
118 do_unop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
119 tree b ATTRIBUTE_UNUSED
, tree bitpos
, tree bitsize
,
122 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
123 return gimplify_build1 (gsi
, code
, inner_type
, a
);
127 do_binop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
128 tree bitpos
, tree bitsize
, enum tree_code code
)
130 if (TREE_CODE (TREE_TYPE (a
)) == VECTOR_TYPE
)
131 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
132 if (TREE_CODE (TREE_TYPE (b
)) == VECTOR_TYPE
)
133 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
134 return gimplify_build2 (gsi
, code
, inner_type
, a
, b
);
137 /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
139 INNER_TYPE is the type of A and B elements
141 returned expression is of signed integer type with the
142 size equal to the size of INNER_TYPE. */
144 do_compare (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
145 tree bitpos
, tree bitsize
, enum tree_code code
)
149 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
150 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
152 comp_type
= build_nonstandard_integer_type
153 (GET_MODE_BITSIZE (TYPE_MODE (inner_type
)), 0);
155 return gimplify_build3 (gsi
, COND_EXPR
, comp_type
,
156 fold_build2 (code
, boolean_type_node
, a
, b
),
157 build_int_cst (comp_type
, -1),
158 build_int_cst (comp_type
, 0));
161 /* Expand vector addition to scalars. This does bit twiddling
162 in order to increase parallelism:
164 a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
167 a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
168 (a ^ ~b) & 0x80808080
170 -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
172 This optimization should be done only if 4 vector items or more
175 do_plus_minus (gimple_stmt_iterator
*gsi
, tree word_type
, tree a
, tree b
,
176 tree bitpos ATTRIBUTE_UNUSED
, tree bitsize ATTRIBUTE_UNUSED
,
179 tree inner_type
= TREE_TYPE (TREE_TYPE (a
));
180 unsigned HOST_WIDE_INT max
;
181 tree low_bits
, high_bits
, a_low
, b_low
, result_low
, signs
;
183 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
184 low_bits
= build_replicated_const (word_type
, inner_type
, max
>> 1);
185 high_bits
= build_replicated_const (word_type
, inner_type
, max
& ~(max
>> 1));
187 a
= tree_vec_extract (gsi
, word_type
, a
, bitsize
, bitpos
);
188 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
190 signs
= gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, a
, b
);
191 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
192 if (code
== PLUS_EXPR
)
193 a_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, a
, low_bits
);
196 a_low
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, word_type
, a
, high_bits
);
197 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, signs
);
200 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
201 result_low
= gimplify_build2 (gsi
, code
, word_type
, a_low
, b_low
);
202 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
206 do_negate (gimple_stmt_iterator
*gsi
, tree word_type
, tree b
,
207 tree unused ATTRIBUTE_UNUSED
, tree bitpos ATTRIBUTE_UNUSED
,
208 tree bitsize ATTRIBUTE_UNUSED
,
209 enum tree_code code ATTRIBUTE_UNUSED
)
211 tree inner_type
= TREE_TYPE (TREE_TYPE (b
));
213 tree low_bits
, high_bits
, b_low
, result_low
, signs
;
215 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
216 low_bits
= build_replicated_const (word_type
, inner_type
, max
>> 1);
217 high_bits
= build_replicated_const (word_type
, inner_type
, max
& ~(max
>> 1));
219 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
221 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
222 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, b
);
223 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
224 result_low
= gimplify_build2 (gsi
, MINUS_EXPR
, word_type
, high_bits
, b_low
);
225 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
228 /* Expand a vector operation to scalars, by using many operations
229 whose type is the vector type's inner type. */
231 expand_vector_piecewise (gimple_stmt_iterator
*gsi
, elem_op_func f
,
232 tree type
, tree inner_type
,
233 tree a
, tree b
, enum tree_code code
)
235 vec
<constructor_elt
, va_gc
> *v
;
236 tree part_width
= TYPE_SIZE (inner_type
);
237 tree index
= bitsize_int (0);
238 int nunits
= TYPE_VECTOR_SUBPARTS (type
);
239 int delta
= tree_low_cst (part_width
, 1)
240 / tree_low_cst (TYPE_SIZE (TREE_TYPE (type
)), 1);
242 location_t loc
= gimple_location (gsi_stmt (*gsi
));
244 if (types_compatible_p (gimple_expr_type (gsi_stmt (*gsi
)), type
))
245 warning_at (loc
, OPT_Wvector_operation_performance
,
246 "vector operation will be expanded piecewise");
248 warning_at (loc
, OPT_Wvector_operation_performance
,
249 "vector operation will be expanded in parallel");
251 vec_alloc (v
, (nunits
+ delta
- 1) / delta
);
252 for (i
= 0; i
< nunits
;
253 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
255 tree result
= f (gsi
, inner_type
, a
, b
, index
, part_width
, code
);
256 constructor_elt ce
= {NULL_TREE
, result
};
260 return build_constructor (type
, v
);
263 /* Expand a vector operation to scalars with the freedom to use
264 a scalar integer type, or to use a different size for the items
265 in the vector type. */
267 expand_vector_parallel (gimple_stmt_iterator
*gsi
, elem_op_func f
, tree type
,
271 tree result
, compute_type
;
272 enum machine_mode mode
;
273 int n_words
= tree_low_cst (TYPE_SIZE_UNIT (type
), 1) / UNITS_PER_WORD
;
274 location_t loc
= gimple_location (gsi_stmt (*gsi
));
276 /* We have three strategies. If the type is already correct, just do
277 the operation an element at a time. Else, if the vector is wider than
278 one word, do it a word at a time; finally, if the vector is smaller
279 than one word, do it as a scalar. */
280 if (TYPE_MODE (TREE_TYPE (type
)) == word_mode
)
281 return expand_vector_piecewise (gsi
, f
,
282 type
, TREE_TYPE (type
),
284 else if (n_words
> 1)
286 tree word_type
= build_word_mode_vector_type (n_words
);
287 result
= expand_vector_piecewise (gsi
, f
,
288 word_type
, TREE_TYPE (word_type
),
290 result
= force_gimple_operand_gsi (gsi
, result
, true, NULL
, true,
295 /* Use a single scalar operation with a mode no wider than word_mode. */
296 mode
= mode_for_size (tree_low_cst (TYPE_SIZE (type
), 1), MODE_INT
, 0);
297 compute_type
= lang_hooks
.types
.type_for_mode (mode
, 1);
298 result
= f (gsi
, compute_type
, a
, b
, NULL_TREE
, NULL_TREE
, code
);
299 warning_at (loc
, OPT_Wvector_operation_performance
,
300 "vector operation will be expanded with a "
301 "single scalar operation");
307 /* Expand a vector operation to scalars; for integer types we can use
308 special bit twiddling tricks to do the sums a word at a time, using
309 function F_PARALLEL instead of F. These tricks are done only if
310 they can process at least four items, that is, only if the vector
311 holds at least four items and if a word can hold four items. */
313 expand_vector_addition (gimple_stmt_iterator
*gsi
,
314 elem_op_func f
, elem_op_func f_parallel
,
315 tree type
, tree a
, tree b
, enum tree_code code
)
317 int parts_per_word
= UNITS_PER_WORD
318 / tree_low_cst (TYPE_SIZE_UNIT (TREE_TYPE (type
)), 1);
320 if (INTEGRAL_TYPE_P (TREE_TYPE (type
))
321 && parts_per_word
>= 4
322 && TYPE_VECTOR_SUBPARTS (type
) >= 4)
323 return expand_vector_parallel (gsi
, f_parallel
,
326 return expand_vector_piecewise (gsi
, f
,
327 type
, TREE_TYPE (type
),
331 /* Try to expand vector comparison expression OP0 CODE OP1 by
332 querying optab if the following expression:
333 VEC_COND_EXPR< OP0 CODE OP1, {-1,...}, {0,...}>
336 expand_vector_comparison (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
337 tree op1
, enum tree_code code
)
340 if (! expand_vec_cond_expr_p (type
, TREE_TYPE (op0
)))
341 t
= expand_vector_piecewise (gsi
, do_compare
, type
,
342 TREE_TYPE (TREE_TYPE (op0
)), op0
, op1
, code
);
349 /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
350 of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
351 the result if successful, otherwise return NULL_TREE. */
353 add_rshift (gimple_stmt_iterator
*gsi
, tree type
, tree op0
, int *shiftcnts
)
356 unsigned int i
, nunits
= TYPE_VECTOR_SUBPARTS (type
);
357 bool scalar_shift
= true;
359 for (i
= 1; i
< nunits
; i
++)
361 if (shiftcnts
[i
] != shiftcnts
[0])
362 scalar_shift
= false;
365 if (scalar_shift
&& shiftcnts
[0] == 0)
370 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_scalar
);
371 if (op
!= unknown_optab
372 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
373 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
374 build_int_cst (NULL_TREE
, shiftcnts
[0]));
377 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
378 if (op
!= unknown_optab
379 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
381 tree
*vec
= XALLOCAVEC (tree
, nunits
);
382 for (i
= 0; i
< nunits
; i
++)
383 vec
[i
] = build_int_cst (TREE_TYPE (type
), shiftcnts
[i
]);
384 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
385 build_vector (type
, vec
));
391 /* Try to expand integer vector division by constant using
392 widening multiply, shifts and additions. */
394 expand_vector_divmod (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
395 tree op1
, enum tree_code code
)
397 bool use_pow2
= true;
398 bool has_vector_shift
= true;
399 int mode
= -1, this_mode
;
400 int pre_shift
= -1, post_shift
;
401 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (type
);
402 int *shifts
= XALLOCAVEC (int, nunits
* 4);
403 int *pre_shifts
= shifts
+ nunits
;
404 int *post_shifts
= pre_shifts
+ nunits
;
405 int *shift_temps
= post_shifts
+ nunits
;
406 unsigned HOST_WIDE_INT
*mulc
= XALLOCAVEC (unsigned HOST_WIDE_INT
, nunits
);
407 int prec
= TYPE_PRECISION (TREE_TYPE (type
));
409 unsigned int i
, unsignedp
= TYPE_UNSIGNED (TREE_TYPE (type
));
410 unsigned HOST_WIDE_INT mask
= GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type
)));
412 tree cur_op
, mulcst
, tem
;
415 if (prec
> HOST_BITS_PER_WIDE_INT
)
418 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
419 if (op
== unknown_optab
420 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
421 has_vector_shift
= false;
423 /* Analysis phase. Determine if all op1 elements are either power
424 of two and it is possible to expand it using shifts (or for remainder
425 using masking). Additionally compute the multiplicative constants
426 and pre and post shifts if the division is to be expanded using
427 widening or high part multiplication plus shifts. */
428 for (i
= 0; i
< nunits
; i
++)
430 tree cst
= VECTOR_CST_ELT (op1
, i
);
431 unsigned HOST_WIDE_INT ml
;
433 if (!host_integerp (cst
, unsignedp
) || integer_zerop (cst
))
439 && (!integer_pow2p (cst
) || tree_int_cst_sgn (cst
) != 1))
443 shifts
[i
] = tree_log2 (cst
);
444 if (shifts
[i
] != shifts
[0]
445 && code
== TRUNC_DIV_EXPR
446 && !has_vector_shift
)
453 unsigned HOST_WIDE_INT mh
;
454 unsigned HOST_WIDE_INT d
= tree_low_cst (cst
, 1) & mask
;
456 if (d
>= ((unsigned HOST_WIDE_INT
) 1 << (prec
- 1)))
457 /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
466 /* Find a suitable multiplier and right shift count
467 instead of multiplying with D. */
468 mh
= choose_multiplier (d
, prec
, prec
, &ml
, &post_shift
, &dummy_int
);
470 /* If the suggested multiplier is more than SIZE bits, we can
471 do better for even divisors, using an initial right shift. */
472 if ((mh
!= 0 && (d
& 1) == 0)
473 || (!has_vector_shift
&& pre_shift
!= -1))
475 if (has_vector_shift
)
476 pre_shift
= floor_log2 (d
& -d
);
477 else if (pre_shift
== -1)
480 for (j
= 0; j
< nunits
; j
++)
482 tree cst2
= VECTOR_CST_ELT (op1
, j
);
483 unsigned HOST_WIDE_INT d2
;
486 if (!host_integerp (cst2
, 1))
488 d2
= tree_low_cst (cst2
, 1) & mask
;
491 this_pre_shift
= floor_log2 (d2
& -d2
);
492 if (pre_shift
== -1 || this_pre_shift
< pre_shift
)
493 pre_shift
= this_pre_shift
;
495 if (i
!= 0 && pre_shift
!= 0)
505 if ((d
>> pre_shift
) <= 1)
510 mh
= choose_multiplier (d
>> pre_shift
, prec
,
512 &ml
, &post_shift
, &dummy_int
);
514 pre_shifts
[i
] = pre_shift
;
524 HOST_WIDE_INT d
= tree_low_cst (cst
, 0);
525 unsigned HOST_WIDE_INT abs_d
;
530 /* Since d might be INT_MIN, we have to cast to
531 unsigned HOST_WIDE_INT before negating to avoid
532 undefined signed overflow. */
534 ? (unsigned HOST_WIDE_INT
) d
535 : - (unsigned HOST_WIDE_INT
) d
);
537 /* n rem d = n rem -d */
538 if (code
== TRUNC_MOD_EXPR
&& d
< 0)
540 else if (abs_d
== (unsigned HOST_WIDE_INT
) 1 << (prec
- 1))
542 /* This case is not handled correctly below. */
552 choose_multiplier (abs_d
, prec
, prec
- 1, &ml
,
553 &post_shift
, &dummy_int
);
554 if (ml
>= (unsigned HOST_WIDE_INT
) 1 << (prec
- 1))
556 this_mode
= 4 + (d
< 0);
557 ml
|= (~(unsigned HOST_WIDE_INT
) 0) << (prec
- 1);
560 this_mode
= 2 + (d
< 0);
563 post_shifts
[i
] = post_shift
;
564 if ((i
&& !has_vector_shift
&& post_shifts
[0] != post_shift
)
565 || post_shift
>= prec
566 || pre_shifts
[i
] >= prec
)
571 else if (mode
!= this_mode
)
575 vec
= XALLOCAVEC (tree
, nunits
);
579 tree addend
= NULL_TREE
;
584 /* Both division and remainder sequences need
585 op0 < 0 ? mask : 0 computed. It can be either computed as
586 (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
587 if none of the shifts is 0, or as the conditional. */
588 for (i
= 0; i
< nunits
; i
++)
592 = build_vector_type (build_nonstandard_integer_type (prec
, 1),
594 if (i
== nunits
&& TYPE_MODE (uns_type
) == TYPE_MODE (type
))
596 for (i
= 0; i
< nunits
; i
++)
597 shift_temps
[i
] = prec
- 1;
598 cur_op
= add_rshift (gsi
, type
, op0
, shift_temps
);
599 if (cur_op
!= NULL_TREE
)
601 cur_op
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
603 for (i
= 0; i
< nunits
; i
++)
604 shift_temps
[i
] = prec
- shifts
[i
];
605 cur_op
= add_rshift (gsi
, uns_type
, cur_op
, shift_temps
);
606 if (cur_op
!= NULL_TREE
)
607 addend
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
611 if (addend
== NULL_TREE
612 && expand_vec_cond_expr_p (type
, type
))
614 tree zero
, cst
, cond
;
617 zero
= build_zero_cst (type
);
618 cond
= build2 (LT_EXPR
, type
, op0
, zero
);
619 for (i
= 0; i
< nunits
; i
++)
620 vec
[i
] = build_int_cst (TREE_TYPE (type
),
621 ((unsigned HOST_WIDE_INT
) 1
623 cst
= build_vector (type
, vec
);
624 addend
= make_ssa_name (type
, NULL
);
625 stmt
= gimple_build_assign_with_ops (VEC_COND_EXPR
, addend
,
627 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
630 if (code
== TRUNC_DIV_EXPR
)
634 /* q = op0 >> shift; */
635 cur_op
= add_rshift (gsi
, type
, op0
, shifts
);
636 if (cur_op
!= NULL_TREE
)
639 else if (addend
!= NULL_TREE
)
641 /* t1 = op0 + addend;
643 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
644 if (op
!= unknown_optab
645 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
647 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
, addend
);
648 cur_op
= add_rshift (gsi
, type
, cur_op
, shifts
);
649 if (cur_op
!= NULL_TREE
)
657 for (i
= 0; i
< nunits
; i
++)
658 vec
[i
] = build_int_cst (TREE_TYPE (type
),
659 ((unsigned HOST_WIDE_INT
) 1
661 mask
= build_vector (type
, vec
);
662 op
= optab_for_tree_code (BIT_AND_EXPR
, type
, optab_default
);
663 if (op
!= unknown_optab
664 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
667 /* r = op0 & mask; */
668 return gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, op0
, mask
);
669 else if (addend
!= NULL_TREE
)
671 /* t1 = op0 + addend;
674 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
675 if (op
!= unknown_optab
676 && optab_handler (op
, TYPE_MODE (type
))
679 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
,
681 cur_op
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
,
683 op
= optab_for_tree_code (MINUS_EXPR
, type
,
685 if (op
!= unknown_optab
686 && optab_handler (op
, TYPE_MODE (type
))
688 return gimplify_build2 (gsi
, MINUS_EXPR
, type
,
696 if (mode
== -2 || BYTES_BIG_ENDIAN
!= WORDS_BIG_ENDIAN
)
699 if (!can_mult_highpart_p (TYPE_MODE (type
), TYPE_UNSIGNED (type
)))
707 gcc_assert (unsignedp
);
708 /* t1 = oprnd0 >> pre_shift;
710 q = t2 >> post_shift; */
711 cur_op
= add_rshift (gsi
, type
, cur_op
, pre_shifts
);
712 if (cur_op
== NULL_TREE
)
716 gcc_assert (unsignedp
);
717 for (i
= 0; i
< nunits
; i
++)
727 gcc_assert (!unsignedp
);
728 for (i
= 0; i
< nunits
; i
++)
729 shift_temps
[i
] = prec
- 1;
735 for (i
= 0; i
< nunits
; i
++)
736 vec
[i
] = build_int_cst (TREE_TYPE (type
), mulc
[i
]);
737 mulcst
= build_vector (type
, vec
);
739 cur_op
= gimplify_build2 (gsi
, MULT_HIGHPART_EXPR
, type
, cur_op
, mulcst
);
744 /* t1 = oprnd0 >> pre_shift;
746 q = t2 >> post_shift; */
747 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
750 /* t1 = oprnd0 h* ml;
754 q = t4 >> (post_shift - 1); */
755 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
756 if (op
== unknown_optab
757 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
759 tem
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, cur_op
);
760 tem
= add_rshift (gsi
, type
, tem
, shift_temps
);
761 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
762 if (op
== unknown_optab
763 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
765 tem
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, tem
);
766 cur_op
= add_rshift (gsi
, type
, tem
, post_shifts
);
767 if (cur_op
== NULL_TREE
)
774 /* t1 = oprnd0 h* ml;
775 t2 = t1; [ iff (mode & 2) != 0 ]
776 t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
777 t3 = t2 >> post_shift;
778 t4 = oprnd0 >> (prec - 1);
779 q = t3 - t4; [ iff (mode & 1) == 0 ]
780 q = t4 - t3; [ iff (mode & 1) != 0 ] */
783 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
784 if (op
== unknown_optab
785 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
787 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, op0
);
789 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
790 if (cur_op
== NULL_TREE
)
792 tem
= add_rshift (gsi
, type
, op0
, shift_temps
);
793 if (tem
== NULL_TREE
)
795 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
796 if (op
== unknown_optab
797 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
800 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, cur_op
, tem
);
802 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, tem
, cur_op
);
808 if (code
== TRUNC_DIV_EXPR
)
811 /* We divided. Now finish by:
814 op
= optab_for_tree_code (MULT_EXPR
, type
, optab_default
);
815 if (op
== unknown_optab
816 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
818 tem
= gimplify_build2 (gsi
, MULT_EXPR
, type
, cur_op
, op1
);
819 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
820 if (op
== unknown_optab
821 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
823 return gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, tem
);
826 /* Expand a vector condition to scalars, by using many conditions
827 on the vector's elements. */
829 expand_vector_condition (gimple_stmt_iterator
*gsi
)
831 gimple stmt
= gsi_stmt (*gsi
);
832 tree type
= gimple_expr_type (stmt
);
833 tree a
= gimple_assign_rhs1 (stmt
);
836 bool a_is_comparison
= false;
837 tree b
= gimple_assign_rhs2 (stmt
);
838 tree c
= gimple_assign_rhs3 (stmt
);
839 vec
<constructor_elt
, va_gc
> *v
;
841 tree inner_type
= TREE_TYPE (type
);
842 tree cond_type
= TREE_TYPE (TREE_TYPE (a
));
843 tree comp_inner_type
= cond_type
;
844 tree width
= TYPE_SIZE (inner_type
);
845 tree index
= bitsize_int (0);
846 int nunits
= TYPE_VECTOR_SUBPARTS (type
);
848 location_t loc
= gimple_location (gsi_stmt (*gsi
));
850 if (!is_gimple_val (a
))
852 gcc_assert (COMPARISON_CLASS_P (a
));
853 a_is_comparison
= true;
854 a1
= TREE_OPERAND (a
, 0);
855 a2
= TREE_OPERAND (a
, 1);
856 comp_inner_type
= TREE_TYPE (TREE_TYPE (a1
));
859 if (expand_vec_cond_expr_p (type
, TREE_TYPE (a1
)))
862 /* TODO: try and find a smaller vector type. */
864 warning_at (loc
, OPT_Wvector_operation_performance
,
865 "vector condition will be expanded piecewise");
867 vec_alloc (v
, nunits
);
868 for (i
= 0; i
< nunits
;
869 i
++, index
= int_const_binop (PLUS_EXPR
, index
, width
))
872 tree bb
= tree_vec_extract (gsi
, inner_type
, b
, width
, index
);
873 tree cc
= tree_vec_extract (gsi
, inner_type
, c
, width
, index
);
876 tree aa1
= tree_vec_extract (gsi
, comp_inner_type
, a1
, width
, index
);
877 tree aa2
= tree_vec_extract (gsi
, comp_inner_type
, a2
, width
, index
);
878 aa
= build2 (TREE_CODE (a
), cond_type
, aa1
, aa2
);
881 aa
= tree_vec_extract (gsi
, cond_type
, a
, width
, index
);
882 result
= gimplify_build3 (gsi
, COND_EXPR
, inner_type
, aa
, bb
, cc
);
883 constructor_elt ce
= {NULL_TREE
, result
};
887 constr
= build_constructor (type
, v
);
888 gimple_assign_set_rhs_from_tree (gsi
, constr
);
889 update_stmt (gsi_stmt (*gsi
));
893 expand_vector_operation (gimple_stmt_iterator
*gsi
, tree type
, tree compute_type
,
894 gimple assign
, enum tree_code code
)
896 enum machine_mode compute_mode
= TYPE_MODE (compute_type
);
898 /* If the compute mode is not a vector mode (hence we are not decomposing
899 a BLKmode vector to smaller, hardware-supported vectors), we may want
900 to expand the operations in parallel. */
901 if (GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_INT
902 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_FLOAT
903 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_FRACT
904 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_UFRACT
905 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_ACCUM
906 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_UACCUM
)
911 if (!TYPE_OVERFLOW_TRAPS (type
))
912 return expand_vector_addition (gsi
, do_binop
, do_plus_minus
, type
,
913 gimple_assign_rhs1 (assign
),
914 gimple_assign_rhs2 (assign
), code
);
918 if (!TYPE_OVERFLOW_TRAPS (type
))
919 return expand_vector_addition (gsi
, do_unop
, do_negate
, type
,
920 gimple_assign_rhs1 (assign
),
927 return expand_vector_parallel (gsi
, do_binop
, type
,
928 gimple_assign_rhs1 (assign
),
929 gimple_assign_rhs2 (assign
), code
);
932 return expand_vector_parallel (gsi
, do_unop
, type
,
933 gimple_assign_rhs1 (assign
),
950 tree rhs1
= gimple_assign_rhs1 (assign
);
951 tree rhs2
= gimple_assign_rhs2 (assign
);
953 return expand_vector_comparison (gsi
, type
, rhs1
, rhs2
, code
);
959 tree rhs1
= gimple_assign_rhs1 (assign
);
960 tree rhs2
= gimple_assign_rhs2 (assign
);
964 || !VECTOR_INTEGER_TYPE_P (type
)
965 || TREE_CODE (rhs2
) != VECTOR_CST
)
968 ret
= expand_vector_divmod (gsi
, type
, rhs1
, rhs2
, code
);
969 if (ret
!= NULL_TREE
)
978 if (TREE_CODE_CLASS (code
) == tcc_unary
)
979 return expand_vector_piecewise (gsi
, do_unop
, type
, compute_type
,
980 gimple_assign_rhs1 (assign
),
983 return expand_vector_piecewise (gsi
, do_binop
, type
, compute_type
,
984 gimple_assign_rhs1 (assign
),
985 gimple_assign_rhs2 (assign
), code
);
988 /* Return a type for the widest vector mode whose components are of type
989 TYPE, or NULL_TREE if none is found. */
992 type_for_widest_vector_mode (tree type
, optab op
)
994 enum machine_mode inner_mode
= TYPE_MODE (type
);
995 enum machine_mode best_mode
= VOIDmode
, mode
;
998 if (SCALAR_FLOAT_MODE_P (inner_mode
))
999 mode
= MIN_MODE_VECTOR_FLOAT
;
1000 else if (SCALAR_FRACT_MODE_P (inner_mode
))
1001 mode
= MIN_MODE_VECTOR_FRACT
;
1002 else if (SCALAR_UFRACT_MODE_P (inner_mode
))
1003 mode
= MIN_MODE_VECTOR_UFRACT
;
1004 else if (SCALAR_ACCUM_MODE_P (inner_mode
))
1005 mode
= MIN_MODE_VECTOR_ACCUM
;
1006 else if (SCALAR_UACCUM_MODE_P (inner_mode
))
1007 mode
= MIN_MODE_VECTOR_UACCUM
;
1009 mode
= MIN_MODE_VECTOR_INT
;
1011 for (; mode
!= VOIDmode
; mode
= GET_MODE_WIDER_MODE (mode
))
1012 if (GET_MODE_INNER (mode
) == inner_mode
1013 && GET_MODE_NUNITS (mode
) > best_nunits
1014 && optab_handler (op
, mode
) != CODE_FOR_nothing
)
1015 best_mode
= mode
, best_nunits
= GET_MODE_NUNITS (mode
);
1017 if (best_mode
== VOIDmode
)
1020 return build_vector_type_for_mode (type
, best_mode
);
1024 /* Build a reference to the element of the vector VECT. Function
1025 returns either the element itself, either BIT_FIELD_REF, or an
1026 ARRAY_REF expression.
1028 GSI is required to insert temporary variables while building a
1029 refernece to the element of the vector VECT.
1031 PTMPVEC is a pointer to the temporary variable for caching
1032 purposes. In case when PTMPVEC is NULL new temporary variable
1035 vector_element (gimple_stmt_iterator
*gsi
, tree vect
, tree idx
, tree
*ptmpvec
)
1037 tree vect_type
, vect_elt_type
;
1041 bool need_asgn
= true;
1042 unsigned int elements
;
1044 vect_type
= TREE_TYPE (vect
);
1045 vect_elt_type
= TREE_TYPE (vect_type
);
1046 elements
= TYPE_VECTOR_SUBPARTS (vect_type
);
1048 if (TREE_CODE (idx
) == INTEGER_CST
)
1050 unsigned HOST_WIDE_INT index
;
1052 /* Given that we're about to compute a binary modulus,
1053 we don't care about the high bits of the value. */
1054 index
= TREE_INT_CST_LOW (idx
);
1055 if (!host_integerp (idx
, 1) || index
>= elements
)
1057 index
&= elements
- 1;
1058 idx
= build_int_cst (TREE_TYPE (idx
), index
);
1061 /* When lowering a vector statement sequence do some easy
1062 simplification by looking through intermediate vector results. */
1063 if (TREE_CODE (vect
) == SSA_NAME
)
1065 gimple def_stmt
= SSA_NAME_DEF_STMT (vect
);
1066 if (is_gimple_assign (def_stmt
)
1067 && (gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
1068 || gimple_assign_rhs_code (def_stmt
) == CONSTRUCTOR
))
1069 vect
= gimple_assign_rhs1 (def_stmt
);
1072 if (TREE_CODE (vect
) == VECTOR_CST
)
1073 return VECTOR_CST_ELT (vect
, index
);
1074 else if (TREE_CODE (vect
) == CONSTRUCTOR
1075 && (CONSTRUCTOR_NELTS (vect
) == 0
1076 || TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (vect
, 0)->value
))
1079 if (index
< CONSTRUCTOR_NELTS (vect
))
1080 return CONSTRUCTOR_ELT (vect
, index
)->value
;
1081 return build_zero_cst (vect_elt_type
);
1085 tree size
= TYPE_SIZE (vect_elt_type
);
1086 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
, bitsize_int (index
),
1088 return fold_build3 (BIT_FIELD_REF
, vect_elt_type
, vect
, size
, pos
);
1093 tmpvec
= create_tmp_var (vect_type
, "vectmp");
1095 tmpvec
= *ptmpvec
= create_tmp_var (vect_type
, "vectmp");
1104 TREE_ADDRESSABLE (tmpvec
) = 1;
1105 asgn
= gimple_build_assign (tmpvec
, vect
);
1106 gsi_insert_before (gsi
, asgn
, GSI_SAME_STMT
);
1109 arraytype
= build_array_type_nelts (vect_elt_type
, elements
);
1110 return build4 (ARRAY_REF
, vect_elt_type
,
1111 build1 (VIEW_CONVERT_EXPR
, arraytype
, tmpvec
),
1112 idx
, NULL_TREE
, NULL_TREE
);
1115 /* Check if VEC_PERM_EXPR within the given setting is supported
1116 by hardware, or lower it piecewise.
1118 When VEC_PERM_EXPR has the same first and second operands:
1119 VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1120 {v0[mask[0]], v0[mask[1]], ...}
1121 MASK and V0 must have the same number of elements.
1123 Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1124 {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1125 V0 and V1 must have the same type. MASK, V0, V1 must have the
1126 same number of arguments. */
1129 lower_vec_perm (gimple_stmt_iterator
*gsi
)
1131 gimple stmt
= gsi_stmt (*gsi
);
1132 tree mask
= gimple_assign_rhs3 (stmt
);
1133 tree vec0
= gimple_assign_rhs1 (stmt
);
1134 tree vec1
= gimple_assign_rhs2 (stmt
);
1135 tree vect_type
= TREE_TYPE (vec0
);
1136 tree mask_type
= TREE_TYPE (mask
);
1137 tree vect_elt_type
= TREE_TYPE (vect_type
);
1138 tree mask_elt_type
= TREE_TYPE (mask_type
);
1139 unsigned int elements
= TYPE_VECTOR_SUBPARTS (vect_type
);
1140 vec
<constructor_elt
, va_gc
> *v
;
1141 tree constr
, t
, si
, i_val
;
1142 tree vec0tmp
= NULL_TREE
, vec1tmp
= NULL_TREE
, masktmp
= NULL_TREE
;
1143 bool two_operand_p
= !operand_equal_p (vec0
, vec1
, 0);
1144 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1147 if (TREE_CODE (mask
) == SSA_NAME
)
1149 gimple def_stmt
= SSA_NAME_DEF_STMT (mask
);
1150 if (is_gimple_assign (def_stmt
)
1151 && gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
)
1152 mask
= gimple_assign_rhs1 (def_stmt
);
1155 if (TREE_CODE (mask
) == VECTOR_CST
)
1157 unsigned char *sel_int
= XALLOCAVEC (unsigned char, elements
);
1159 for (i
= 0; i
< elements
; ++i
)
1160 sel_int
[i
] = (TREE_INT_CST_LOW (VECTOR_CST_ELT (mask
, i
))
1161 & (2 * elements
- 1));
1163 if (can_vec_perm_p (TYPE_MODE (vect_type
), false, sel_int
))
1165 gimple_assign_set_rhs3 (stmt
, mask
);
1170 else if (can_vec_perm_p (TYPE_MODE (vect_type
), true, NULL
))
1173 warning_at (loc
, OPT_Wvector_operation_performance
,
1174 "vector shuffling operation will be expanded piecewise");
1176 vec_alloc (v
, elements
);
1177 for (i
= 0; i
< elements
; i
++)
1180 i_val
= vector_element (gsi
, mask
, si
, &masktmp
);
1182 if (TREE_CODE (i_val
) == INTEGER_CST
)
1184 unsigned HOST_WIDE_INT index
;
1186 index
= TREE_INT_CST_LOW (i_val
);
1187 if (!host_integerp (i_val
, 1) || index
>= elements
)
1188 i_val
= build_int_cst (mask_elt_type
, index
& (elements
- 1));
1190 if (two_operand_p
&& (index
& elements
) != 0)
1191 t
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1193 t
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1195 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
1196 true, GSI_SAME_STMT
);
1200 tree cond
= NULL_TREE
, v0_val
;
1204 cond
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1205 build_int_cst (mask_elt_type
, elements
));
1206 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1207 true, GSI_SAME_STMT
);
1210 i_val
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1211 build_int_cst (mask_elt_type
, elements
- 1));
1212 i_val
= force_gimple_operand_gsi (gsi
, i_val
, true, NULL_TREE
,
1213 true, GSI_SAME_STMT
);
1215 v0_val
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1216 v0_val
= force_gimple_operand_gsi (gsi
, v0_val
, true, NULL_TREE
,
1217 true, GSI_SAME_STMT
);
1223 v1_val
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1224 v1_val
= force_gimple_operand_gsi (gsi
, v1_val
, true, NULL_TREE
,
1225 true, GSI_SAME_STMT
);
1227 cond
= fold_build2 (EQ_EXPR
, boolean_type_node
,
1228 cond
, build_zero_cst (mask_elt_type
));
1229 cond
= fold_build3 (COND_EXPR
, vect_elt_type
,
1230 cond
, v0_val
, v1_val
);
1231 t
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1232 true, GSI_SAME_STMT
);
1238 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, t
);
1241 constr
= build_constructor (vect_type
, v
);
1242 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1243 update_stmt (gsi_stmt (*gsi
));
1246 /* Process one statement. If we identify a vector operation, expand it. */
1249 expand_vector_operations_1 (gimple_stmt_iterator
*gsi
)
1251 gimple stmt
= gsi_stmt (*gsi
);
1252 tree lhs
, rhs1
, rhs2
= NULL
, type
, compute_type
;
1253 enum tree_code code
;
1254 enum machine_mode compute_mode
;
1255 optab op
= unknown_optab
;
1256 enum gimple_rhs_class rhs_class
;
1259 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
1262 code
= gimple_assign_rhs_code (stmt
);
1263 rhs_class
= get_gimple_rhs_class (code
);
1264 lhs
= gimple_assign_lhs (stmt
);
1266 if (code
== VEC_PERM_EXPR
)
1268 lower_vec_perm (gsi
);
1272 if (code
== VEC_COND_EXPR
)
1274 expand_vector_condition (gsi
);
1277 if (rhs_class
!= GIMPLE_UNARY_RHS
&& rhs_class
!= GIMPLE_BINARY_RHS
)
1280 rhs1
= gimple_assign_rhs1 (stmt
);
1281 type
= gimple_expr_type (stmt
);
1282 if (rhs_class
== GIMPLE_BINARY_RHS
)
1283 rhs2
= gimple_assign_rhs2 (stmt
);
1285 if (TREE_CODE (type
) != VECTOR_TYPE
)
1288 if (code
== NOP_EXPR
1289 || code
== FLOAT_EXPR
1290 || code
== FIX_TRUNC_EXPR
1291 || code
== VIEW_CONVERT_EXPR
)
1294 gcc_assert (code
!= CONVERT_EXPR
);
1296 /* The signedness is determined from input argument. */
1297 if (code
== VEC_UNPACK_FLOAT_HI_EXPR
1298 || code
== VEC_UNPACK_FLOAT_LO_EXPR
)
1299 type
= TREE_TYPE (rhs1
);
1301 /* For widening/narrowing vector operations, the relevant type is of the
1302 arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
1303 calculated in the same way above. */
1304 if (code
== WIDEN_SUM_EXPR
1305 || code
== VEC_WIDEN_MULT_HI_EXPR
1306 || code
== VEC_WIDEN_MULT_LO_EXPR
1307 || code
== VEC_WIDEN_MULT_EVEN_EXPR
1308 || code
== VEC_WIDEN_MULT_ODD_EXPR
1309 || code
== VEC_UNPACK_HI_EXPR
1310 || code
== VEC_UNPACK_LO_EXPR
1311 || code
== VEC_PACK_TRUNC_EXPR
1312 || code
== VEC_PACK_SAT_EXPR
1313 || code
== VEC_PACK_FIX_TRUNC_EXPR
1314 || code
== VEC_WIDEN_LSHIFT_HI_EXPR
1315 || code
== VEC_WIDEN_LSHIFT_LO_EXPR
)
1316 type
= TREE_TYPE (rhs1
);
1318 /* Choose between vector shift/rotate by vector and vector shift/rotate by
1320 if (code
== LSHIFT_EXPR
1321 || code
== RSHIFT_EXPR
1322 || code
== LROTATE_EXPR
1323 || code
== RROTATE_EXPR
)
1327 /* Check whether we have vector <op> {x,x,x,x} where x
1328 could be a scalar variable or a constant. Transform
1329 vector <op> {x,x,x,x} ==> vector <op> scalar. */
1330 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
1335 if ((TREE_CODE (rhs2
) == VECTOR_CST
1336 && (first
= uniform_vector_p (rhs2
)) != NULL_TREE
)
1337 || (TREE_CODE (rhs2
) == SSA_NAME
1338 && (def_stmt
= SSA_NAME_DEF_STMT (rhs2
))
1339 && gimple_assign_single_p (def_stmt
)
1340 && (first
= uniform_vector_p
1341 (gimple_assign_rhs1 (def_stmt
))) != NULL_TREE
))
1343 gimple_assign_set_rhs2 (stmt
, first
);
1349 opv
= optab_for_tree_code (code
, type
, optab_vector
);
1350 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
1354 op
= optab_for_tree_code (code
, type
, optab_scalar
);
1356 /* The rtl expander will expand vector/scalar as vector/vector
1357 if necessary. Don't bother converting the stmt here. */
1358 if (optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
1359 && optab_handler (opv
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
1364 op
= optab_for_tree_code (code
, type
, optab_default
);
1366 /* Optabs will try converting a negation into a subtraction, so
1367 look for it as well. TODO: negation of floating-point vectors
1368 might be turned into an exclusive OR toggling the sign bit. */
1369 if (op
== unknown_optab
1370 && code
== NEGATE_EXPR
1371 && INTEGRAL_TYPE_P (TREE_TYPE (type
)))
1372 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
1374 /* For very wide vectors, try using a smaller vector mode. */
1375 compute_type
= type
;
1376 if (!VECTOR_MODE_P (TYPE_MODE (type
)) && op
)
1378 tree vector_compute_type
1379 = type_for_widest_vector_mode (TREE_TYPE (type
), op
);
1380 if (vector_compute_type
!= NULL_TREE
1381 && (TYPE_VECTOR_SUBPARTS (vector_compute_type
)
1382 < TYPE_VECTOR_SUBPARTS (compute_type
))
1383 && (optab_handler (op
, TYPE_MODE (vector_compute_type
))
1384 != CODE_FOR_nothing
))
1385 compute_type
= vector_compute_type
;
1388 /* If we are breaking a BLKmode vector into smaller pieces,
1389 type_for_widest_vector_mode has already looked into the optab,
1390 so skip these checks. */
1391 if (compute_type
== type
)
1393 compute_mode
= TYPE_MODE (compute_type
);
1394 if (VECTOR_MODE_P (compute_mode
))
1396 if (op
&& optab_handler (op
, compute_mode
) != CODE_FOR_nothing
)
1398 if (code
== MULT_HIGHPART_EXPR
1399 && can_mult_highpart_p (compute_mode
,
1400 TYPE_UNSIGNED (compute_type
)))
1403 /* There is no operation in hardware, so fall back to scalars. */
1404 compute_type
= TREE_TYPE (type
);
1407 gcc_assert (code
!= VEC_LSHIFT_EXPR
&& code
!= VEC_RSHIFT_EXPR
);
1408 new_rhs
= expand_vector_operation (gsi
, type
, compute_type
, stmt
, code
);
1410 /* Leave expression untouched for later expansion. */
1411 if (new_rhs
== NULL_TREE
)
1414 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
1415 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
1418 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1419 way to do it is change expand_vector_operation and its callees to
1420 return a tree_code, RHS1 and RHS2 instead of a tree. */
1421 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
1422 update_stmt (gsi_stmt (*gsi
));
1425 /* Use this to lower vector operations introduced by the vectorizer,
1426 if it may need the bit-twiddling tricks implemented in this file. */
1429 gate_expand_vector_operations_ssa (void)
1431 return !(cfun
->curr_properties
& PROP_gimple_lvec
);
1435 expand_vector_operations (void)
1437 gimple_stmt_iterator gsi
;
1439 bool cfg_changed
= false;
1443 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1445 expand_vector_operations_1 (&gsi
);
1446 /* ??? If we do not cleanup EH then we will ICE in
1447 verification. But in reality we have created wrong-code
1448 as we did not properly transition EH info and edges to
1449 the piecewise computations. */
1450 if (maybe_clean_eh_stmt (gsi_stmt (gsi
))
1451 && gimple_purge_dead_eh_edges (bb
))
1456 return cfg_changed
? TODO_cleanup_cfg
: 0;
1461 const pass_data pass_data_lower_vector
=
1463 GIMPLE_PASS
, /* type */
1464 "veclower", /* name */
1465 OPTGROUP_VEC
, /* optinfo_flags */
1466 true, /* has_gate */
1467 true, /* has_execute */
1468 TV_NONE
, /* tv_id */
1469 PROP_cfg
, /* properties_required */
1470 PROP_gimple_lvec
, /* properties_provided */
1471 0, /* properties_destroyed */
1472 0, /* todo_flags_start */
1473 ( TODO_update_ssa
| TODO_verify_ssa
1476 | TODO_cleanup_cfg
), /* todo_flags_finish */
1479 class pass_lower_vector
: public gimple_opt_pass
1482 pass_lower_vector (gcc::context
*ctxt
)
1483 : gimple_opt_pass (pass_data_lower_vector
, ctxt
)
1486 /* opt_pass methods: */
1487 bool gate () { return gate_expand_vector_operations_ssa (); }
1488 unsigned int execute () { return expand_vector_operations (); }
1490 }; // class pass_lower_vector
1495 make_pass_lower_vector (gcc::context
*ctxt
)
1497 return new pass_lower_vector (ctxt
);
1502 const pass_data pass_data_lower_vector_ssa
=
1504 GIMPLE_PASS
, /* type */
1505 "veclower2", /* name */
1506 OPTGROUP_VEC
, /* optinfo_flags */
1507 false, /* has_gate */
1508 true, /* has_execute */
1509 TV_NONE
, /* tv_id */
1510 PROP_cfg
, /* properties_required */
1511 PROP_gimple_lvec
, /* properties_provided */
1512 0, /* properties_destroyed */
1513 0, /* todo_flags_start */
1514 ( TODO_update_ssa
| TODO_verify_ssa
1517 | TODO_cleanup_cfg
), /* todo_flags_finish */
1520 class pass_lower_vector_ssa
: public gimple_opt_pass
1523 pass_lower_vector_ssa (gcc::context
*ctxt
)
1524 : gimple_opt_pass (pass_data_lower_vector_ssa
, ctxt
)
1527 /* opt_pass methods: */
1528 opt_pass
* clone () { return new pass_lower_vector_ssa (m_ctxt
); }
1529 unsigned int execute () { return expand_vector_operations (); }
1531 }; // class pass_lower_vector_ssa
1536 make_pass_lower_vector_ssa (gcc::context
*ctxt
)
1538 return new pass_lower_vector_ssa (ctxt
);
1541 #include "gt-tree-vect-generic.h"