1 /* Lower vector operations to scalar operations.
2 Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
3 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by the
9 Free Software Foundation; either version 3, or (at your option) any
12 GCC is distributed in the hope that it will be useful, but WITHOUT
13 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
26 #include "langhooks.h"
27 #include "tree-flow.h"
29 #include "tree-iterator.h"
30 #include "tree-pass.h"
33 #include "diagnostic.h"
36 /* Need to include rtl.h, expr.h, etc. for optabs. */
41 static void expand_vector_operations_1 (gimple_stmt_iterator
*);
44 /* Build a constant of type TYPE, made of VALUE's bits replicated
45 every TYPE_SIZE (INNER_TYPE) bits to fit TYPE's precision. */
47 build_replicated_const (tree type
, tree inner_type
, HOST_WIDE_INT value
)
49 int width
= tree_low_cst (TYPE_SIZE (inner_type
), 1);
50 int n
= HOST_BITS_PER_WIDE_INT
/ width
;
51 unsigned HOST_WIDE_INT low
, high
, mask
;
56 if (width
== HOST_BITS_PER_WIDE_INT
)
60 mask
= ((HOST_WIDE_INT
)1 << width
) - 1;
61 low
= (unsigned HOST_WIDE_INT
) ~0 / mask
* (value
& mask
);
64 if (TYPE_PRECISION (type
) < HOST_BITS_PER_WIDE_INT
)
65 low
&= ((HOST_WIDE_INT
)1 << TYPE_PRECISION (type
)) - 1, high
= 0;
66 else if (TYPE_PRECISION (type
) == HOST_BITS_PER_WIDE_INT
)
68 else if (TYPE_PRECISION (type
) == HOST_BITS_PER_DOUBLE_INT
)
73 ret
= build_int_cst_wide (type
, low
, high
);
77 static GTY(()) tree vector_inner_type
;
78 static GTY(()) tree vector_last_type
;
79 static GTY(()) int vector_last_nunits
;
81 /* Return a suitable vector types made of SUBPARTS units each of mode
82 "word_mode" (the global variable). */
84 build_word_mode_vector_type (int nunits
)
86 if (!vector_inner_type
)
87 vector_inner_type
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
88 else if (vector_last_nunits
== nunits
)
90 gcc_assert (TREE_CODE (vector_last_type
) == VECTOR_TYPE
);
91 return vector_last_type
;
94 /* We build a new type, but we canonicalize it nevertheless,
95 because it still saves some memory. */
96 vector_last_nunits
= nunits
;
97 vector_last_type
= type_hash_canon (nunits
,
98 build_vector_type (vector_inner_type
,
100 return vector_last_type
;
103 typedef tree (*elem_op_func
) (gimple_stmt_iterator
*,
104 tree
, tree
, tree
, tree
, tree
, enum tree_code
);
107 tree_vec_extract (gimple_stmt_iterator
*gsi
, tree type
,
108 tree t
, tree bitsize
, tree bitpos
)
111 return gimplify_build3 (gsi
, BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
113 return gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, type
, t
);
117 do_unop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
118 tree b ATTRIBUTE_UNUSED
, tree bitpos
, tree bitsize
,
121 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
122 return gimplify_build1 (gsi
, code
, inner_type
, a
);
126 do_binop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
127 tree bitpos
, tree bitsize
, enum tree_code code
)
129 if (TREE_CODE (TREE_TYPE (a
)) == VECTOR_TYPE
)
130 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
131 if (TREE_CODE (TREE_TYPE (b
)) == VECTOR_TYPE
)
132 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
133 return gimplify_build2 (gsi
, code
, inner_type
, a
, b
);
136 /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
138 INNER_TYPE is the type of A and B elements
140 returned expression is of signed integer type with the
141 size equal to the size of INNER_TYPE. */
143 do_compare (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
144 tree bitpos
, tree bitsize
, enum tree_code code
)
148 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
149 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
151 comp_type
= build_nonstandard_integer_type
152 (GET_MODE_BITSIZE (TYPE_MODE (inner_type
)), 0);
154 return gimplify_build3 (gsi
, COND_EXPR
, comp_type
,
155 fold_build2 (code
, boolean_type_node
, a
, b
),
156 build_int_cst (comp_type
, -1),
157 build_int_cst (comp_type
, 0));
160 /* Expand vector addition to scalars. This does bit twiddling
161 in order to increase parallelism:
163 a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
166 a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
167 (a ^ ~b) & 0x80808080
169 -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
171 This optimization should be done only if 4 vector items or more
174 do_plus_minus (gimple_stmt_iterator
*gsi
, tree word_type
, tree a
, tree b
,
175 tree bitpos ATTRIBUTE_UNUSED
, tree bitsize ATTRIBUTE_UNUSED
,
178 tree inner_type
= TREE_TYPE (TREE_TYPE (a
));
179 unsigned HOST_WIDE_INT max
;
180 tree low_bits
, high_bits
, a_low
, b_low
, result_low
, signs
;
182 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
183 low_bits
= build_replicated_const (word_type
, inner_type
, max
>> 1);
184 high_bits
= build_replicated_const (word_type
, inner_type
, max
& ~(max
>> 1));
186 a
= tree_vec_extract (gsi
, word_type
, a
, bitsize
, bitpos
);
187 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
189 signs
= gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, a
, b
);
190 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
191 if (code
== PLUS_EXPR
)
192 a_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, a
, low_bits
);
195 a_low
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, word_type
, a
, high_bits
);
196 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, signs
);
199 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
200 result_low
= gimplify_build2 (gsi
, code
, word_type
, a_low
, b_low
);
201 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
205 do_negate (gimple_stmt_iterator
*gsi
, tree word_type
, tree b
,
206 tree unused ATTRIBUTE_UNUSED
, tree bitpos ATTRIBUTE_UNUSED
,
207 tree bitsize ATTRIBUTE_UNUSED
,
208 enum tree_code code ATTRIBUTE_UNUSED
)
210 tree inner_type
= TREE_TYPE (TREE_TYPE (b
));
212 tree low_bits
, high_bits
, b_low
, result_low
, signs
;
214 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
215 low_bits
= build_replicated_const (word_type
, inner_type
, max
>> 1);
216 high_bits
= build_replicated_const (word_type
, inner_type
, max
& ~(max
>> 1));
218 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
220 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
221 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, b
);
222 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
223 result_low
= gimplify_build2 (gsi
, MINUS_EXPR
, word_type
, high_bits
, b_low
);
224 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
227 /* Expand a vector operation to scalars, by using many operations
228 whose type is the vector type's inner type. */
230 expand_vector_piecewise (gimple_stmt_iterator
*gsi
, elem_op_func f
,
231 tree type
, tree inner_type
,
232 tree a
, tree b
, enum tree_code code
)
234 VEC(constructor_elt
,gc
) *v
;
235 tree part_width
= TYPE_SIZE (inner_type
);
236 tree index
= bitsize_int (0);
237 int nunits
= TYPE_VECTOR_SUBPARTS (type
);
238 int delta
= tree_low_cst (part_width
, 1)
239 / tree_low_cst (TYPE_SIZE (TREE_TYPE (type
)), 1);
241 location_t loc
= gimple_location (gsi_stmt (*gsi
));
243 if (types_compatible_p (gimple_expr_type (gsi_stmt (*gsi
)), type
))
244 warning_at (loc
, OPT_Wvector_operation_performance
,
245 "vector operation will be expanded piecewise");
247 warning_at (loc
, OPT_Wvector_operation_performance
,
248 "vector operation will be expanded in parallel");
250 v
= VEC_alloc(constructor_elt
, gc
, (nunits
+ delta
- 1) / delta
);
251 for (i
= 0; i
< nunits
;
252 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
254 tree result
= f (gsi
, inner_type
, a
, b
, index
, part_width
, code
);
255 constructor_elt
*ce
= VEC_quick_push (constructor_elt
, v
, NULL
);
256 ce
->index
= NULL_TREE
;
260 return build_constructor (type
, v
);
263 /* Expand a vector operation to scalars with the freedom to use
264 a scalar integer type, or to use a different size for the items
265 in the vector type. */
267 expand_vector_parallel (gimple_stmt_iterator
*gsi
, elem_op_func f
, tree type
,
271 tree result
, compute_type
;
272 enum machine_mode mode
;
273 int n_words
= tree_low_cst (TYPE_SIZE_UNIT (type
), 1) / UNITS_PER_WORD
;
274 location_t loc
= gimple_location (gsi_stmt (*gsi
));
276 /* We have three strategies. If the type is already correct, just do
277 the operation an element at a time. Else, if the vector is wider than
278 one word, do it a word at a time; finally, if the vector is smaller
279 than one word, do it as a scalar. */
280 if (TYPE_MODE (TREE_TYPE (type
)) == word_mode
)
281 return expand_vector_piecewise (gsi
, f
,
282 type
, TREE_TYPE (type
),
284 else if (n_words
> 1)
286 tree word_type
= build_word_mode_vector_type (n_words
);
287 result
= expand_vector_piecewise (gsi
, f
,
288 word_type
, TREE_TYPE (word_type
),
290 result
= force_gimple_operand_gsi (gsi
, result
, true, NULL
, true,
295 /* Use a single scalar operation with a mode no wider than word_mode. */
296 mode
= mode_for_size (tree_low_cst (TYPE_SIZE (type
), 1), MODE_INT
, 0);
297 compute_type
= lang_hooks
.types
.type_for_mode (mode
, 1);
298 result
= f (gsi
, compute_type
, a
, b
, NULL_TREE
, NULL_TREE
, code
);
299 warning_at (loc
, OPT_Wvector_operation_performance
,
300 "vector operation will be expanded with a "
301 "single scalar operation");
307 /* Expand a vector operation to scalars; for integer types we can use
308 special bit twiddling tricks to do the sums a word at a time, using
309 function F_PARALLEL instead of F. These tricks are done only if
310 they can process at least four items, that is, only if the vector
311 holds at least four items and if a word can hold four items. */
313 expand_vector_addition (gimple_stmt_iterator
*gsi
,
314 elem_op_func f
, elem_op_func f_parallel
,
315 tree type
, tree a
, tree b
, enum tree_code code
)
317 int parts_per_word
= UNITS_PER_WORD
318 / tree_low_cst (TYPE_SIZE_UNIT (TREE_TYPE (type
)), 1);
320 if (INTEGRAL_TYPE_P (TREE_TYPE (type
))
321 && parts_per_word
>= 4
322 && TYPE_VECTOR_SUBPARTS (type
) >= 4)
323 return expand_vector_parallel (gsi
, f_parallel
,
326 return expand_vector_piecewise (gsi
, f
,
327 type
, TREE_TYPE (type
),
331 /* Check if vector VEC consists of all the equal elements and
332 that the number of elements corresponds to the type of VEC.
333 The function returns first element of the vector
334 or NULL_TREE if the vector is not uniform. */
336 uniform_vector_p (tree vec
)
341 if (vec
== NULL_TREE
)
344 if (TREE_CODE (vec
) == VECTOR_CST
)
346 first
= VECTOR_CST_ELT (vec
, 0);
347 for (i
= 1; i
< VECTOR_CST_NELTS (vec
); ++i
)
348 if (!operand_equal_p (first
, VECTOR_CST_ELT (vec
, i
), 0))
354 else if (TREE_CODE (vec
) == CONSTRUCTOR
)
356 first
= error_mark_node
;
358 FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (vec
), i
, t
)
365 if (!operand_equal_p (first
, t
, 0))
368 if (i
!= TYPE_VECTOR_SUBPARTS (TREE_TYPE (vec
)))
377 /* Try to expand vector comparison expression OP0 CODE OP1 by
378 querying optab if the following expression:
379 VEC_COND_EXPR< OP0 CODE OP1, {-1,...}, {0,...}>
382 expand_vector_comparison (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
383 tree op1
, enum tree_code code
)
386 if (! expand_vec_cond_expr_p (type
, TREE_TYPE (op0
)))
387 t
= expand_vector_piecewise (gsi
, do_compare
, type
,
388 TREE_TYPE (TREE_TYPE (op0
)), op0
, op1
, code
);
395 /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
396 of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
397 the result if successful, otherwise return NULL_TREE. */
399 add_rshift (gimple_stmt_iterator
*gsi
, tree type
, tree op0
, int *shiftcnts
)
402 unsigned int i
, nunits
= TYPE_VECTOR_SUBPARTS (type
);
403 bool scalar_shift
= true;
405 for (i
= 1; i
< nunits
; i
++)
407 if (shiftcnts
[i
] != shiftcnts
[0])
408 scalar_shift
= false;
411 if (scalar_shift
&& shiftcnts
[0] == 0)
416 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_scalar
);
417 if (op
!= unknown_optab
418 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
419 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
420 build_int_cst (NULL_TREE
, shiftcnts
[0]));
423 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
424 if (op
!= unknown_optab
425 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
427 tree
*vec
= XALLOCAVEC (tree
, nunits
);
428 for (i
= 0; i
< nunits
; i
++)
429 vec
[i
] = build_int_cst (TREE_TYPE (type
), shiftcnts
[i
]);
430 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
431 build_vector (type
, vec
));
437 /* Try to expand integer vector division by constant using
438 widening multiply, shifts and additions. */
440 expand_vector_divmod (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
441 tree op1
, enum tree_code code
)
443 bool use_pow2
= true;
444 bool has_vector_shift
= true;
445 int mode
= -1, this_mode
;
446 int pre_shift
= -1, post_shift
;
447 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (type
);
448 int *shifts
= XALLOCAVEC (int, nunits
* 4);
449 int *pre_shifts
= shifts
+ nunits
;
450 int *post_shifts
= pre_shifts
+ nunits
;
451 int *shift_temps
= post_shifts
+ nunits
;
452 unsigned HOST_WIDE_INT
*mulc
= XALLOCAVEC (unsigned HOST_WIDE_INT
, nunits
);
453 int prec
= TYPE_PRECISION (TREE_TYPE (type
));
455 unsigned int i
, unsignedp
= TYPE_UNSIGNED (TREE_TYPE (type
));
456 unsigned HOST_WIDE_INT mask
= GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type
)));
458 tree cur_op
, mulcst
, tem
;
461 if (prec
> HOST_BITS_PER_WIDE_INT
)
464 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
465 if (op
== unknown_optab
466 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
467 has_vector_shift
= false;
469 /* Analysis phase. Determine if all op1 elements are either power
470 of two and it is possible to expand it using shifts (or for remainder
471 using masking). Additionally compute the multiplicative constants
472 and pre and post shifts if the division is to be expanded using
473 widening or high part multiplication plus shifts. */
474 for (i
= 0; i
< nunits
; i
++)
476 tree cst
= VECTOR_CST_ELT (op1
, i
);
477 unsigned HOST_WIDE_INT ml
;
479 if (!host_integerp (cst
, unsignedp
) || integer_zerop (cst
))
485 && (!integer_pow2p (cst
) || tree_int_cst_sgn (cst
) != 1))
489 shifts
[i
] = tree_log2 (cst
);
490 if (shifts
[i
] != shifts
[0]
491 && code
== TRUNC_DIV_EXPR
492 && !has_vector_shift
)
499 unsigned HOST_WIDE_INT mh
;
500 unsigned HOST_WIDE_INT d
= tree_low_cst (cst
, 1) & mask
;
502 if (d
>= ((unsigned HOST_WIDE_INT
) 1 << (prec
- 1)))
503 /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
512 /* Find a suitable multiplier and right shift count
513 instead of multiplying with D. */
514 mh
= choose_multiplier (d
, prec
, prec
, &ml
, &post_shift
, &dummy_int
);
516 /* If the suggested multiplier is more than SIZE bits, we can
517 do better for even divisors, using an initial right shift. */
518 if ((mh
!= 0 && (d
& 1) == 0)
519 || (!has_vector_shift
&& pre_shift
!= -1))
521 if (has_vector_shift
)
522 pre_shift
= floor_log2 (d
& -d
);
523 else if (pre_shift
== -1)
526 for (j
= 0; j
< nunits
; j
++)
528 tree cst2
= VECTOR_CST_ELT (op1
, j
);
529 unsigned HOST_WIDE_INT d2
;
532 if (!host_integerp (cst2
, 1))
534 d2
= tree_low_cst (cst2
, 1) & mask
;
537 this_pre_shift
= floor_log2 (d2
& -d2
);
538 if (pre_shift
== -1 || this_pre_shift
< pre_shift
)
539 pre_shift
= this_pre_shift
;
541 if (i
!= 0 && pre_shift
!= 0)
551 if ((d
>> pre_shift
) <= 1)
556 mh
= choose_multiplier (d
>> pre_shift
, prec
,
558 &ml
, &post_shift
, &dummy_int
);
560 pre_shifts
[i
] = pre_shift
;
570 HOST_WIDE_INT d
= tree_low_cst (cst
, 0);
571 unsigned HOST_WIDE_INT abs_d
;
576 /* Since d might be INT_MIN, we have to cast to
577 unsigned HOST_WIDE_INT before negating to avoid
578 undefined signed overflow. */
580 ? (unsigned HOST_WIDE_INT
) d
581 : - (unsigned HOST_WIDE_INT
) d
);
583 /* n rem d = n rem -d */
584 if (code
== TRUNC_MOD_EXPR
&& d
< 0)
586 else if (abs_d
== (unsigned HOST_WIDE_INT
) 1 << (prec
- 1))
588 /* This case is not handled correctly below. */
598 choose_multiplier (abs_d
, prec
, prec
- 1, &ml
,
599 &post_shift
, &dummy_int
);
600 if (ml
>= (unsigned HOST_WIDE_INT
) 1 << (prec
- 1))
602 this_mode
= 4 + (d
< 0);
603 ml
|= (~(unsigned HOST_WIDE_INT
) 0) << (prec
- 1);
606 this_mode
= 2 + (d
< 0);
609 post_shifts
[i
] = post_shift
;
610 if ((i
&& !has_vector_shift
&& post_shifts
[0] != post_shift
)
611 || post_shift
>= prec
612 || pre_shifts
[i
] >= prec
)
617 else if (mode
!= this_mode
)
621 vec
= XALLOCAVEC (tree
, nunits
);
625 tree addend
= NULL_TREE
;
630 /* Both division and remainder sequences need
631 op0 < 0 ? mask : 0 computed. It can be either computed as
632 (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
633 if none of the shifts is 0, or as the conditional. */
634 for (i
= 0; i
< nunits
; i
++)
638 = build_vector_type (build_nonstandard_integer_type (prec
, 1),
640 if (i
== nunits
&& TYPE_MODE (uns_type
) == TYPE_MODE (type
))
642 for (i
= 0; i
< nunits
; i
++)
643 shift_temps
[i
] = prec
- 1;
644 cur_op
= add_rshift (gsi
, type
, op0
, shift_temps
);
645 if (cur_op
!= NULL_TREE
)
647 cur_op
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
649 for (i
= 0; i
< nunits
; i
++)
650 shift_temps
[i
] = prec
- shifts
[i
];
651 cur_op
= add_rshift (gsi
, uns_type
, cur_op
, shift_temps
);
652 if (cur_op
!= NULL_TREE
)
653 addend
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
657 if (addend
== NULL_TREE
658 && expand_vec_cond_expr_p (type
, type
))
660 tree zero
, cst
, cond
;
663 zero
= build_zero_cst (type
);
664 cond
= build2 (LT_EXPR
, type
, op0
, zero
);
665 for (i
= 0; i
< nunits
; i
++)
666 vec
[i
] = build_int_cst (TREE_TYPE (type
),
667 ((unsigned HOST_WIDE_INT
) 1
669 cst
= build_vector (type
, vec
);
670 addend
= create_tmp_reg (type
, NULL
);
671 add_referenced_var (addend
);
672 addend
= make_ssa_name (addend
, NULL
);
673 stmt
= gimple_build_assign_with_ops3 (VEC_COND_EXPR
, addend
,
675 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
678 if (code
== TRUNC_DIV_EXPR
)
682 /* q = op0 >> shift; */
683 cur_op
= add_rshift (gsi
, type
, op0
, shifts
);
684 if (cur_op
!= NULL_TREE
)
687 else if (addend
!= NULL_TREE
)
689 /* t1 = op0 + addend;
691 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
692 if (op
!= unknown_optab
693 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
695 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
, addend
);
696 cur_op
= add_rshift (gsi
, type
, cur_op
, shifts
);
697 if (cur_op
!= NULL_TREE
)
705 for (i
= 0; i
< nunits
; i
++)
706 vec
[i
] = build_int_cst (TREE_TYPE (type
),
707 ((unsigned HOST_WIDE_INT
) 1
709 mask
= build_vector (type
, vec
);
710 op
= optab_for_tree_code (BIT_AND_EXPR
, type
, optab_default
);
711 if (op
!= unknown_optab
712 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
715 /* r = op0 & mask; */
716 return gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, op0
, mask
);
717 else if (addend
!= NULL_TREE
)
719 /* t1 = op0 + addend;
722 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
723 if (op
!= unknown_optab
724 && optab_handler (op
, TYPE_MODE (type
))
727 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
,
729 cur_op
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
,
731 op
= optab_for_tree_code (MINUS_EXPR
, type
,
733 if (op
!= unknown_optab
734 && optab_handler (op
, TYPE_MODE (type
))
736 return gimplify_build2 (gsi
, MINUS_EXPR
, type
,
744 if (mode
== -2 || BYTES_BIG_ENDIAN
!= WORDS_BIG_ENDIAN
)
747 if (!can_mult_highpart_p (TYPE_MODE (type
), TYPE_UNSIGNED (type
)))
755 gcc_assert (unsignedp
);
756 /* t1 = oprnd0 >> pre_shift;
758 q = t2 >> post_shift; */
759 cur_op
= add_rshift (gsi
, type
, cur_op
, pre_shifts
);
760 if (cur_op
== NULL_TREE
)
764 gcc_assert (unsignedp
);
765 for (i
= 0; i
< nunits
; i
++)
775 gcc_assert (!unsignedp
);
776 for (i
= 0; i
< nunits
; i
++)
777 shift_temps
[i
] = prec
- 1;
783 for (i
= 0; i
< nunits
; i
++)
784 vec
[i
] = build_int_cst (TREE_TYPE (type
), mulc
[i
]);
785 mulcst
= build_vector (type
, vec
);
787 cur_op
= gimplify_build2 (gsi
, MULT_HIGHPART_EXPR
, type
, cur_op
, mulcst
);
792 /* t1 = oprnd0 >> pre_shift;
794 q = t2 >> post_shift; */
795 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
798 /* t1 = oprnd0 h* ml;
802 q = t4 >> (post_shift - 1); */
803 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
804 if (op
== unknown_optab
805 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
807 tem
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, cur_op
);
808 tem
= add_rshift (gsi
, type
, tem
, shift_temps
);
809 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
810 if (op
== unknown_optab
811 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
813 tem
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, tem
);
814 cur_op
= add_rshift (gsi
, type
, tem
, post_shifts
);
815 if (cur_op
== NULL_TREE
)
822 /* t1 = oprnd0 h* ml;
823 t2 = t1; [ iff (mode & 2) != 0 ]
824 t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
825 t3 = t2 >> post_shift;
826 t4 = oprnd0 >> (prec - 1);
827 q = t3 - t4; [ iff (mode & 1) == 0 ]
828 q = t4 - t3; [ iff (mode & 1) != 0 ] */
831 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
832 if (op
== unknown_optab
833 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
835 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, op0
);
837 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
838 if (cur_op
== NULL_TREE
)
840 tem
= add_rshift (gsi
, type
, op0
, shift_temps
);
841 if (tem
== NULL_TREE
)
843 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
844 if (op
== unknown_optab
845 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
848 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, cur_op
, tem
);
850 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, tem
, cur_op
);
856 if (code
== TRUNC_DIV_EXPR
)
859 /* We divided. Now finish by:
862 op
= optab_for_tree_code (MULT_EXPR
, type
, optab_default
);
863 if (op
== unknown_optab
864 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
866 tem
= gimplify_build2 (gsi
, MULT_EXPR
, type
, cur_op
, op1
);
867 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
868 if (op
== unknown_optab
869 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
871 return gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, tem
);
875 expand_vector_operation (gimple_stmt_iterator
*gsi
, tree type
, tree compute_type
,
876 gimple assign
, enum tree_code code
)
878 enum machine_mode compute_mode
= TYPE_MODE (compute_type
);
880 /* If the compute mode is not a vector mode (hence we are not decomposing
881 a BLKmode vector to smaller, hardware-supported vectors), we may want
882 to expand the operations in parallel. */
883 if (GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_INT
884 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_FLOAT
885 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_FRACT
886 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_UFRACT
887 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_ACCUM
888 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_UACCUM
)
893 if (!TYPE_OVERFLOW_TRAPS (type
))
894 return expand_vector_addition (gsi
, do_binop
, do_plus_minus
, type
,
895 gimple_assign_rhs1 (assign
),
896 gimple_assign_rhs2 (assign
), code
);
900 if (!TYPE_OVERFLOW_TRAPS (type
))
901 return expand_vector_addition (gsi
, do_unop
, do_negate
, type
,
902 gimple_assign_rhs1 (assign
),
909 return expand_vector_parallel (gsi
, do_binop
, type
,
910 gimple_assign_rhs1 (assign
),
911 gimple_assign_rhs2 (assign
), code
);
914 return expand_vector_parallel (gsi
, do_unop
, type
,
915 gimple_assign_rhs1 (assign
),
932 tree rhs1
= gimple_assign_rhs1 (assign
);
933 tree rhs2
= gimple_assign_rhs2 (assign
);
935 return expand_vector_comparison (gsi
, type
, rhs1
, rhs2
, code
);
941 tree rhs1
= gimple_assign_rhs1 (assign
);
942 tree rhs2
= gimple_assign_rhs2 (assign
);
946 || !VECTOR_INTEGER_TYPE_P (type
)
947 || TREE_CODE (rhs2
) != VECTOR_CST
)
950 ret
= expand_vector_divmod (gsi
, type
, rhs1
, rhs2
, code
);
951 if (ret
!= NULL_TREE
)
960 if (TREE_CODE_CLASS (code
) == tcc_unary
)
961 return expand_vector_piecewise (gsi
, do_unop
, type
, compute_type
,
962 gimple_assign_rhs1 (assign
),
965 return expand_vector_piecewise (gsi
, do_binop
, type
, compute_type
,
966 gimple_assign_rhs1 (assign
),
967 gimple_assign_rhs2 (assign
), code
);
970 /* Return a type for the widest vector mode whose components are of type
971 TYPE, or NULL_TREE if none is found. */
974 type_for_widest_vector_mode (tree type
, optab op
)
976 enum machine_mode inner_mode
= TYPE_MODE (type
);
977 enum machine_mode best_mode
= VOIDmode
, mode
;
980 if (SCALAR_FLOAT_MODE_P (inner_mode
))
981 mode
= MIN_MODE_VECTOR_FLOAT
;
982 else if (SCALAR_FRACT_MODE_P (inner_mode
))
983 mode
= MIN_MODE_VECTOR_FRACT
;
984 else if (SCALAR_UFRACT_MODE_P (inner_mode
))
985 mode
= MIN_MODE_VECTOR_UFRACT
;
986 else if (SCALAR_ACCUM_MODE_P (inner_mode
))
987 mode
= MIN_MODE_VECTOR_ACCUM
;
988 else if (SCALAR_UACCUM_MODE_P (inner_mode
))
989 mode
= MIN_MODE_VECTOR_UACCUM
;
991 mode
= MIN_MODE_VECTOR_INT
;
993 for (; mode
!= VOIDmode
; mode
= GET_MODE_WIDER_MODE (mode
))
994 if (GET_MODE_INNER (mode
) == inner_mode
995 && GET_MODE_NUNITS (mode
) > best_nunits
996 && optab_handler (op
, mode
) != CODE_FOR_nothing
)
997 best_mode
= mode
, best_nunits
= GET_MODE_NUNITS (mode
);
999 if (best_mode
== VOIDmode
)
1002 return build_vector_type_for_mode (type
, best_mode
);
1006 /* Build a reference to the element of the vector VECT. Function
1007 returns either the element itself, either BIT_FIELD_REF, or an
1008 ARRAY_REF expression.
1010 GSI is required to insert temporary variables while building a
1011 refernece to the element of the vector VECT.
1013 PTMPVEC is a pointer to the temporary variable for caching
1014 purposes. In case when PTMPVEC is NULL new temporary variable
1017 vector_element (gimple_stmt_iterator
*gsi
, tree vect
, tree idx
, tree
*ptmpvec
)
1019 tree vect_type
, vect_elt_type
;
1023 bool need_asgn
= true;
1024 unsigned int elements
;
1026 vect_type
= TREE_TYPE (vect
);
1027 vect_elt_type
= TREE_TYPE (vect_type
);
1028 elements
= TYPE_VECTOR_SUBPARTS (vect_type
);
1030 if (TREE_CODE (idx
) == INTEGER_CST
)
1032 unsigned HOST_WIDE_INT index
;
1034 /* Given that we're about to compute a binary modulus,
1035 we don't care about the high bits of the value. */
1036 index
= TREE_INT_CST_LOW (idx
);
1037 if (!host_integerp (idx
, 1) || index
>= elements
)
1039 index
&= elements
- 1;
1040 idx
= build_int_cst (TREE_TYPE (idx
), index
);
1043 /* When lowering a vector statement sequence do some easy
1044 simplification by looking through intermediate vector results. */
1045 if (TREE_CODE (vect
) == SSA_NAME
)
1047 gimple def_stmt
= SSA_NAME_DEF_STMT (vect
);
1048 if (is_gimple_assign (def_stmt
)
1049 && (gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
1050 || gimple_assign_rhs_code (def_stmt
) == CONSTRUCTOR
))
1051 vect
= gimple_assign_rhs1 (def_stmt
);
1054 if (TREE_CODE (vect
) == VECTOR_CST
)
1055 return VECTOR_CST_ELT (vect
, index
);
1056 else if (TREE_CODE (vect
) == CONSTRUCTOR
)
1061 FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (vect
), i
, elt_i
, elt_v
)
1062 if (operand_equal_p (elt_i
, idx
, 0))
1064 return build_zero_cst (vect_elt_type
);
1068 tree size
= TYPE_SIZE (vect_elt_type
);
1069 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
, bitsize_int (index
),
1071 return fold_build3 (BIT_FIELD_REF
, vect_elt_type
, vect
, size
, pos
);
1076 tmpvec
= create_tmp_var (vect_type
, "vectmp");
1078 tmpvec
= *ptmpvec
= create_tmp_var (vect_type
, "vectmp");
1087 TREE_ADDRESSABLE (tmpvec
) = 1;
1088 asgn
= gimple_build_assign (tmpvec
, vect
);
1089 gsi_insert_before (gsi
, asgn
, GSI_SAME_STMT
);
1092 arraytype
= build_array_type_nelts (vect_elt_type
, elements
);
1093 return build4 (ARRAY_REF
, vect_elt_type
,
1094 build1 (VIEW_CONVERT_EXPR
, arraytype
, tmpvec
),
1095 idx
, NULL_TREE
, NULL_TREE
);
1098 /* Check if VEC_PERM_EXPR within the given setting is supported
1099 by hardware, or lower it piecewise.
1101 When VEC_PERM_EXPR has the same first and second operands:
1102 VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1103 {v0[mask[0]], v0[mask[1]], ...}
1104 MASK and V0 must have the same number of elements.
1106 Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1107 {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1108 V0 and V1 must have the same type. MASK, V0, V1 must have the
1109 same number of arguments. */
1112 lower_vec_perm (gimple_stmt_iterator
*gsi
)
1114 gimple stmt
= gsi_stmt (*gsi
);
1115 tree mask
= gimple_assign_rhs3 (stmt
);
1116 tree vec0
= gimple_assign_rhs1 (stmt
);
1117 tree vec1
= gimple_assign_rhs2 (stmt
);
1118 tree vect_type
= TREE_TYPE (vec0
);
1119 tree mask_type
= TREE_TYPE (mask
);
1120 tree vect_elt_type
= TREE_TYPE (vect_type
);
1121 tree mask_elt_type
= TREE_TYPE (mask_type
);
1122 unsigned int elements
= TYPE_VECTOR_SUBPARTS (vect_type
);
1123 VEC(constructor_elt
,gc
) *v
;
1124 tree constr
, t
, si
, i_val
;
1125 tree vec0tmp
= NULL_TREE
, vec1tmp
= NULL_TREE
, masktmp
= NULL_TREE
;
1126 bool two_operand_p
= !operand_equal_p (vec0
, vec1
, 0);
1127 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1130 if (TREE_CODE (mask
) == SSA_NAME
)
1132 gimple def_stmt
= SSA_NAME_DEF_STMT (mask
);
1133 if (is_gimple_assign (def_stmt
)
1134 && gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
)
1135 mask
= gimple_assign_rhs1 (def_stmt
);
1138 if (TREE_CODE (mask
) == VECTOR_CST
)
1140 unsigned char *sel_int
= XALLOCAVEC (unsigned char, elements
);
1142 for (i
= 0; i
< elements
; ++i
)
1143 sel_int
[i
] = (TREE_INT_CST_LOW (VECTOR_CST_ELT (mask
, i
))
1144 & (2 * elements
- 1));
1146 if (can_vec_perm_p (TYPE_MODE (vect_type
), false, sel_int
))
1148 gimple_assign_set_rhs3 (stmt
, mask
);
1153 else if (can_vec_perm_p (TYPE_MODE (vect_type
), true, NULL
))
1156 warning_at (loc
, OPT_Wvector_operation_performance
,
1157 "vector shuffling operation will be expanded piecewise");
1159 v
= VEC_alloc (constructor_elt
, gc
, elements
);
1160 for (i
= 0; i
< elements
; i
++)
1163 i_val
= vector_element (gsi
, mask
, si
, &masktmp
);
1165 if (TREE_CODE (i_val
) == INTEGER_CST
)
1167 unsigned HOST_WIDE_INT index
;
1169 index
= TREE_INT_CST_LOW (i_val
);
1170 if (!host_integerp (i_val
, 1) || index
>= elements
)
1171 i_val
= build_int_cst (mask_elt_type
, index
& (elements
- 1));
1173 if (two_operand_p
&& (index
& elements
) != 0)
1174 t
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1176 t
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1178 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
1179 true, GSI_SAME_STMT
);
1183 tree cond
= NULL_TREE
, v0_val
;
1187 cond
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1188 build_int_cst (mask_elt_type
, elements
));
1189 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1190 true, GSI_SAME_STMT
);
1193 i_val
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1194 build_int_cst (mask_elt_type
, elements
- 1));
1195 i_val
= force_gimple_operand_gsi (gsi
, i_val
, true, NULL_TREE
,
1196 true, GSI_SAME_STMT
);
1198 v0_val
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1199 v0_val
= force_gimple_operand_gsi (gsi
, v0_val
, true, NULL_TREE
,
1200 true, GSI_SAME_STMT
);
1206 v1_val
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1207 v1_val
= force_gimple_operand_gsi (gsi
, v1_val
, true, NULL_TREE
,
1208 true, GSI_SAME_STMT
);
1210 cond
= fold_build2 (EQ_EXPR
, boolean_type_node
,
1211 cond
, build_zero_cst (mask_elt_type
));
1212 cond
= fold_build3 (COND_EXPR
, vect_elt_type
,
1213 cond
, v0_val
, v1_val
);
1214 t
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1215 true, GSI_SAME_STMT
);
1221 CONSTRUCTOR_APPEND_ELT (v
, si
, t
);
1224 constr
= build_constructor (vect_type
, v
);
1225 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1226 update_stmt (gsi_stmt (*gsi
));
1229 /* Process one statement. If we identify a vector operation, expand it. */
1232 expand_vector_operations_1 (gimple_stmt_iterator
*gsi
)
1234 gimple stmt
= gsi_stmt (*gsi
);
1235 tree lhs
, rhs1
, rhs2
= NULL
, type
, compute_type
;
1236 enum tree_code code
;
1237 enum machine_mode compute_mode
;
1238 optab op
= unknown_optab
;
1239 enum gimple_rhs_class rhs_class
;
1242 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
1245 code
= gimple_assign_rhs_code (stmt
);
1246 rhs_class
= get_gimple_rhs_class (code
);
1247 lhs
= gimple_assign_lhs (stmt
);
1249 if (code
== VEC_PERM_EXPR
)
1251 lower_vec_perm (gsi
);
1255 if (rhs_class
!= GIMPLE_UNARY_RHS
&& rhs_class
!= GIMPLE_BINARY_RHS
)
1258 rhs1
= gimple_assign_rhs1 (stmt
);
1259 type
= gimple_expr_type (stmt
);
1260 if (rhs_class
== GIMPLE_BINARY_RHS
)
1261 rhs2
= gimple_assign_rhs2 (stmt
);
1263 if (TREE_CODE (type
) != VECTOR_TYPE
)
1266 if (code
== NOP_EXPR
1267 || code
== FLOAT_EXPR
1268 || code
== FIX_TRUNC_EXPR
1269 || code
== VIEW_CONVERT_EXPR
)
1272 gcc_assert (code
!= CONVERT_EXPR
);
1274 /* The signedness is determined from input argument. */
1275 if (code
== VEC_UNPACK_FLOAT_HI_EXPR
1276 || code
== VEC_UNPACK_FLOAT_LO_EXPR
)
1277 type
= TREE_TYPE (rhs1
);
1279 /* For widening/narrowing vector operations, the relevant type is of the
1280 arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
1281 calculated in the same way above. */
1282 if (code
== WIDEN_SUM_EXPR
1283 || code
== VEC_WIDEN_MULT_HI_EXPR
1284 || code
== VEC_WIDEN_MULT_LO_EXPR
1285 || code
== VEC_WIDEN_MULT_EVEN_EXPR
1286 || code
== VEC_WIDEN_MULT_ODD_EXPR
1287 || code
== VEC_UNPACK_HI_EXPR
1288 || code
== VEC_UNPACK_LO_EXPR
1289 || code
== VEC_PACK_TRUNC_EXPR
1290 || code
== VEC_PACK_SAT_EXPR
1291 || code
== VEC_PACK_FIX_TRUNC_EXPR
1292 || code
== VEC_WIDEN_LSHIFT_HI_EXPR
1293 || code
== VEC_WIDEN_LSHIFT_LO_EXPR
)
1294 type
= TREE_TYPE (rhs1
);
1296 /* Choose between vector shift/rotate by vector and vector shift/rotate by
1298 if (code
== LSHIFT_EXPR
1299 || code
== RSHIFT_EXPR
1300 || code
== LROTATE_EXPR
1301 || code
== RROTATE_EXPR
)
1305 /* Check whether we have vector <op> {x,x,x,x} where x
1306 could be a scalar variable or a constant. Transform
1307 vector <op> {x,x,x,x} ==> vector <op> scalar. */
1308 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
1313 if ((TREE_CODE (rhs2
) == VECTOR_CST
1314 && (first
= uniform_vector_p (rhs2
)) != NULL_TREE
)
1315 || (TREE_CODE (rhs2
) == SSA_NAME
1316 && (def_stmt
= SSA_NAME_DEF_STMT (rhs2
))
1317 && gimple_assign_single_p (def_stmt
)
1318 && (first
= uniform_vector_p
1319 (gimple_assign_rhs1 (def_stmt
))) != NULL_TREE
))
1321 gimple_assign_set_rhs2 (stmt
, first
);
1327 opv
= optab_for_tree_code (code
, type
, optab_vector
);
1328 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
1332 op
= optab_for_tree_code (code
, type
, optab_scalar
);
1334 /* The rtl expander will expand vector/scalar as vector/vector
1335 if necessary. Don't bother converting the stmt here. */
1336 if (optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
1337 && optab_handler (opv
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
1342 op
= optab_for_tree_code (code
, type
, optab_default
);
1344 /* Optabs will try converting a negation into a subtraction, so
1345 look for it as well. TODO: negation of floating-point vectors
1346 might be turned into an exclusive OR toggling the sign bit. */
1347 if (op
== unknown_optab
1348 && code
== NEGATE_EXPR
1349 && INTEGRAL_TYPE_P (TREE_TYPE (type
)))
1350 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
1352 /* For very wide vectors, try using a smaller vector mode. */
1353 compute_type
= type
;
1354 if (!VECTOR_MODE_P (TYPE_MODE (type
)) && op
)
1356 tree vector_compute_type
1357 = type_for_widest_vector_mode (TREE_TYPE (type
), op
);
1358 if (vector_compute_type
!= NULL_TREE
1359 && (TYPE_VECTOR_SUBPARTS (vector_compute_type
)
1360 < TYPE_VECTOR_SUBPARTS (compute_type
))
1361 && (optab_handler (op
, TYPE_MODE (vector_compute_type
))
1362 != CODE_FOR_nothing
))
1363 compute_type
= vector_compute_type
;
1366 /* If we are breaking a BLKmode vector into smaller pieces,
1367 type_for_widest_vector_mode has already looked into the optab,
1368 so skip these checks. */
1369 if (compute_type
== type
)
1371 compute_mode
= TYPE_MODE (compute_type
);
1372 if (VECTOR_MODE_P (compute_mode
))
1374 if (op
&& optab_handler (op
, compute_mode
) != CODE_FOR_nothing
)
1376 if (code
== MULT_HIGHPART_EXPR
1377 && can_mult_highpart_p (compute_mode
,
1378 TYPE_UNSIGNED (compute_type
)))
1381 /* There is no operation in hardware, so fall back to scalars. */
1382 compute_type
= TREE_TYPE (type
);
1385 gcc_assert (code
!= VEC_LSHIFT_EXPR
&& code
!= VEC_RSHIFT_EXPR
);
1386 new_rhs
= expand_vector_operation (gsi
, type
, compute_type
, stmt
, code
);
1388 /* Leave expression untouched for later expansion. */
1389 if (new_rhs
== NULL_TREE
)
1392 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
1393 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
1396 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1397 way to do it is change expand_vector_operation and its callees to
1398 return a tree_code, RHS1 and RHS2 instead of a tree. */
1399 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
1400 update_stmt (gsi_stmt (*gsi
));
1403 /* Use this to lower vector operations introduced by the vectorizer,
1404 if it may need the bit-twiddling tricks implemented in this file. */
1407 gate_expand_vector_operations_ssa (void)
1409 return optimize
== 0;
1413 expand_vector_operations (void)
1415 gimple_stmt_iterator gsi
;
1417 bool cfg_changed
= false;
1421 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1423 expand_vector_operations_1 (&gsi
);
1424 /* ??? If we do not cleanup EH then we will ICE in
1425 verification. But in reality we have created wrong-code
1426 as we did not properly transition EH info and edges to
1427 the piecewise computations. */
1428 if (maybe_clean_eh_stmt (gsi_stmt (gsi
))
1429 && gimple_purge_dead_eh_edges (bb
))
1434 return cfg_changed
? TODO_cleanup_cfg
: 0;
1437 struct gimple_opt_pass pass_lower_vector
=
1441 "veclower", /* name */
1442 gate_expand_vector_operations_ssa
, /* gate */
1443 expand_vector_operations
, /* execute */
1446 0, /* static_pass_number */
1447 TV_NONE
, /* tv_id */
1448 PROP_cfg
, /* properties_required */
1449 0, /* properties_provided */
1450 0, /* properties_destroyed */
1451 0, /* todo_flags_start */
1452 TODO_update_ssa
/* todo_flags_finish */
1454 | TODO_verify_stmts
| TODO_verify_flow
1459 struct gimple_opt_pass pass_lower_vector_ssa
=
1463 "veclower2", /* name */
1465 expand_vector_operations
, /* execute */
1468 0, /* static_pass_number */
1469 TV_NONE
, /* tv_id */
1470 PROP_cfg
, /* properties_required */
1471 0, /* properties_provided */
1472 0, /* properties_destroyed */
1473 0, /* todo_flags_start */
1474 TODO_update_ssa
/* todo_flags_finish */
1476 | TODO_verify_stmts
| TODO_verify_flow
1481 #include "gt-tree-vect-generic.h"