1 /* Lower vector operations to scalar operations.
2 Copyright (C) 2004-2023 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 GCC is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
27 #include "tree-pass.h"
30 #include "optabs-tree.h"
31 #include "diagnostic.h"
32 #include "fold-const.h"
33 #include "stor-layout.h"
34 #include "langhooks.h"
36 #include "gimple-iterator.h"
37 #include "gimplify-me.h"
40 #include "tree-vector-builder.h"
41 #include "vec-perm-indices.h"
42 #include "insn-config.h"
43 #include "tree-ssa-dce.h"
44 #include "gimple-fold.h"
45 #include "gimple-match.h"
46 #include "recog.h" /* FIXME: for insn_data */
49 /* Build a ternary operation and gimplify it. Emit code before GSI.
50 Return the gimple_val holding the result. */
53 gimplify_build3 (gimple_stmt_iterator
*gsi
, enum tree_code code
,
54 tree type
, tree a
, tree b
, tree c
)
56 location_t loc
= gimple_location (gsi_stmt (*gsi
));
57 return gimple_build (gsi
, true, GSI_SAME_STMT
, loc
, code
, type
, a
, b
, c
);
60 /* Build a binary operation and gimplify it. Emit code before GSI.
61 Return the gimple_val holding the result. */
64 gimplify_build2 (gimple_stmt_iterator
*gsi
, enum tree_code code
,
65 tree type
, tree a
, tree b
)
67 location_t loc
= gimple_location (gsi_stmt (*gsi
));
68 return gimple_build (gsi
, true, GSI_SAME_STMT
, loc
, code
, type
, a
, b
);
71 /* Build a unary operation and gimplify it. Emit code before GSI.
72 Return the gimple_val holding the result. */
75 gimplify_build1 (gimple_stmt_iterator
*gsi
, enum tree_code code
, tree type
,
78 location_t loc
= gimple_location (gsi_stmt (*gsi
));
79 return gimple_build (gsi
, true, GSI_SAME_STMT
, loc
, code
, type
, a
);
83 static void expand_vector_operations_1 (gimple_stmt_iterator
*, bitmap
);
85 /* Return the number of elements in a vector type TYPE that we have
86 already decided needs to be expanded piecewise. We don't support
87 this kind of expansion for variable-length vectors, since we should
88 always check for target support before introducing uses of those. */
90 nunits_for_known_piecewise_op (const_tree type
)
92 return TYPE_VECTOR_SUBPARTS (type
).to_constant ();
95 /* Return true if TYPE1 has more elements than TYPE2, where either
96 type may be a vector or a scalar. */
99 subparts_gt (tree type1
, tree type2
)
101 poly_uint64 n1
= VECTOR_TYPE_P (type1
) ? TYPE_VECTOR_SUBPARTS (type1
) : 1;
102 poly_uint64 n2
= VECTOR_TYPE_P (type2
) ? TYPE_VECTOR_SUBPARTS (type2
) : 1;
103 return known_gt (n1
, n2
);
106 static GTY(()) tree vector_inner_type
;
107 static GTY(()) tree vector_last_type
;
108 static GTY(()) int vector_last_nunits
;
110 /* Return a suitable vector types made of SUBPARTS units each of mode
111 "word_mode" (the global variable). */
113 build_word_mode_vector_type (int nunits
)
115 if (!vector_inner_type
)
116 vector_inner_type
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
117 else if (vector_last_nunits
== nunits
)
119 gcc_assert (TREE_CODE (vector_last_type
) == VECTOR_TYPE
);
120 return vector_last_type
;
123 vector_last_nunits
= nunits
;
124 vector_last_type
= build_vector_type (vector_inner_type
, nunits
);
125 return vector_last_type
;
128 typedef tree (*elem_op_func
) (gimple_stmt_iterator
*,
129 tree
, tree
, tree
, tree
, tree
, enum tree_code
,
132 /* Extract the vector element of type TYPE at BITPOS with BITSIZE from T
136 tree_vec_extract (gimple_stmt_iterator
*gsi
, tree type
,
137 tree t
, tree bitsize
, tree bitpos
)
139 /* We're using the resimplify API and maybe_push_res_to_seq to
140 simplify the BIT_FIELD_REF but restrict the simplification to
141 a single stmt while at the same time following SSA edges for
142 simplification with already emitted CTORs. */
144 opr
.set_op (BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
145 opr
.resimplify (NULL
, follow_all_ssa_edges
);
146 gimple_seq stmts
= NULL
;
147 tree res
= maybe_push_res_to_seq (&opr
, &stmts
);
150 /* This can happen if SSA_NAME_OCCURS_IN_ABNORMAL_PHI are
151 used. Build BIT_FIELD_REF manually otherwise. */
152 t
= build3 (BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
153 res
= make_ssa_name (type
);
154 gimple
*g
= gimple_build_assign (res
, t
);
155 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
158 gsi_insert_seq_before (gsi
, stmts
, GSI_SAME_STMT
);
163 do_unop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
164 tree b ATTRIBUTE_UNUSED
, tree bitpos
, tree bitsize
,
165 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
167 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
168 return gimplify_build1 (gsi
, code
, inner_type
, a
);
172 do_binop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
173 tree bitpos
, tree bitsize
, enum tree_code code
,
174 tree type ATTRIBUTE_UNUSED
)
176 if (TREE_CODE (TREE_TYPE (a
)) == VECTOR_TYPE
)
177 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
178 if (TREE_CODE (TREE_TYPE (b
)) == VECTOR_TYPE
)
179 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
180 return gimplify_build2 (gsi
, code
, inner_type
, a
, b
);
183 /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
185 INNER_TYPE is the type of A and B elements
187 returned expression is of signed integer type with the
188 size equal to the size of INNER_TYPE. */
190 do_compare (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
191 tree bitpos
, tree bitsize
, enum tree_code code
, tree type
)
193 tree stype
= TREE_TYPE (type
);
194 tree cst_false
= build_zero_cst (stype
);
195 tree cst_true
= build_all_ones_cst (stype
);
198 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
199 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
201 cmp
= build2 (code
, boolean_type_node
, a
, b
);
202 return gimplify_build3 (gsi
, COND_EXPR
, stype
, cmp
, cst_true
, cst_false
);
205 /* Expand vector addition to scalars. This does bit twiddling
206 in order to increase parallelism:
208 a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
211 a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
212 (a ^ ~b) & 0x80808080
214 -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
216 This optimization should be done only if 4 vector items or more
219 do_plus_minus (gimple_stmt_iterator
*gsi
, tree word_type
, tree a
, tree b
,
220 tree bitpos ATTRIBUTE_UNUSED
, tree bitsize ATTRIBUTE_UNUSED
,
221 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
223 unsigned int width
= vector_element_bits (TREE_TYPE (a
));
224 tree inner_type
= TREE_TYPE (TREE_TYPE (a
));
225 unsigned HOST_WIDE_INT max
;
226 tree low_bits
, high_bits
, a_low
, b_low
, result_low
, signs
;
228 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
229 low_bits
= build_replicated_int_cst (word_type
, width
, max
>> 1);
230 high_bits
= build_replicated_int_cst (word_type
, width
, max
& ~(max
>> 1));
232 a
= tree_vec_extract (gsi
, word_type
, a
, bitsize
, bitpos
);
233 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
235 signs
= gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, a
, b
);
236 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
237 if (code
== PLUS_EXPR
)
238 a_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, a
, low_bits
);
241 a_low
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, word_type
, a
, high_bits
);
242 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, signs
);
245 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
246 result_low
= gimplify_build2 (gsi
, code
, word_type
, a_low
, b_low
);
247 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
251 do_negate (gimple_stmt_iterator
*gsi
, tree word_type
, tree b
,
252 tree unused ATTRIBUTE_UNUSED
, tree bitpos ATTRIBUTE_UNUSED
,
253 tree bitsize ATTRIBUTE_UNUSED
,
254 enum tree_code code ATTRIBUTE_UNUSED
,
255 tree type ATTRIBUTE_UNUSED
)
257 unsigned int width
= vector_element_bits (TREE_TYPE (b
));
258 tree inner_type
= TREE_TYPE (TREE_TYPE (b
));
260 tree low_bits
, high_bits
, b_low
, result_low
, signs
;
262 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
263 low_bits
= build_replicated_int_cst (word_type
, width
, max
>> 1);
264 high_bits
= build_replicated_int_cst (word_type
, width
, max
& ~(max
>> 1));
266 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
268 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
269 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, b
);
270 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
271 result_low
= gimplify_build2 (gsi
, MINUS_EXPR
, word_type
, high_bits
, b_low
);
272 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
275 /* Expand a vector operation to scalars, by using many operations
276 whose type is the vector type's inner type. */
278 expand_vector_piecewise (gimple_stmt_iterator
*gsi
, elem_op_func f
,
279 tree type
, tree inner_type
,
280 tree a
, tree b
, enum tree_code code
,
281 bool parallel_p
, tree ret_type
= NULL_TREE
)
283 vec
<constructor_elt
, va_gc
> *v
;
284 tree part_width
= TYPE_SIZE (inner_type
);
285 tree index
= bitsize_int (0);
286 int nunits
= nunits_for_known_piecewise_op (type
);
287 int delta
= tree_to_uhwi (part_width
) / vector_element_bits (type
);
289 location_t loc
= gimple_location (gsi_stmt (*gsi
));
292 || warning_suppressed_p (gsi_stmt (*gsi
),
293 OPT_Wvector_operation_performance
))
294 /* Do not diagnose decomposing single element vectors or when
295 decomposing vectorizer produced operations. */
297 else if (ret_type
|| !parallel_p
)
298 warning_at (loc
, OPT_Wvector_operation_performance
,
299 "vector operation will be expanded piecewise");
301 warning_at (loc
, OPT_Wvector_operation_performance
,
302 "vector operation will be expanded in parallel");
306 vec_alloc (v
, (nunits
+ delta
- 1) / delta
);
307 bool constant_p
= true;
308 for (i
= 0; i
< nunits
;
309 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
311 tree result
= f (gsi
, inner_type
, a
, b
, index
, part_width
, code
,
313 if (!CONSTANT_CLASS_P (result
))
315 constructor_elt ce
= {NULL_TREE
, result
};
320 return build_vector_from_ctor (ret_type
, v
);
322 return build_constructor (ret_type
, v
);
325 /* Expand a vector operation to scalars with the freedom to use
326 a scalar integer type, or to use a different size for the items
327 in the vector type. */
329 expand_vector_parallel (gimple_stmt_iterator
*gsi
, elem_op_func f
, tree type
,
330 tree a
, tree b
, enum tree_code code
)
332 tree result
, compute_type
;
333 int n_words
= tree_to_uhwi (TYPE_SIZE_UNIT (type
)) / UNITS_PER_WORD
;
334 location_t loc
= gimple_location (gsi_stmt (*gsi
));
336 /* We have three strategies. If the type is already correct, just do
337 the operation an element at a time. Else, if the vector is wider than
338 one word, do it a word at a time; finally, if the vector is smaller
339 than one word, do it as a scalar. */
340 if (TYPE_MODE (TREE_TYPE (type
)) == word_mode
)
341 return expand_vector_piecewise (gsi
, f
,
342 type
, TREE_TYPE (type
),
344 else if (n_words
> 1)
346 tree word_type
= build_word_mode_vector_type (n_words
);
347 result
= expand_vector_piecewise (gsi
, f
,
348 word_type
, TREE_TYPE (word_type
),
350 result
= force_gimple_operand_gsi (gsi
, result
, true, NULL
, true,
355 /* Use a single scalar operation with a mode no wider than word_mode. */
356 if (!warning_suppressed_p (gsi_stmt (*gsi
),
357 OPT_Wvector_operation_performance
))
358 warning_at (loc
, OPT_Wvector_operation_performance
,
359 "vector operation will be expanded with a "
360 "single scalar operation");
362 = int_mode_for_size (tree_to_uhwi (TYPE_SIZE (type
)), 0).require ();
363 compute_type
= lang_hooks
.types
.type_for_mode (mode
, 1);
364 result
= f (gsi
, compute_type
, a
, b
, bitsize_zero_node
,
365 TYPE_SIZE (compute_type
), code
, type
);
371 /* Expand a vector operation to scalars; for integer types we can use
372 special bit twiddling tricks to do the sums a word at a time, using
373 function F_PARALLEL instead of F. These tricks are done only if
374 they can process at least four items, that is, only if the vector
375 holds at least four items and if a word can hold four items. */
377 expand_vector_addition (gimple_stmt_iterator
*gsi
,
378 elem_op_func f
, elem_op_func f_parallel
,
379 tree type
, tree a
, tree b
, enum tree_code code
)
381 int parts_per_word
= BITS_PER_WORD
/ vector_element_bits (type
);
383 if (INTEGRAL_TYPE_P (TREE_TYPE (type
))
384 && parts_per_word
>= 4
385 && nunits_for_known_piecewise_op (type
) >= 4)
386 return expand_vector_parallel (gsi
, f_parallel
,
389 return expand_vector_piecewise (gsi
, f
,
390 type
, TREE_TYPE (type
),
395 expand_vector_condition (gimple_stmt_iterator
*gsi
, bitmap dce_ssa_names
);
397 /* Try to expand vector comparison expression OP0 CODE OP1 by
398 querying optab if the following expression:
399 VEC_COND_EXPR< OP0 CODE OP1, {-1,...}, {0,...}>
402 expand_vector_comparison (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
403 tree op1
, enum tree_code code
,
404 bitmap dce_ssa_names
)
406 tree lhs
= gimple_assign_lhs (gsi_stmt (*gsi
));
408 imm_use_iterator iterator
;
409 bool vec_cond_expr_only
= true;
411 /* As seen in PR95830, we should not expand comparisons that are only
412 feeding a VEC_COND_EXPR statement. */
413 auto_vec
<gimple
*> uses
;
414 FOR_EACH_IMM_USE_FAST (use_p
, iterator
, lhs
)
416 gimple
*use
= USE_STMT (use_p
);
417 if (is_gimple_debug (use
))
419 if (is_gimple_assign (use
)
420 && gimple_assign_rhs_code (use
) == VEC_COND_EXPR
421 && gimple_assign_rhs1 (use
) == lhs
422 && gimple_assign_rhs2 (use
) != lhs
423 && gimple_assign_rhs3 (use
) != lhs
)
424 uses
.safe_push (use
);
426 vec_cond_expr_only
= false;
429 if (vec_cond_expr_only
)
430 for (gimple
*use
: uses
)
432 gimple_stmt_iterator it
= gsi_for_stmt (use
);
433 if (!expand_vector_condition (&it
, dce_ssa_names
))
435 vec_cond_expr_only
= false;
440 if (!uses
.is_empty () && vec_cond_expr_only
)
444 if (!expand_vec_cmp_expr_p (TREE_TYPE (op0
), type
, code
))
446 if (VECTOR_BOOLEAN_TYPE_P (type
)
447 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
448 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type
)),
449 TYPE_VECTOR_SUBPARTS (type
)
450 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
451 (TREE_TYPE (type
)))))
453 tree inner_type
= TREE_TYPE (TREE_TYPE (op0
));
454 tree part_width
= vector_element_bits_tree (TREE_TYPE (op0
));
455 tree index
= bitsize_int (0);
456 int nunits
= nunits_for_known_piecewise_op (TREE_TYPE (op0
));
457 int prec
= GET_MODE_PRECISION (SCALAR_TYPE_MODE (type
));
458 tree ret_type
= build_nonstandard_integer_type (prec
, 1);
459 tree ret_inner_type
= boolean_type_node
;
461 location_t loc
= gimple_location (gsi_stmt (*gsi
));
462 t
= build_zero_cst (ret_type
);
464 if (TYPE_PRECISION (ret_inner_type
) != 1)
465 ret_inner_type
= build_nonstandard_integer_type (1, 1);
466 if (!warning_suppressed_p (gsi_stmt (*gsi
),
467 OPT_Wvector_operation_performance
))
468 warning_at (loc
, OPT_Wvector_operation_performance
,
469 "vector operation will be expanded piecewise");
470 for (i
= 0; i
< nunits
;
471 i
++, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
473 tree a
= tree_vec_extract (gsi
, inner_type
, op0
, part_width
,
475 tree b
= tree_vec_extract (gsi
, inner_type
, op1
, part_width
,
477 tree result
= gimplify_build2 (gsi
, code
, ret_inner_type
, a
, b
);
478 t
= gimplify_build3 (gsi
, BIT_INSERT_EXPR
, ret_type
, t
, result
,
481 t
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, type
, t
);
484 t
= expand_vector_piecewise (gsi
, do_compare
, type
,
485 TREE_TYPE (TREE_TYPE (op0
)), op0
, op1
,
494 /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
495 of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
496 the result if successful, otherwise return NULL_TREE. */
498 add_rshift (gimple_stmt_iterator
*gsi
, tree type
, tree op0
, int *shiftcnts
)
501 unsigned int i
, nunits
= nunits_for_known_piecewise_op (type
);
502 bool scalar_shift
= true;
504 for (i
= 1; i
< nunits
; i
++)
506 if (shiftcnts
[i
] != shiftcnts
[0])
507 scalar_shift
= false;
510 if (scalar_shift
&& shiftcnts
[0] == 0)
515 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_scalar
);
516 if (op
!= unknown_optab
517 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
518 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
519 build_int_cst (NULL_TREE
, shiftcnts
[0]));
522 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
523 if (op
!= unknown_optab
524 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
526 tree_vector_builder
vec (type
, nunits
, 1);
527 for (i
= 0; i
< nunits
; i
++)
528 vec
.quick_push (build_int_cst (TREE_TYPE (type
), shiftcnts
[i
]));
529 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
, vec
.build ());
535 /* Try to expand integer vector division by constant using
536 widening multiply, shifts and additions. */
538 expand_vector_divmod (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
539 tree op1
, enum tree_code code
)
541 bool use_pow2
= true;
542 bool has_vector_shift
= true;
543 bool use_abs_op1
= false;
544 int mode
= -1, this_mode
;
545 int pre_shift
= -1, post_shift
;
546 unsigned int nunits
= nunits_for_known_piecewise_op (type
);
547 int *shifts
= XALLOCAVEC (int, nunits
* 4);
548 int *pre_shifts
= shifts
+ nunits
;
549 int *post_shifts
= pre_shifts
+ nunits
;
550 int *shift_temps
= post_shifts
+ nunits
;
551 unsigned HOST_WIDE_INT
*mulc
= XALLOCAVEC (unsigned HOST_WIDE_INT
, nunits
);
552 int prec
= TYPE_PRECISION (TREE_TYPE (type
));
555 signop sign_p
= TYPE_SIGN (TREE_TYPE (type
));
556 unsigned HOST_WIDE_INT mask
= GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type
)));
557 tree cur_op
, mulcst
, tem
;
560 if (prec
> HOST_BITS_PER_WIDE_INT
)
563 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
564 if (op
== unknown_optab
565 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
566 has_vector_shift
= false;
568 /* Analysis phase. Determine if all op1 elements are either power
569 of two and it is possible to expand it using shifts (or for remainder
570 using masking). Additionally compute the multiplicative constants
571 and pre and post shifts if the division is to be expanded using
572 widening or high part multiplication plus shifts. */
573 for (i
= 0; i
< nunits
; i
++)
575 tree cst
= VECTOR_CST_ELT (op1
, i
);
576 unsigned HOST_WIDE_INT ml
;
578 if (TREE_CODE (cst
) != INTEGER_CST
|| integer_zerop (cst
))
584 && (!integer_pow2p (cst
) || tree_int_cst_sgn (cst
) != 1))
588 shifts
[i
] = tree_log2 (cst
);
589 if (shifts
[i
] != shifts
[0]
590 && code
== TRUNC_DIV_EXPR
591 && !has_vector_shift
)
596 if (sign_p
== UNSIGNED
)
598 unsigned HOST_WIDE_INT mh
;
599 unsigned HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
) & mask
;
601 if (d
>= (HOST_WIDE_INT_1U
<< (prec
- 1)))
602 /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
611 /* Find a suitable multiplier and right shift count
612 instead of multiplying with D. */
613 mh
= choose_multiplier (d
, prec
, prec
, &ml
, &post_shift
, &dummy_int
);
615 /* If the suggested multiplier is more than SIZE bits, we can
616 do better for even divisors, using an initial right shift. */
617 if ((mh
!= 0 && (d
& 1) == 0)
618 || (!has_vector_shift
&& pre_shift
!= -1))
620 if (has_vector_shift
)
621 pre_shift
= ctz_or_zero (d
);
622 else if (pre_shift
== -1)
625 for (j
= 0; j
< nunits
; j
++)
627 tree cst2
= VECTOR_CST_ELT (op1
, j
);
628 unsigned HOST_WIDE_INT d2
;
631 if (!tree_fits_uhwi_p (cst2
))
633 d2
= tree_to_uhwi (cst2
) & mask
;
636 this_pre_shift
= floor_log2 (d2
& -d2
);
637 if (pre_shift
== -1 || this_pre_shift
< pre_shift
)
638 pre_shift
= this_pre_shift
;
640 if (i
!= 0 && pre_shift
!= 0)
650 if ((d
>> pre_shift
) <= 1)
655 mh
= choose_multiplier (d
>> pre_shift
, prec
,
657 &ml
, &post_shift
, &dummy_int
);
659 pre_shifts
[i
] = pre_shift
;
669 HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
);
670 unsigned HOST_WIDE_INT abs_d
;
675 /* Since d might be INT_MIN, we have to cast to
676 unsigned HOST_WIDE_INT before negating to avoid
677 undefined signed overflow. */
679 ? (unsigned HOST_WIDE_INT
) d
680 : - (unsigned HOST_WIDE_INT
) d
);
682 /* n rem d = n rem -d */
683 if (code
== TRUNC_MOD_EXPR
&& d
< 0)
688 if (abs_d
== HOST_WIDE_INT_1U
<< (prec
- 1))
690 /* This case is not handled correctly below. */
700 choose_multiplier (abs_d
, prec
, prec
- 1, &ml
,
701 &post_shift
, &dummy_int
);
702 if (ml
>= HOST_WIDE_INT_1U
<< (prec
- 1))
704 this_mode
= 4 + (d
< 0);
705 ml
|= HOST_WIDE_INT_M1U
<< (prec
- 1);
708 this_mode
= 2 + (d
< 0);
711 post_shifts
[i
] = post_shift
;
712 if ((i
&& !has_vector_shift
&& post_shifts
[0] != post_shift
)
713 || post_shift
>= prec
714 || pre_shifts
[i
] >= prec
)
719 else if (mode
!= this_mode
)
725 tree addend
= NULL_TREE
;
726 if (sign_p
== SIGNED
)
730 /* Both division and remainder sequences need
731 op0 < 0 ? mask : 0 computed. It can be either computed as
732 (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
733 if none of the shifts is 0, or as the conditional. */
734 for (i
= 0; i
< nunits
; i
++)
738 = build_vector_type (build_nonstandard_integer_type (prec
, 1),
740 if (i
== nunits
&& TYPE_MODE (uns_type
) == TYPE_MODE (type
))
742 for (i
= 0; i
< nunits
; i
++)
743 shift_temps
[i
] = prec
- 1;
744 cur_op
= add_rshift (gsi
, type
, op0
, shift_temps
);
745 if (cur_op
!= NULL_TREE
)
747 cur_op
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
749 for (i
= 0; i
< nunits
; i
++)
750 shift_temps
[i
] = prec
- shifts
[i
];
751 cur_op
= add_rshift (gsi
, uns_type
, cur_op
, shift_temps
);
752 if (cur_op
!= NULL_TREE
)
753 addend
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
757 if (addend
== NULL_TREE
758 && expand_vec_cond_expr_p (type
, type
, LT_EXPR
))
760 tree zero
, cst
, mask_type
, mask
;
763 mask_type
= truth_type_for (type
);
764 zero
= build_zero_cst (type
);
765 mask
= make_ssa_name (mask_type
);
766 cond
= gimple_build_assign (mask
, LT_EXPR
, op0
, zero
);
767 gsi_insert_before (gsi
, cond
, GSI_SAME_STMT
);
768 tree_vector_builder
vec (type
, nunits
, 1);
769 for (i
= 0; i
< nunits
; i
++)
770 vec
.quick_push (build_int_cst (TREE_TYPE (type
),
774 addend
= make_ssa_name (type
);
776 = gimple_build_assign (addend
, VEC_COND_EXPR
, mask
, cst
, zero
);
777 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
780 if (code
== TRUNC_DIV_EXPR
)
782 if (sign_p
== UNSIGNED
)
784 /* q = op0 >> shift; */
785 cur_op
= add_rshift (gsi
, type
, op0
, shifts
);
786 if (cur_op
!= NULL_TREE
)
789 else if (addend
!= NULL_TREE
)
791 /* t1 = op0 + addend;
793 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
794 if (op
!= unknown_optab
795 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
797 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
, addend
);
798 cur_op
= add_rshift (gsi
, type
, cur_op
, shifts
);
799 if (cur_op
!= NULL_TREE
)
807 tree_vector_builder
vec (type
, nunits
, 1);
808 for (i
= 0; i
< nunits
; i
++)
809 vec
.quick_push (build_int_cst (TREE_TYPE (type
),
813 op
= optab_for_tree_code (BIT_AND_EXPR
, type
, optab_default
);
814 if (op
!= unknown_optab
815 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
817 if (sign_p
== UNSIGNED
)
818 /* r = op0 & mask; */
819 return gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, op0
, mask
);
820 else if (addend
!= NULL_TREE
)
822 /* t1 = op0 + addend;
825 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
826 if (op
!= unknown_optab
827 && optab_handler (op
, TYPE_MODE (type
))
830 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
,
832 cur_op
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
,
834 op
= optab_for_tree_code (MINUS_EXPR
, type
,
836 if (op
!= unknown_optab
837 && optab_handler (op
, TYPE_MODE (type
))
839 return gimplify_build2 (gsi
, MINUS_EXPR
, type
,
847 if (mode
== -2 || BYTES_BIG_ENDIAN
!= WORDS_BIG_ENDIAN
)
850 if (!can_mult_highpart_p (TYPE_MODE (type
), TYPE_UNSIGNED (type
)))
858 gcc_assert (sign_p
== UNSIGNED
);
859 /* t1 = oprnd0 >> pre_shift;
861 q = t2 >> post_shift; */
862 cur_op
= add_rshift (gsi
, type
, cur_op
, pre_shifts
);
863 if (cur_op
== NULL_TREE
)
867 gcc_assert (sign_p
== UNSIGNED
);
868 for (i
= 0; i
< nunits
; i
++)
878 gcc_assert (sign_p
== SIGNED
);
879 for (i
= 0; i
< nunits
; i
++)
880 shift_temps
[i
] = prec
- 1;
886 tree_vector_builder
vec (type
, nunits
, 1);
887 for (i
= 0; i
< nunits
; i
++)
888 vec
.quick_push (build_int_cst (TREE_TYPE (type
), mulc
[i
]));
889 mulcst
= vec
.build ();
891 cur_op
= gimplify_build2 (gsi
, MULT_HIGHPART_EXPR
, type
, cur_op
, mulcst
);
896 /* t1 = oprnd0 >> pre_shift;
898 q = t2 >> post_shift; */
899 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
902 /* t1 = oprnd0 h* ml;
906 q = t4 >> (post_shift - 1); */
907 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
908 if (op
== unknown_optab
909 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
911 tem
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, cur_op
);
912 tem
= add_rshift (gsi
, type
, tem
, shift_temps
);
913 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
914 if (op
== unknown_optab
915 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
917 tem
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, tem
);
918 cur_op
= add_rshift (gsi
, type
, tem
, post_shifts
);
919 if (cur_op
== NULL_TREE
)
926 /* t1 = oprnd0 h* ml;
927 t2 = t1; [ iff (mode & 2) != 0 ]
928 t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
929 t3 = t2 >> post_shift;
930 t4 = oprnd0 >> (prec - 1);
931 q = t3 - t4; [ iff (mode & 1) == 0 ]
932 q = t4 - t3; [ iff (mode & 1) != 0 ] */
935 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
936 if (op
== unknown_optab
937 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
939 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, op0
);
941 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
942 if (cur_op
== NULL_TREE
)
944 tem
= add_rshift (gsi
, type
, op0
, shift_temps
);
945 if (tem
== NULL_TREE
)
947 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
948 if (op
== unknown_optab
949 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
952 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, cur_op
, tem
);
954 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, tem
, cur_op
);
960 if (code
== TRUNC_DIV_EXPR
)
963 /* We divided. Now finish by:
966 op
= optab_for_tree_code (MULT_EXPR
, type
, optab_default
);
967 if (op
== unknown_optab
968 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
972 tree_vector_builder elts
;
973 if (!elts
.new_unary_operation (type
, op1
, false))
975 unsigned int count
= elts
.encoded_nelts ();
976 for (unsigned int i
= 0; i
< count
; ++i
)
978 tree elem1
= VECTOR_CST_ELT (op1
, i
);
980 tree elt
= const_unop (ABS_EXPR
, TREE_TYPE (elem1
), elem1
);
981 if (elt
== NULL_TREE
)
983 elts
.quick_push (elt
);
987 tem
= gimplify_build2 (gsi
, MULT_EXPR
, type
, cur_op
, op1
);
988 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
989 if (op
== unknown_optab
990 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
992 return gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, tem
);
995 /* Expand a vector condition to scalars, by using many conditions
996 on the vector's elements. */
999 expand_vector_condition (gimple_stmt_iterator
*gsi
, bitmap dce_ssa_names
)
1001 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1002 tree type
= TREE_TYPE (gimple_assign_lhs (stmt
));
1003 tree a
= gimple_assign_rhs1 (stmt
);
1005 tree a2
= NULL_TREE
;
1006 bool a_is_comparison
= false;
1007 bool a_is_scalar_bitmask
= false;
1008 tree b
= gimple_assign_rhs2 (stmt
);
1009 tree c
= gimple_assign_rhs3 (stmt
);
1010 vec
<constructor_elt
, va_gc
> *v
;
1012 tree inner_type
= TREE_TYPE (type
);
1013 tree width
= vector_element_bits_tree (type
);
1014 tree cond_type
= TREE_TYPE (TREE_TYPE (a
));
1015 tree comp_inner_type
= cond_type
;
1016 tree index
= bitsize_int (0);
1017 tree comp_width
= width
;
1018 tree comp_index
= index
;
1019 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1020 tree_code code
= TREE_CODE (a
);
1021 gassign
*assign
= NULL
;
1023 if (code
== SSA_NAME
)
1025 assign
= dyn_cast
<gassign
*> (SSA_NAME_DEF_STMT (a
));
1027 && TREE_CODE_CLASS (gimple_assign_rhs_code (assign
)) == tcc_comparison
)
1029 a_is_comparison
= true;
1030 a1
= gimple_assign_rhs1 (assign
);
1031 a2
= gimple_assign_rhs2 (assign
);
1032 code
= gimple_assign_rhs_code (assign
);
1033 comp_inner_type
= TREE_TYPE (TREE_TYPE (a1
));
1034 comp_width
= vector_element_bits_tree (TREE_TYPE (a1
));
1038 if (expand_vec_cond_expr_p (type
, TREE_TYPE (a1
), code
)
1039 || (integer_all_onesp (b
) && integer_zerop (c
)
1040 && expand_vec_cmp_expr_p (type
, TREE_TYPE (a1
), code
)))
1042 gcc_assert (TREE_CODE (a
) == SSA_NAME
|| TREE_CODE (a
) == VECTOR_CST
);
1046 /* If a has vector boolean type and is a comparison, above
1047 expand_vec_cond_expr_p might fail, even if both the comparison and
1048 VEC_COND_EXPR could be supported individually. See PR109176. */
1050 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
))
1051 && expand_vec_cond_expr_p (type
, TREE_TYPE (a
), SSA_NAME
)
1052 && expand_vec_cmp_expr_p (TREE_TYPE (a1
), TREE_TYPE (a
), code
))
1055 /* Handle vector boolean types with bitmasks. If there is a comparison
1056 and we can expand the comparison into the vector boolean bitmask,
1057 or otherwise if it is compatible with type, we can transform
1058 vbfld_1 = x_2 < y_3 ? vbfld_4 : vbfld_5;
1061 tmp_7 = tmp_6 & vbfld_4;
1063 tmp_9 = tmp_8 & vbfld_5;
1064 vbfld_1 = tmp_7 | tmp_9;
1065 Similarly for vbfld_10 instead of x_2 < y_3. */
1066 if (VECTOR_BOOLEAN_TYPE_P (type
)
1067 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
1068 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type
)),
1069 TYPE_VECTOR_SUBPARTS (type
)
1070 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (type
))))
1072 ? useless_type_conversion_p (type
, TREE_TYPE (a
))
1073 : expand_vec_cmp_expr_p (TREE_TYPE (a1
), type
, TREE_CODE (a
))))
1075 if (a_is_comparison
)
1076 a
= gimplify_build2 (gsi
, code
, type
, a1
, a2
);
1077 a1
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, a
, b
);
1078 a2
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, type
, a
);
1079 a2
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, a2
, c
);
1080 a
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, type
, a1
, a2
);
1081 gimple_assign_set_rhs_from_tree (gsi
, a
);
1082 update_stmt (gsi_stmt (*gsi
));
1086 /* TODO: try and find a smaller vector type. */
1088 if (!warning_suppressed_p (stmt
, OPT_Wvector_operation_performance
))
1089 warning_at (loc
, OPT_Wvector_operation_performance
,
1090 "vector condition will be expanded piecewise");
1092 if (!a_is_comparison
1093 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
))
1094 && SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (a
)))
1095 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (a
))),
1096 TYPE_VECTOR_SUBPARTS (TREE_TYPE (a
))
1097 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
1098 (TREE_TYPE (TREE_TYPE (a
))))))
1100 a_is_scalar_bitmask
= true;
1101 int prec
= GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (a
)));
1102 tree atype
= build_nonstandard_integer_type (prec
, 1);
1103 a
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, atype
, a
);
1105 else if (!a_is_comparison
1106 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
)))
1107 comp_width
= vector_element_bits_tree (TREE_TYPE (a
));
1109 int nunits
= nunits_for_known_piecewise_op (type
);
1110 vec_alloc (v
, nunits
);
1111 bool constant_p
= true;
1112 for (int i
= 0; i
< nunits
; i
++)
1115 tree bb
= tree_vec_extract (gsi
, inner_type
, b
, width
, index
);
1116 tree cc
= tree_vec_extract (gsi
, inner_type
, c
, width
, index
);
1117 if (a_is_comparison
)
1119 tree aa1
= tree_vec_extract (gsi
, comp_inner_type
, a1
,
1120 comp_width
, comp_index
);
1121 tree aa2
= tree_vec_extract (gsi
, comp_inner_type
, a2
,
1122 comp_width
, comp_index
);
1123 aa
= gimplify_build2 (gsi
, code
, cond_type
, aa1
, aa2
);
1125 else if (a_is_scalar_bitmask
)
1127 wide_int w
= wi::set_bit_in_zero (i
, TYPE_PRECISION (TREE_TYPE (a
)));
1128 result
= gimplify_build2 (gsi
, BIT_AND_EXPR
, TREE_TYPE (a
),
1129 a
, wide_int_to_tree (TREE_TYPE (a
), w
));
1130 aa
= gimplify_build2 (gsi
, NE_EXPR
, boolean_type_node
, result
,
1131 build_zero_cst (TREE_TYPE (a
)));
1134 aa
= tree_vec_extract (gsi
, cond_type
, a
, comp_width
, comp_index
);
1135 result
= gimplify_build3 (gsi
, COND_EXPR
, inner_type
, aa
, bb
, cc
);
1136 if (!CONSTANT_CLASS_P (result
))
1138 constructor_elt ce
= {NULL_TREE
, result
};
1140 index
= int_const_binop (PLUS_EXPR
, index
, width
);
1141 if (width
== comp_width
)
1144 comp_index
= int_const_binop (PLUS_EXPR
, comp_index
, comp_width
);
1148 constr
= build_vector_from_ctor (type
, v
);
1150 constr
= build_constructor (type
, v
);
1151 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1152 update_stmt (gsi_stmt (*gsi
));
1154 if (a_is_comparison
)
1155 bitmap_set_bit (dce_ssa_names
,
1156 SSA_NAME_VERSION (gimple_assign_lhs (assign
)));
1162 expand_vector_operation (gimple_stmt_iterator
*gsi
, tree type
, tree compute_type
,
1163 gassign
*assign
, enum tree_code code
,
1164 bitmap dce_ssa_names
)
1166 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1168 /* If the compute mode is not a vector mode (hence we are not decomposing
1169 a BLKmode vector to smaller, hardware-supported vectors), we may want
1170 to expand the operations in parallel. */
1171 if (!VECTOR_MODE_P (compute_mode
))
1176 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
1177 return expand_vector_addition (gsi
, do_binop
, do_plus_minus
, type
,
1178 gimple_assign_rhs1 (assign
),
1179 gimple_assign_rhs2 (assign
), code
);
1183 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
1184 return expand_vector_addition (gsi
, do_unop
, do_negate
, type
,
1185 gimple_assign_rhs1 (assign
),
1192 return expand_vector_parallel (gsi
, do_binop
, type
,
1193 gimple_assign_rhs1 (assign
),
1194 gimple_assign_rhs2 (assign
), code
);
1197 return expand_vector_parallel (gsi
, do_unop
, type
,
1198 gimple_assign_rhs1 (assign
),
1213 case UNORDERED_EXPR
:
1215 tree rhs1
= gimple_assign_rhs1 (assign
);
1216 tree rhs2
= gimple_assign_rhs2 (assign
);
1218 return expand_vector_comparison (gsi
, type
, rhs1
, rhs2
, code
,
1222 case TRUNC_DIV_EXPR
:
1223 case TRUNC_MOD_EXPR
:
1225 tree rhs1
= gimple_assign_rhs1 (assign
);
1226 tree rhs2
= gimple_assign_rhs2 (assign
);
1230 || !VECTOR_INTEGER_TYPE_P (type
)
1231 || TREE_CODE (rhs2
) != VECTOR_CST
1232 || !VECTOR_MODE_P (TYPE_MODE (type
)))
1235 ret
= expand_vector_divmod (gsi
, type
, rhs1
, rhs2
, code
);
1236 if (ret
!= NULL_TREE
)
1245 if (TREE_CODE_CLASS (code
) == tcc_unary
)
1246 return expand_vector_piecewise (gsi
, do_unop
, type
, compute_type
,
1247 gimple_assign_rhs1 (assign
),
1248 NULL_TREE
, code
, false);
1250 return expand_vector_piecewise (gsi
, do_binop
, type
, compute_type
,
1251 gimple_assign_rhs1 (assign
),
1252 gimple_assign_rhs2 (assign
), code
, false);
1256 a_5 = { b_7, b_7 + 3, b_7 + 6, b_7 + 9 };
1258 _9 = { b_7, b_7, b_7, b_7 };
1259 a_5 = _9 + { 0, 3, 6, 9 };
1260 because vector splat operation is usually more efficient
1261 than piecewise initialization of the vector. */
1264 optimize_vector_constructor (gimple_stmt_iterator
*gsi
)
1266 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1267 tree lhs
= gimple_assign_lhs (stmt
);
1268 tree rhs
= gimple_assign_rhs1 (stmt
);
1269 tree type
= TREE_TYPE (rhs
);
1271 unsigned HOST_WIDE_INT nelts
;
1272 bool all_same
= true;
1273 constructor_elt
*elt
;
1275 tree base
= NULL_TREE
;
1278 if (!TYPE_VECTOR_SUBPARTS (type
).is_constant (&nelts
)
1280 || CONSTRUCTOR_NELTS (rhs
) != nelts
)
1282 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
1283 if (op
== unknown_optab
1284 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
1286 FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (rhs
), i
, elt
)
1287 if (TREE_CODE (elt
->value
) != SSA_NAME
1288 || TREE_CODE (TREE_TYPE (elt
->value
)) == VECTOR_TYPE
)
1292 tree this_base
= elt
->value
;
1293 if (this_base
!= CONSTRUCTOR_ELT (rhs
, 0)->value
)
1295 for (j
= 0; j
< nelts
+ 1; j
++)
1297 g
= SSA_NAME_DEF_STMT (this_base
);
1298 if (is_gimple_assign (g
)
1299 && gimple_assign_rhs_code (g
) == PLUS_EXPR
1300 && TREE_CODE (gimple_assign_rhs2 (g
)) == INTEGER_CST
1301 && TREE_CODE (gimple_assign_rhs1 (g
)) == SSA_NAME
1302 && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g
)))
1303 this_base
= gimple_assign_rhs1 (g
);
1309 else if (this_base
!= base
)
1314 tree_vector_builder
cst (type
, nelts
, 1);
1315 for (i
= 0; i
< nelts
; i
++)
1317 tree this_base
= CONSTRUCTOR_ELT (rhs
, i
)->value
;
1318 tree elt
= build_zero_cst (TREE_TYPE (base
));
1319 while (this_base
!= base
)
1321 g
= SSA_NAME_DEF_STMT (this_base
);
1322 elt
= fold_binary (PLUS_EXPR
, TREE_TYPE (base
),
1323 elt
, gimple_assign_rhs2 (g
));
1324 if (elt
== NULL_TREE
1325 || TREE_CODE (elt
) != INTEGER_CST
1326 || TREE_OVERFLOW (elt
))
1328 this_base
= gimple_assign_rhs1 (g
);
1330 cst
.quick_push (elt
);
1332 for (i
= 0; i
< nelts
; i
++)
1333 CONSTRUCTOR_ELT (rhs
, i
)->value
= base
;
1334 g
= gimple_build_assign (make_ssa_name (type
), rhs
);
1335 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1336 g
= gimple_build_assign (lhs
, PLUS_EXPR
, gimple_assign_lhs (g
),
1338 gsi_replace (gsi
, g
, false);
1341 /* Return a type for the widest vector mode whose components are of type
1342 TYPE, or NULL_TREE if none is found. */
1345 type_for_widest_vector_mode (tree type
, optab op
)
1347 machine_mode inner_mode
= TYPE_MODE (type
);
1348 machine_mode best_mode
= VOIDmode
, mode
;
1349 poly_int64 best_nunits
= 0;
1351 if (SCALAR_FLOAT_MODE_P (inner_mode
))
1352 mode
= MIN_MODE_VECTOR_FLOAT
;
1353 else if (SCALAR_FRACT_MODE_P (inner_mode
))
1354 mode
= MIN_MODE_VECTOR_FRACT
;
1355 else if (SCALAR_UFRACT_MODE_P (inner_mode
))
1356 mode
= MIN_MODE_VECTOR_UFRACT
;
1357 else if (SCALAR_ACCUM_MODE_P (inner_mode
))
1358 mode
= MIN_MODE_VECTOR_ACCUM
;
1359 else if (SCALAR_UACCUM_MODE_P (inner_mode
))
1360 mode
= MIN_MODE_VECTOR_UACCUM
;
1361 else if (inner_mode
== BImode
)
1362 mode
= MIN_MODE_VECTOR_BOOL
;
1364 mode
= MIN_MODE_VECTOR_INT
;
1366 FOR_EACH_MODE_FROM (mode
, mode
)
1367 if (GET_MODE_INNER (mode
) == inner_mode
1368 && maybe_gt (GET_MODE_NUNITS (mode
), best_nunits
)
1369 && optab_handler (op
, mode
) != CODE_FOR_nothing
)
1370 best_mode
= mode
, best_nunits
= GET_MODE_NUNITS (mode
);
1372 if (best_mode
== VOIDmode
)
1375 return build_vector_type_for_mode (type
, best_mode
);
1379 /* Build a reference to the element of the vector VECT. Function
1380 returns either the element itself, either BIT_FIELD_REF, or an
1381 ARRAY_REF expression.
1383 GSI is required to insert temporary variables while building a
1384 refernece to the element of the vector VECT.
1386 PTMPVEC is a pointer to the temporary variable for caching
1387 purposes. In case when PTMPVEC is NULL new temporary variable
1390 vector_element (gimple_stmt_iterator
*gsi
, tree vect
, tree idx
, tree
*ptmpvec
)
1392 tree vect_type
, vect_elt_type
;
1396 bool need_asgn
= true;
1397 unsigned int elements
;
1399 vect_type
= TREE_TYPE (vect
);
1400 vect_elt_type
= TREE_TYPE (vect_type
);
1401 elements
= nunits_for_known_piecewise_op (vect_type
);
1403 if (TREE_CODE (idx
) == INTEGER_CST
)
1405 unsigned HOST_WIDE_INT index
;
1407 /* Given that we're about to compute a binary modulus,
1408 we don't care about the high bits of the value. */
1409 index
= TREE_INT_CST_LOW (idx
);
1410 if (!tree_fits_uhwi_p (idx
) || index
>= elements
)
1412 index
&= elements
- 1;
1413 idx
= build_int_cst (TREE_TYPE (idx
), index
);
1416 /* When lowering a vector statement sequence do some easy
1417 simplification by looking through intermediate vector results. */
1418 if (TREE_CODE (vect
) == SSA_NAME
)
1420 gimple
*def_stmt
= SSA_NAME_DEF_STMT (vect
);
1421 if (is_gimple_assign (def_stmt
)
1422 && (gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
1423 || gimple_assign_rhs_code (def_stmt
) == CONSTRUCTOR
))
1424 vect
= gimple_assign_rhs1 (def_stmt
);
1427 if (TREE_CODE (vect
) == VECTOR_CST
)
1428 return VECTOR_CST_ELT (vect
, index
);
1429 else if (TREE_CODE (vect
) == CONSTRUCTOR
1430 && (CONSTRUCTOR_NELTS (vect
) == 0
1431 || TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (vect
, 0)->value
))
1434 if (index
< CONSTRUCTOR_NELTS (vect
))
1435 return CONSTRUCTOR_ELT (vect
, index
)->value
;
1436 return build_zero_cst (vect_elt_type
);
1440 tree size
= vector_element_bits_tree (vect_type
);
1441 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
, bitsize_int (index
),
1443 return fold_build3 (BIT_FIELD_REF
, vect_elt_type
, vect
, size
, pos
);
1448 tmpvec
= create_tmp_var (vect_type
, "vectmp");
1450 tmpvec
= *ptmpvec
= create_tmp_var (vect_type
, "vectmp");
1459 TREE_ADDRESSABLE (tmpvec
) = 1;
1460 asgn
= gimple_build_assign (tmpvec
, vect
);
1461 gsi_insert_before (gsi
, asgn
, GSI_SAME_STMT
);
1464 arraytype
= build_array_type_nelts (vect_elt_type
, elements
);
1465 return build4 (ARRAY_REF
, vect_elt_type
,
1466 build1 (VIEW_CONVERT_EXPR
, arraytype
, tmpvec
),
1467 idx
, NULL_TREE
, NULL_TREE
);
1470 /* Check if VEC_PERM_EXPR within the given setting is supported
1471 by hardware, or lower it piecewise.
1473 When VEC_PERM_EXPR has the same first and second operands:
1474 VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1475 {v0[mask[0]], v0[mask[1]], ...}
1476 MASK and V0 must have the same number of elements.
1478 Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1479 {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1480 V0 and V1 must have the same type. MASK, V0, V1 must have the
1481 same number of arguments. */
1484 lower_vec_perm (gimple_stmt_iterator
*gsi
)
1486 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1487 tree mask
= gimple_assign_rhs3 (stmt
);
1488 tree vec0
= gimple_assign_rhs1 (stmt
);
1489 tree vec1
= gimple_assign_rhs2 (stmt
);
1490 tree vect_type
= TREE_TYPE (vec0
);
1491 tree mask_type
= TREE_TYPE (mask
);
1492 tree vect_elt_type
= TREE_TYPE (vect_type
);
1493 tree mask_elt_type
= TREE_TYPE (mask_type
);
1494 unsigned HOST_WIDE_INT elements
;
1495 vec
<constructor_elt
, va_gc
> *v
;
1496 tree constr
, t
, si
, i_val
;
1497 tree vec0tmp
= NULL_TREE
, vec1tmp
= NULL_TREE
, masktmp
= NULL_TREE
;
1498 bool two_operand_p
= !operand_equal_p (vec0
, vec1
, 0);
1499 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1502 if (!TYPE_VECTOR_SUBPARTS (vect_type
).is_constant (&elements
))
1505 if (TREE_CODE (mask
) == SSA_NAME
)
1507 gimple
*def_stmt
= SSA_NAME_DEF_STMT (mask
);
1508 if (is_gimple_assign (def_stmt
)
1509 && gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
)
1510 mask
= gimple_assign_rhs1 (def_stmt
);
1513 vec_perm_builder sel_int
;
1515 if (TREE_CODE (mask
) == VECTOR_CST
1516 && tree_to_vec_perm_builder (&sel_int
, mask
))
1518 vec_perm_indices
indices (sel_int
, 2, elements
);
1519 machine_mode vmode
= TYPE_MODE (vect_type
);
1520 tree lhs_type
= TREE_TYPE (gimple_assign_lhs (stmt
));
1521 machine_mode lhs_mode
= TYPE_MODE (lhs_type
);
1522 if (can_vec_perm_const_p (lhs_mode
, vmode
, indices
))
1524 gimple_assign_set_rhs3 (stmt
, mask
);
1528 /* Also detect vec_shr pattern - VEC_PERM_EXPR with zero
1529 vector as VEC1 and a right element shift MASK. */
1530 if (optab_handler (vec_shr_optab
, TYPE_MODE (vect_type
))
1532 && TREE_CODE (vec1
) == VECTOR_CST
1533 && initializer_zerop (vec1
)
1534 && maybe_ne (indices
[0], 0)
1535 && known_lt (poly_uint64 (indices
[0]), elements
))
1537 bool ok_p
= indices
.series_p (0, 1, indices
[0], 1);
1540 for (i
= 1; i
< elements
; ++i
)
1542 poly_uint64 actual
= indices
[i
];
1543 poly_uint64 expected
= i
+ indices
[0];
1544 /* Indices into the second vector are all equivalent. */
1545 if (maybe_lt (actual
, elements
)
1546 ? maybe_ne (actual
, expected
)
1547 : maybe_lt (expected
, elements
))
1550 ok_p
= i
== elements
;
1554 gimple_assign_set_rhs3 (stmt
, mask
);
1559 /* And similarly vec_shl pattern. */
1560 if (optab_handler (vec_shl_optab
, TYPE_MODE (vect_type
))
1562 && TREE_CODE (vec0
) == VECTOR_CST
1563 && initializer_zerop (vec0
))
1565 unsigned int first
= 0;
1566 for (i
= 0; i
< elements
; ++i
)
1567 if (known_eq (poly_uint64 (indices
[i
]), elements
))
1569 if (i
== 0 || first
)
1574 ? maybe_ne (poly_uint64 (indices
[i
]),
1575 elements
+ i
- first
)
1576 : maybe_ge (poly_uint64 (indices
[i
]), elements
))
1578 if (first
&& i
== elements
)
1580 gimple_assign_set_rhs3 (stmt
, mask
);
1586 else if (can_vec_perm_var_p (TYPE_MODE (vect_type
)))
1589 if (!warning_suppressed_p (stmt
, OPT_Wvector_operation_performance
))
1590 warning_at (loc
, OPT_Wvector_operation_performance
,
1591 "vector shuffling operation will be expanded piecewise");
1593 vec_alloc (v
, elements
);
1594 bool constant_p
= true;
1595 for (i
= 0; i
< elements
; i
++)
1598 i_val
= vector_element (gsi
, mask
, si
, &masktmp
);
1600 if (TREE_CODE (i_val
) == INTEGER_CST
)
1602 unsigned HOST_WIDE_INT index
;
1604 index
= TREE_INT_CST_LOW (i_val
);
1605 if (!tree_fits_uhwi_p (i_val
) || index
>= elements
)
1606 i_val
= build_int_cst (mask_elt_type
, index
& (elements
- 1));
1608 if (two_operand_p
&& (index
& elements
) != 0)
1609 t
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1611 t
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1613 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
1614 true, GSI_SAME_STMT
);
1618 tree cond
= NULL_TREE
, v0_val
;
1622 cond
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1623 build_int_cst (mask_elt_type
, elements
));
1624 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1625 true, GSI_SAME_STMT
);
1628 i_val
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1629 build_int_cst (mask_elt_type
, elements
- 1));
1630 i_val
= force_gimple_operand_gsi (gsi
, i_val
, true, NULL_TREE
,
1631 true, GSI_SAME_STMT
);
1633 v0_val
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1634 v0_val
= force_gimple_operand_gsi (gsi
, v0_val
, true, NULL_TREE
,
1635 true, GSI_SAME_STMT
);
1641 v1_val
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1642 v1_val
= force_gimple_operand_gsi (gsi
, v1_val
, true, NULL_TREE
,
1643 true, GSI_SAME_STMT
);
1645 cond
= fold_build2 (EQ_EXPR
, boolean_type_node
,
1646 cond
, build_zero_cst (mask_elt_type
));
1647 cond
= fold_build3 (COND_EXPR
, vect_elt_type
,
1648 cond
, v0_val
, v1_val
);
1649 t
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1650 true, GSI_SAME_STMT
);
1656 if (!CONSTANT_CLASS_P (t
))
1658 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, t
);
1662 constr
= build_vector_from_ctor (vect_type
, v
);
1664 constr
= build_constructor (vect_type
, v
);
1665 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1666 update_stmt (gsi_stmt (*gsi
));
1669 /* If OP is a uniform vector return the element it is a splat from. */
1672 ssa_uniform_vector_p (tree op
)
1674 if (TREE_CODE (op
) == VECTOR_CST
1675 || TREE_CODE (op
) == VEC_DUPLICATE_EXPR
1676 || TREE_CODE (op
) == CONSTRUCTOR
)
1677 return uniform_vector_p (op
);
1678 if (TREE_CODE (op
) == SSA_NAME
)
1680 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
1681 if (gimple_assign_single_p (def_stmt
))
1682 return uniform_vector_p (gimple_assign_rhs1 (def_stmt
));
1687 /* Return type in which CODE operation with optab OP can be
1691 get_compute_type (enum tree_code code
, optab op
, tree type
)
1693 /* For very wide vectors, try using a smaller vector mode. */
1694 tree compute_type
= type
;
1696 && (!VECTOR_MODE_P (TYPE_MODE (type
))
1697 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
))
1699 tree vector_compute_type
1700 = type_for_widest_vector_mode (TREE_TYPE (type
), op
);
1701 if (vector_compute_type
!= NULL_TREE
1702 && subparts_gt (compute_type
, vector_compute_type
)
1703 && maybe_ne (TYPE_VECTOR_SUBPARTS (vector_compute_type
), 1U)
1704 && (optab_handler (op
, TYPE_MODE (vector_compute_type
))
1705 != CODE_FOR_nothing
))
1706 compute_type
= vector_compute_type
;
1709 /* If we are breaking a BLKmode vector into smaller pieces,
1710 type_for_widest_vector_mode has already looked into the optab,
1711 so skip these checks. */
1712 if (compute_type
== type
)
1714 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1715 if (VECTOR_MODE_P (compute_mode
))
1717 if (op
&& optab_handler (op
, compute_mode
) != CODE_FOR_nothing
)
1718 return compute_type
;
1719 if (code
== MULT_HIGHPART_EXPR
1720 && can_mult_highpart_p (compute_mode
,
1721 TYPE_UNSIGNED (compute_type
)))
1722 return compute_type
;
1724 /* There is no operation in hardware, so fall back to scalars. */
1725 compute_type
= TREE_TYPE (type
);
1728 return compute_type
;
1732 do_cond (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
1733 tree bitpos
, tree bitsize
, enum tree_code code
,
1734 tree type ATTRIBUTE_UNUSED
)
1736 if (TREE_CODE (TREE_TYPE (a
)) == VECTOR_TYPE
)
1737 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1738 if (TREE_CODE (TREE_TYPE (b
)) == VECTOR_TYPE
)
1739 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
1740 tree cond
= gimple_assign_rhs1 (gsi_stmt (*gsi
));
1741 return gimplify_build3 (gsi
, code
, inner_type
, unshare_expr (cond
), a
, b
);
1744 /* Expand a vector COND_EXPR to scalars, piecewise. */
1746 expand_vector_scalar_condition (gimple_stmt_iterator
*gsi
)
1748 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1749 tree lhs
= gimple_assign_lhs (stmt
);
1750 tree type
= TREE_TYPE (lhs
);
1751 tree compute_type
= get_compute_type (COND_EXPR
, mov_optab
, type
);
1752 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1753 gcc_assert (compute_mode
!= BLKmode
);
1754 tree rhs2
= gimple_assign_rhs2 (stmt
);
1755 tree rhs3
= gimple_assign_rhs3 (stmt
);
1758 /* If the compute mode is not a vector mode (hence we are not decomposing
1759 a BLKmode vector to smaller, hardware-supported vectors), we may want
1760 to expand the operations in parallel. */
1761 if (!VECTOR_MODE_P (compute_mode
))
1762 new_rhs
= expand_vector_parallel (gsi
, do_cond
, type
, rhs2
, rhs3
,
1765 new_rhs
= expand_vector_piecewise (gsi
, do_cond
, type
, compute_type
,
1766 rhs2
, rhs3
, COND_EXPR
, false);
1767 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
1768 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
1771 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1772 way to do it is change expand_vector_operation and its callees to
1773 return a tree_code, RHS1 and RHS2 instead of a tree. */
1774 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
1775 update_stmt (gsi_stmt (*gsi
));
1778 /* Callback for expand_vector_piecewise to do VEC_CONVERT ifn call
1779 lowering. If INNER_TYPE is not a vector type, this is a scalar
1783 do_vec_conversion (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
1784 tree decl
, tree bitpos
, tree bitsize
,
1785 enum tree_code code
, tree type
)
1787 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1788 if (!VECTOR_TYPE_P (inner_type
))
1789 return gimplify_build1 (gsi
, code
, TREE_TYPE (type
), a
);
1790 if (code
== CALL_EXPR
)
1792 gimple
*g
= gimple_build_call (decl
, 1, a
);
1793 tree lhs
= make_ssa_name (TREE_TYPE (TREE_TYPE (decl
)));
1794 gimple_call_set_lhs (g
, lhs
);
1795 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1800 tree outer_type
= build_vector_type (TREE_TYPE (type
),
1801 TYPE_VECTOR_SUBPARTS (inner_type
));
1802 return gimplify_build1 (gsi
, code
, outer_type
, a
);
1806 /* Similarly, but for narrowing conversion. */
1809 do_vec_narrow_conversion (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
1810 tree
, tree bitpos
, tree
, enum tree_code code
,
1813 tree itype
= build_vector_type (TREE_TYPE (inner_type
),
1814 exact_div (TYPE_VECTOR_SUBPARTS (inner_type
),
1816 tree b
= tree_vec_extract (gsi
, itype
, a
, TYPE_SIZE (itype
), bitpos
);
1817 tree c
= tree_vec_extract (gsi
, itype
, a
, TYPE_SIZE (itype
),
1818 int_const_binop (PLUS_EXPR
, bitpos
,
1819 TYPE_SIZE (itype
)));
1820 tree outer_type
= build_vector_type (TREE_TYPE (type
),
1821 TYPE_VECTOR_SUBPARTS (inner_type
));
1822 return gimplify_build2 (gsi
, code
, outer_type
, b
, c
);
1825 /* Expand VEC_CONVERT ifn call. */
1828 expand_vector_conversion (gimple_stmt_iterator
*gsi
)
1830 gimple
*stmt
= gsi_stmt (*gsi
);
1832 tree lhs
= gimple_call_lhs (stmt
);
1833 if (lhs
== NULL_TREE
)
1835 g
= gimple_build_nop ();
1836 gsi_replace (gsi
, g
, false);
1839 tree arg
= gimple_call_arg (stmt
, 0);
1840 tree ret_type
= TREE_TYPE (lhs
);
1841 tree arg_type
= TREE_TYPE (arg
);
1842 tree new_rhs
, compute_type
= TREE_TYPE (arg_type
);
1843 enum tree_code code
= NOP_EXPR
;
1844 enum tree_code code1
= ERROR_MARK
;
1845 enum { NARROW
, NONE
, WIDEN
} modifier
= NONE
;
1846 optab optab1
= unknown_optab
;
1848 gcc_checking_assert (VECTOR_TYPE_P (ret_type
) && VECTOR_TYPE_P (arg_type
));
1849 if (INTEGRAL_TYPE_P (TREE_TYPE (ret_type
))
1850 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg_type
)))
1851 code
= FIX_TRUNC_EXPR
;
1852 else if (INTEGRAL_TYPE_P (TREE_TYPE (arg_type
))
1853 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (ret_type
)))
1855 unsigned int ret_elt_bits
= vector_element_bits (ret_type
);
1856 unsigned int arg_elt_bits
= vector_element_bits (arg_type
);
1857 if (ret_elt_bits
< arg_elt_bits
)
1859 else if (ret_elt_bits
> arg_elt_bits
)
1862 if (modifier
== NONE
&& (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
))
1864 if (supportable_convert_operation (code
, ret_type
, arg_type
, &code1
))
1866 g
= gimple_build_assign (lhs
, code1
, arg
);
1867 gsi_replace (gsi
, g
, false);
1870 /* Can't use get_compute_type here, as supportable_convert_operation
1871 doesn't necessarily use an optab and needs two arguments. */
1872 tree vec_compute_type
1873 = type_for_widest_vector_mode (TREE_TYPE (arg_type
), mov_optab
);
1874 if (vec_compute_type
1875 && VECTOR_MODE_P (TYPE_MODE (vec_compute_type
))
1876 && subparts_gt (arg_type
, vec_compute_type
))
1878 unsigned HOST_WIDE_INT nelts
1879 = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vec_compute_type
));
1882 tree ret1_type
= build_vector_type (TREE_TYPE (ret_type
), nelts
);
1883 tree arg1_type
= build_vector_type (TREE_TYPE (arg_type
), nelts
);
1884 if (supportable_convert_operation (code
, ret1_type
, arg1_type
,
1887 new_rhs
= expand_vector_piecewise (gsi
, do_vec_conversion
,
1888 ret_type
, arg1_type
, arg
,
1889 NULL_TREE
, code1
, false);
1890 g
= gimple_build_assign (lhs
, new_rhs
);
1891 gsi_replace (gsi
, g
, false);
1898 else if (modifier
== NARROW
)
1903 code1
= VEC_PACK_TRUNC_EXPR
;
1904 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1906 case FIX_TRUNC_EXPR
:
1907 code1
= VEC_PACK_FIX_TRUNC_EXPR
;
1908 /* The signedness is determined from output operand. */
1909 optab1
= optab_for_tree_code (code1
, ret_type
, optab_default
);
1912 code1
= VEC_PACK_FLOAT_EXPR
;
1913 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1920 compute_type
= get_compute_type (code1
, optab1
, arg_type
);
1921 enum insn_code icode1
;
1922 if (VECTOR_TYPE_P (compute_type
)
1923 && ((icode1
= optab_handler (optab1
, TYPE_MODE (compute_type
)))
1924 != CODE_FOR_nothing
)
1925 && VECTOR_MODE_P (insn_data
[icode1
].operand
[0].mode
))
1928 = build_vector_type (TREE_TYPE (ret_type
),
1929 TYPE_VECTOR_SUBPARTS (compute_type
) * 2);
1930 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (cretd_type
))
1932 if (compute_type
== arg_type
)
1934 new_rhs
= gimplify_build2 (gsi
, code1
, cretd_type
,
1935 arg
, build_zero_cst (arg_type
));
1936 new_rhs
= tree_vec_extract (gsi
, ret_type
, new_rhs
,
1937 TYPE_SIZE (ret_type
),
1939 g
= gimple_build_assign (lhs
, new_rhs
);
1940 gsi_replace (gsi
, g
, false);
1944 = build_vector_type (TREE_TYPE (compute_type
),
1945 TYPE_VECTOR_SUBPARTS (compute_type
) * 2);
1946 if (TYPE_MAIN_VARIANT (dcompute_type
)
1947 == TYPE_MAIN_VARIANT (arg_type
))
1948 new_rhs
= do_vec_narrow_conversion (gsi
, dcompute_type
, arg
,
1949 NULL_TREE
, bitsize_int (0),
1953 new_rhs
= expand_vector_piecewise (gsi
,
1954 do_vec_narrow_conversion
,
1955 arg_type
, dcompute_type
,
1956 arg
, NULL_TREE
, code1
,
1958 g
= gimple_build_assign (lhs
, new_rhs
);
1959 gsi_replace (gsi
, g
, false);
1964 else if (modifier
== WIDEN
)
1966 enum tree_code code2
= ERROR_MARK
;
1967 optab optab2
= unknown_optab
;
1971 code1
= VEC_UNPACK_LO_EXPR
;
1972 code2
= VEC_UNPACK_HI_EXPR
;
1974 case FIX_TRUNC_EXPR
:
1975 code1
= VEC_UNPACK_FIX_TRUNC_LO_EXPR
;
1976 code2
= VEC_UNPACK_FIX_TRUNC_HI_EXPR
;
1979 code1
= VEC_UNPACK_FLOAT_LO_EXPR
;
1980 code2
= VEC_UNPACK_FLOAT_HI_EXPR
;
1985 if (BYTES_BIG_ENDIAN
)
1986 std::swap (code1
, code2
);
1988 if (code
== FIX_TRUNC_EXPR
)
1990 /* The signedness is determined from output operand. */
1991 optab1
= optab_for_tree_code (code1
, ret_type
, optab_default
);
1992 optab2
= optab_for_tree_code (code2
, ret_type
, optab_default
);
1996 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1997 optab2
= optab_for_tree_code (code2
, arg_type
, optab_default
);
2000 if (optab1
&& optab2
)
2001 compute_type
= get_compute_type (code1
, optab1
, arg_type
);
2003 enum insn_code icode1
, icode2
;
2004 if (VECTOR_TYPE_P (compute_type
)
2005 && ((icode1
= optab_handler (optab1
, TYPE_MODE (compute_type
)))
2006 != CODE_FOR_nothing
)
2007 && ((icode2
= optab_handler (optab2
, TYPE_MODE (compute_type
)))
2008 != CODE_FOR_nothing
)
2009 && VECTOR_MODE_P (insn_data
[icode1
].operand
[0].mode
)
2010 && (insn_data
[icode1
].operand
[0].mode
2011 == insn_data
[icode2
].operand
[0].mode
))
2014 = exact_div (TYPE_VECTOR_SUBPARTS (compute_type
), 2);
2015 tree cretd_type
= build_vector_type (TREE_TYPE (ret_type
), nunits
);
2016 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (cretd_type
))
2018 vec
<constructor_elt
, va_gc
> *v
;
2019 tree part_width
= TYPE_SIZE (compute_type
);
2020 tree index
= bitsize_int (0);
2021 int nunits
= nunits_for_known_piecewise_op (arg_type
);
2022 int delta
= tree_to_uhwi (part_width
) / arg_elt_bits
;
2024 location_t loc
= gimple_location (gsi_stmt (*gsi
));
2026 if (compute_type
!= arg_type
)
2028 if (!warning_suppressed_p (gsi_stmt (*gsi
),
2029 OPT_Wvector_operation_performance
))
2030 warning_at (loc
, OPT_Wvector_operation_performance
,
2031 "vector operation will be expanded piecewise");
2039 vec_alloc (v
, (nunits
+ delta
- 1) / delta
* 2);
2040 bool constant_p
= true;
2041 for (i
= 0; i
< nunits
;
2042 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
,
2046 if (compute_type
!= arg_type
)
2047 a
= tree_vec_extract (gsi
, compute_type
, a
, part_width
,
2049 tree result
= gimplify_build1 (gsi
, code1
, cretd_type
, a
);
2050 constructor_elt ce
= { NULL_TREE
, result
};
2051 if (!CONSTANT_CLASS_P (ce
.value
))
2054 ce
.value
= gimplify_build1 (gsi
, code2
, cretd_type
, a
);
2055 if (!CONSTANT_CLASS_P (ce
.value
))
2061 new_rhs
= build_vector_from_ctor (ret_type
, v
);
2063 new_rhs
= build_constructor (ret_type
, v
);
2064 g
= gimple_build_assign (lhs
, new_rhs
);
2065 gsi_replace (gsi
, g
, false);
2071 new_rhs
= expand_vector_piecewise (gsi
, do_vec_conversion
, arg_type
,
2072 TREE_TYPE (arg_type
), arg
,
2073 NULL_TREE
, code
, false, ret_type
);
2074 g
= gimple_build_assign (lhs
, new_rhs
);
2075 gsi_replace (gsi
, g
, false);
2078 /* Process one statement. If we identify a vector operation, expand it. */
2081 expand_vector_operations_1 (gimple_stmt_iterator
*gsi
,
2082 bitmap dce_ssa_names
)
2084 tree lhs
, rhs1
, rhs2
= NULL
, type
, compute_type
= NULL_TREE
;
2085 enum tree_code code
;
2086 optab op
= unknown_optab
;
2087 enum gimple_rhs_class rhs_class
;
2090 /* Only consider code == GIMPLE_ASSIGN. */
2091 gassign
*stmt
= dyn_cast
<gassign
*> (gsi_stmt (*gsi
));
2094 if (gimple_call_internal_p (gsi_stmt (*gsi
), IFN_VEC_CONVERT
))
2095 expand_vector_conversion (gsi
);
2099 code
= gimple_assign_rhs_code (stmt
);
2100 rhs_class
= get_gimple_rhs_class (code
);
2101 lhs
= gimple_assign_lhs (stmt
);
2103 if (code
== VEC_PERM_EXPR
)
2105 lower_vec_perm (gsi
);
2109 if (code
== VEC_COND_EXPR
)
2111 expand_vector_condition (gsi
, dce_ssa_names
);
2115 if (code
== COND_EXPR
2116 && TREE_CODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == VECTOR_TYPE
2117 && TYPE_MODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == BLKmode
)
2119 expand_vector_scalar_condition (gsi
);
2123 if (code
== CONSTRUCTOR
2124 && TREE_CODE (lhs
) == SSA_NAME
2125 && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (lhs
)))
2126 && !gimple_clobber_p (stmt
)
2129 optimize_vector_constructor (gsi
);
2133 if (rhs_class
!= GIMPLE_UNARY_RHS
&& rhs_class
!= GIMPLE_BINARY_RHS
)
2136 rhs1
= gimple_assign_rhs1 (stmt
);
2137 if (rhs_class
== GIMPLE_BINARY_RHS
)
2138 rhs2
= gimple_assign_rhs2 (stmt
);
2140 type
= TREE_TYPE (lhs
);
2141 if (!VECTOR_TYPE_P (type
)
2142 || !VECTOR_TYPE_P (TREE_TYPE (rhs1
)))
2145 /* A scalar operation pretending to be a vector one. */
2146 if (VECTOR_BOOLEAN_TYPE_P (type
)
2147 && !VECTOR_MODE_P (TYPE_MODE (type
))
2148 && TYPE_MODE (type
) != BLKmode
2149 && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt
)) != tcc_comparison
2150 || (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1
))
2151 && !VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (rhs1
)))
2152 && TYPE_MODE (TREE_TYPE (rhs1
)) != BLKmode
)))
2155 /* If the vector operation is operating on all same vector elements
2156 implement it with a scalar operation and a splat if the target
2157 supports the scalar operation. */
2158 tree srhs1
, srhs2
= NULL_TREE
;
2159 if ((srhs1
= ssa_uniform_vector_p (rhs1
)) != NULL_TREE
2160 && (rhs2
== NULL_TREE
2161 || (! VECTOR_TYPE_P (TREE_TYPE (rhs2
))
2163 || (srhs2
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
2164 /* As we query direct optabs restrict to non-convert operations. */
2165 && TYPE_MODE (TREE_TYPE (type
)) == TYPE_MODE (TREE_TYPE (srhs1
)))
2167 op
= optab_for_tree_code (code
, TREE_TYPE (type
), optab_scalar
);
2168 if (op
>= FIRST_NORM_OPTAB
&& op
<= LAST_NORM_OPTAB
2169 && optab_handler (op
, TYPE_MODE (TREE_TYPE (type
))) != CODE_FOR_nothing
)
2171 tree stype
= TREE_TYPE (TREE_TYPE (lhs
));
2172 tree slhs
= (rhs2
!= NULL_TREE
)
2173 ? gimplify_build2 (gsi
, code
, stype
, srhs1
, srhs2
)
2174 : gimplify_build1 (gsi
, code
, stype
, srhs1
);
2175 gimple_assign_set_rhs_from_tree (gsi
,
2176 build_vector_from_val (type
, slhs
));
2182 if (CONVERT_EXPR_CODE_P (code
)
2183 || code
== FLOAT_EXPR
2184 || code
== FIX_TRUNC_EXPR
2185 || code
== VIEW_CONVERT_EXPR
)
2188 /* The signedness is determined from input argument. */
2189 if (code
== VEC_UNPACK_FLOAT_HI_EXPR
2190 || code
== VEC_UNPACK_FLOAT_LO_EXPR
2191 || code
== VEC_PACK_FLOAT_EXPR
)
2193 /* We do not know how to scalarize those. */
2197 /* For widening/narrowing vector operations, the relevant type is of the
2198 arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
2199 calculated in the same way above. */
2200 if (code
== WIDEN_SUM_EXPR
2201 || code
== VEC_WIDEN_PLUS_HI_EXPR
2202 || code
== VEC_WIDEN_PLUS_LO_EXPR
2203 || code
== VEC_WIDEN_MINUS_HI_EXPR
2204 || code
== VEC_WIDEN_MINUS_LO_EXPR
2205 || code
== VEC_WIDEN_MULT_HI_EXPR
2206 || code
== VEC_WIDEN_MULT_LO_EXPR
2207 || code
== VEC_WIDEN_MULT_EVEN_EXPR
2208 || code
== VEC_WIDEN_MULT_ODD_EXPR
2209 || code
== VEC_UNPACK_HI_EXPR
2210 || code
== VEC_UNPACK_LO_EXPR
2211 || code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
2212 || code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
2213 || code
== VEC_PACK_TRUNC_EXPR
2214 || code
== VEC_PACK_SAT_EXPR
2215 || code
== VEC_PACK_FIX_TRUNC_EXPR
2216 || code
== VEC_WIDEN_LSHIFT_HI_EXPR
2217 || code
== VEC_WIDEN_LSHIFT_LO_EXPR
)
2219 /* We do not know how to scalarize those. */
2223 /* Choose between vector shift/rotate by vector and vector shift/rotate by
2225 if (code
== LSHIFT_EXPR
2226 || code
== RSHIFT_EXPR
2227 || code
== LROTATE_EXPR
2228 || code
== RROTATE_EXPR
)
2232 /* Check whether we have vector <op> {x,x,x,x} where x
2233 could be a scalar variable or a constant. Transform
2234 vector <op> {x,x,x,x} ==> vector <op> scalar. */
2235 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2239 if ((first
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
2241 gimple_assign_set_rhs2 (stmt
, first
);
2247 opv
= optab_for_tree_code (code
, type
, optab_vector
);
2248 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2252 op
= optab_for_tree_code (code
, type
, optab_scalar
);
2254 compute_type
= get_compute_type (code
, op
, type
);
2255 if (compute_type
== type
)
2257 /* The rtl expander will expand vector/scalar as vector/vector
2258 if necessary. Pick one with wider vector type. */
2259 tree compute_vtype
= get_compute_type (code
, opv
, type
);
2260 if (subparts_gt (compute_vtype
, compute_type
))
2262 compute_type
= compute_vtype
;
2267 if (code
== LROTATE_EXPR
|| code
== RROTATE_EXPR
)
2269 if (compute_type
== NULL_TREE
)
2270 compute_type
= get_compute_type (code
, op
, type
);
2271 if (compute_type
== type
)
2273 /* Before splitting vector rotates into scalar rotates,
2274 see if we can't use vector shifts and BIT_IOR_EXPR
2275 instead. For vector by vector rotates we'd also
2276 need to check BIT_AND_EXPR and NEGATE_EXPR, punt there
2277 for now, fold doesn't seem to create such rotates anyway. */
2278 if (compute_type
== TREE_TYPE (type
)
2279 && !VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2281 optab oplv
= vashl_optab
, opl
= ashl_optab
;
2282 optab oprv
= vlshr_optab
, opr
= lshr_optab
, opo
= ior_optab
;
2283 tree compute_lvtype
= get_compute_type (LSHIFT_EXPR
, oplv
, type
);
2284 tree compute_rvtype
= get_compute_type (RSHIFT_EXPR
, oprv
, type
);
2285 tree compute_otype
= get_compute_type (BIT_IOR_EXPR
, opo
, type
);
2286 tree compute_ltype
= get_compute_type (LSHIFT_EXPR
, opl
, type
);
2287 tree compute_rtype
= get_compute_type (RSHIFT_EXPR
, opr
, type
);
2288 /* The rtl expander will expand vector/scalar as vector/vector
2289 if necessary. Pick one with wider vector type. */
2290 if (subparts_gt (compute_lvtype
, compute_ltype
))
2292 compute_ltype
= compute_lvtype
;
2295 if (subparts_gt (compute_rvtype
, compute_rtype
))
2297 compute_rtype
= compute_rvtype
;
2300 /* Pick the narrowest type from LSHIFT_EXPR, RSHIFT_EXPR and
2302 compute_type
= compute_ltype
;
2303 if (subparts_gt (compute_type
, compute_rtype
))
2304 compute_type
= compute_rtype
;
2305 if (subparts_gt (compute_type
, compute_otype
))
2306 compute_type
= compute_otype
;
2307 /* Verify all 3 operations can be performed in that type. */
2308 if (compute_type
!= TREE_TYPE (type
))
2310 if (optab_handler (opl
, TYPE_MODE (compute_type
))
2312 || optab_handler (opr
, TYPE_MODE (compute_type
))
2314 || optab_handler (opo
, TYPE_MODE (compute_type
))
2315 == CODE_FOR_nothing
)
2316 compute_type
= TREE_TYPE (type
);
2322 op
= optab_for_tree_code (code
, type
, optab_default
);
2324 /* Optabs will try converting a negation into a subtraction, so
2325 look for it as well. TODO: negation of floating-point vectors
2326 might be turned into an exclusive OR toggling the sign bit. */
2327 if (op
== unknown_optab
2328 && code
== NEGATE_EXPR
2329 && INTEGRAL_TYPE_P (TREE_TYPE (type
)))
2330 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
2332 if (compute_type
== NULL_TREE
)
2333 compute_type
= get_compute_type (code
, op
, type
);
2334 if (compute_type
== type
)
2337 new_rhs
= expand_vector_operation (gsi
, type
, compute_type
, stmt
, code
,
2340 /* Leave expression untouched for later expansion. */
2341 if (new_rhs
== NULL_TREE
)
2344 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
2345 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
2348 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
2349 way to do it is change expand_vector_operation and its callees to
2350 return a tree_code, RHS1 and RHS2 instead of a tree. */
2351 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
2352 update_stmt (gsi_stmt (*gsi
));
2355 /* Use this to lower vector operations introduced by the vectorizer,
2356 if it may need the bit-twiddling tricks implemented in this file. */
2359 expand_vector_operations (void)
2361 gimple_stmt_iterator gsi
;
2363 bool cfg_changed
= false;
2365 auto_bitmap dce_ssa_names
;
2367 FOR_EACH_BB_FN (bb
, cfun
)
2369 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
2371 expand_vector_operations_1 (&gsi
, dce_ssa_names
);
2372 /* ??? If we do not cleanup EH then we will ICE in
2373 verification. But in reality we have created wrong-code
2374 as we did not properly transition EH info and edges to
2375 the piecewise computations. */
2376 if (maybe_clean_eh_stmt (gsi_stmt (gsi
))
2377 && gimple_purge_dead_eh_edges (bb
))
2379 /* If a .LOOP_DIST_ALIAS call prevailed loops got elided
2380 before vectorization got a chance to get at them. Simply
2381 fold as if loop distribution wasn't performed. */
2382 if (gimple_call_internal_p (gsi_stmt (gsi
), IFN_LOOP_DIST_ALIAS
))
2384 fold_loop_internal_call (gsi_stmt (gsi
), boolean_false_node
);
2390 simple_dce_from_worklist (dce_ssa_names
);
2392 return cfg_changed
? TODO_cleanup_cfg
: 0;
2397 const pass_data pass_data_lower_vector
=
2399 GIMPLE_PASS
, /* type */
2400 "veclower", /* name */
2401 OPTGROUP_VEC
, /* optinfo_flags */
2402 TV_NONE
, /* tv_id */
2403 PROP_cfg
, /* properties_required */
2404 PROP_gimple_lvec
, /* properties_provided */
2405 0, /* properties_destroyed */
2406 0, /* todo_flags_start */
2407 TODO_update_ssa
, /* todo_flags_finish */
2410 class pass_lower_vector
: public gimple_opt_pass
2413 pass_lower_vector (gcc::context
*ctxt
)
2414 : gimple_opt_pass (pass_data_lower_vector
, ctxt
)
2417 /* opt_pass methods: */
2418 bool gate (function
*fun
) final override
2420 return !(fun
->curr_properties
& PROP_gimple_lvec
);
2423 unsigned int execute (function
*) final override
2425 return expand_vector_operations ();
2428 }; // class pass_lower_vector
2433 make_pass_lower_vector (gcc::context
*ctxt
)
2435 return new pass_lower_vector (ctxt
);
2440 const pass_data pass_data_lower_vector_ssa
=
2442 GIMPLE_PASS
, /* type */
2443 "veclower2", /* name */
2444 OPTGROUP_VEC
, /* optinfo_flags */
2445 TV_NONE
, /* tv_id */
2446 PROP_cfg
, /* properties_required */
2447 PROP_gimple_lvec
, /* properties_provided */
2448 0, /* properties_destroyed */
2449 0, /* todo_flags_start */
2451 | TODO_cleanup_cfg
), /* todo_flags_finish */
2454 class pass_lower_vector_ssa
: public gimple_opt_pass
2457 pass_lower_vector_ssa (gcc::context
*ctxt
)
2458 : gimple_opt_pass (pass_data_lower_vector_ssa
, ctxt
)
2461 /* opt_pass methods: */
2462 opt_pass
* clone () final override
2464 return new pass_lower_vector_ssa (m_ctxt
);
2466 unsigned int execute (function
*) final override
2468 return expand_vector_operations ();
2471 }; // class pass_lower_vector_ssa
2476 make_pass_lower_vector_ssa (gcc::context
*ctxt
)
2478 return new pass_lower_vector_ssa (ctxt
);
2481 #include "gt-tree-vect-generic.h"