1 /* Lower vector operations to scalar operations.
2 Copyright (C) 2004-2022 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 GCC is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
27 #include "tree-pass.h"
30 #include "optabs-tree.h"
31 #include "diagnostic.h"
32 #include "fold-const.h"
33 #include "stor-layout.h"
34 #include "langhooks.h"
36 #include "gimple-iterator.h"
37 #include "gimplify-me.h"
40 #include "tree-vector-builder.h"
41 #include "vec-perm-indices.h"
42 #include "insn-config.h"
43 #include "tree-ssa-dce.h"
44 #include "gimple-fold.h"
45 #include "gimple-match.h"
46 #include "recog.h" /* FIXME: for insn_data */
49 /* Build a ternary operation and gimplify it. Emit code before GSI.
50 Return the gimple_val holding the result. */
53 gimplify_build3 (gimple_stmt_iterator
*gsi
, enum tree_code code
,
54 tree type
, tree a
, tree b
, tree c
)
56 location_t loc
= gimple_location (gsi_stmt (*gsi
));
57 gimple_seq stmts
= NULL
;
58 tree ret
= gimple_build (&stmts
, loc
, code
, type
, a
, b
, c
);
59 gsi_insert_seq_before (gsi
, stmts
, GSI_SAME_STMT
);
63 /* Build a binary operation and gimplify it. Emit code before GSI.
64 Return the gimple_val holding the result. */
67 gimplify_build2 (gimple_stmt_iterator
*gsi
, enum tree_code code
,
68 tree type
, tree a
, tree b
)
70 location_t loc
= gimple_location (gsi_stmt (*gsi
));
71 gimple_seq stmts
= NULL
;
72 tree ret
= gimple_build (&stmts
, loc
, code
, type
, a
, b
);
73 gsi_insert_seq_before (gsi
, stmts
, GSI_SAME_STMT
);
77 /* Build a unary operation and gimplify it. Emit code before GSI.
78 Return the gimple_val holding the result. */
81 gimplify_build1 (gimple_stmt_iterator
*gsi
, enum tree_code code
, tree type
,
84 location_t loc
= gimple_location (gsi_stmt (*gsi
));
85 gimple_seq stmts
= NULL
;
86 tree ret
= gimple_build (&stmts
, loc
, code
, type
, a
);
87 gsi_insert_seq_before (gsi
, stmts
, GSI_SAME_STMT
);
92 static void expand_vector_operations_1 (gimple_stmt_iterator
*, bitmap
);
94 /* Return the number of elements in a vector type TYPE that we have
95 already decided needs to be expanded piecewise. We don't support
96 this kind of expansion for variable-length vectors, since we should
97 always check for target support before introducing uses of those. */
99 nunits_for_known_piecewise_op (const_tree type
)
101 return TYPE_VECTOR_SUBPARTS (type
).to_constant ();
104 /* Return true if TYPE1 has more elements than TYPE2, where either
105 type may be a vector or a scalar. */
108 subparts_gt (tree type1
, tree type2
)
110 poly_uint64 n1
= VECTOR_TYPE_P (type1
) ? TYPE_VECTOR_SUBPARTS (type1
) : 1;
111 poly_uint64 n2
= VECTOR_TYPE_P (type2
) ? TYPE_VECTOR_SUBPARTS (type2
) : 1;
112 return known_gt (n1
, n2
);
115 /* Build a constant of type TYPE, made of VALUE's bits replicated
116 every WIDTH bits to fit TYPE's precision. */
118 build_replicated_const (tree type
, unsigned int width
, HOST_WIDE_INT value
)
120 int n
= (TYPE_PRECISION (type
) + HOST_BITS_PER_WIDE_INT
- 1)
121 / HOST_BITS_PER_WIDE_INT
;
122 unsigned HOST_WIDE_INT low
, mask
;
123 HOST_WIDE_INT a
[WIDE_INT_MAX_ELTS
];
126 gcc_assert (n
&& n
<= WIDE_INT_MAX_ELTS
);
128 if (width
== HOST_BITS_PER_WIDE_INT
)
132 mask
= ((HOST_WIDE_INT
)1 << width
) - 1;
133 low
= (unsigned HOST_WIDE_INT
) ~0 / mask
* (value
& mask
);
136 for (i
= 0; i
< n
; i
++)
139 gcc_assert (TYPE_PRECISION (type
) <= MAX_BITSIZE_MODE_ANY_INT
);
140 return wide_int_to_tree
141 (type
, wide_int::from_array (a
, n
, TYPE_PRECISION (type
)));
144 static GTY(()) tree vector_inner_type
;
145 static GTY(()) tree vector_last_type
;
146 static GTY(()) int vector_last_nunits
;
148 /* Return a suitable vector types made of SUBPARTS units each of mode
149 "word_mode" (the global variable). */
151 build_word_mode_vector_type (int nunits
)
153 if (!vector_inner_type
)
154 vector_inner_type
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
155 else if (vector_last_nunits
== nunits
)
157 gcc_assert (TREE_CODE (vector_last_type
) == VECTOR_TYPE
);
158 return vector_last_type
;
161 vector_last_nunits
= nunits
;
162 vector_last_type
= build_vector_type (vector_inner_type
, nunits
);
163 return vector_last_type
;
166 typedef tree (*elem_op_func
) (gimple_stmt_iterator
*,
167 tree
, tree
, tree
, tree
, tree
, enum tree_code
,
170 /* Extract the vector element of type TYPE at BITPOS with BITSIZE from T
174 tree_vec_extract (gimple_stmt_iterator
*gsi
, tree type
,
175 tree t
, tree bitsize
, tree bitpos
)
177 /* We're using the resimplify API and maybe_push_res_to_seq to
178 simplify the BIT_FIELD_REF but restrict the simplification to
179 a single stmt while at the same time following SSA edges for
180 simplification with already emitted CTORs. */
182 opr
.set_op (BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
183 opr
.resimplify (NULL
, follow_all_ssa_edges
);
184 gimple_seq stmts
= NULL
;
185 tree res
= maybe_push_res_to_seq (&opr
, &stmts
);
187 gsi_insert_seq_before (gsi
, stmts
, GSI_SAME_STMT
);
192 do_unop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
193 tree b ATTRIBUTE_UNUSED
, tree bitpos
, tree bitsize
,
194 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
196 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
197 return gimplify_build1 (gsi
, code
, inner_type
, a
);
201 do_binop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
202 tree bitpos
, tree bitsize
, enum tree_code code
,
203 tree type ATTRIBUTE_UNUSED
)
205 if (TREE_CODE (TREE_TYPE (a
)) == VECTOR_TYPE
)
206 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
207 if (TREE_CODE (TREE_TYPE (b
)) == VECTOR_TYPE
)
208 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
209 return gimplify_build2 (gsi
, code
, inner_type
, a
, b
);
212 /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
214 INNER_TYPE is the type of A and B elements
216 returned expression is of signed integer type with the
217 size equal to the size of INNER_TYPE. */
219 do_compare (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
220 tree bitpos
, tree bitsize
, enum tree_code code
, tree type
)
222 tree stype
= TREE_TYPE (type
);
223 tree cst_false
= build_zero_cst (stype
);
224 tree cst_true
= build_all_ones_cst (stype
);
227 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
228 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
230 cmp
= build2 (code
, boolean_type_node
, a
, b
);
231 return gimplify_build3 (gsi
, COND_EXPR
, stype
, cmp
, cst_true
, cst_false
);
234 /* Expand vector addition to scalars. This does bit twiddling
235 in order to increase parallelism:
237 a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
240 a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
241 (a ^ ~b) & 0x80808080
243 -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
245 This optimization should be done only if 4 vector items or more
248 do_plus_minus (gimple_stmt_iterator
*gsi
, tree word_type
, tree a
, tree b
,
249 tree bitpos ATTRIBUTE_UNUSED
, tree bitsize ATTRIBUTE_UNUSED
,
250 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
252 unsigned int width
= vector_element_bits (TREE_TYPE (a
));
253 tree inner_type
= TREE_TYPE (TREE_TYPE (a
));
254 unsigned HOST_WIDE_INT max
;
255 tree low_bits
, high_bits
, a_low
, b_low
, result_low
, signs
;
257 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
258 low_bits
= build_replicated_const (word_type
, width
, max
>> 1);
259 high_bits
= build_replicated_const (word_type
, width
, max
& ~(max
>> 1));
261 a
= tree_vec_extract (gsi
, word_type
, a
, bitsize
, bitpos
);
262 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
264 signs
= gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, a
, b
);
265 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
266 if (code
== PLUS_EXPR
)
267 a_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, a
, low_bits
);
270 a_low
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, word_type
, a
, high_bits
);
271 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, signs
);
274 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
275 result_low
= gimplify_build2 (gsi
, code
, word_type
, a_low
, b_low
);
276 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
280 do_negate (gimple_stmt_iterator
*gsi
, tree word_type
, tree b
,
281 tree unused ATTRIBUTE_UNUSED
, tree bitpos ATTRIBUTE_UNUSED
,
282 tree bitsize ATTRIBUTE_UNUSED
,
283 enum tree_code code ATTRIBUTE_UNUSED
,
284 tree type ATTRIBUTE_UNUSED
)
286 unsigned int width
= vector_element_bits (TREE_TYPE (b
));
287 tree inner_type
= TREE_TYPE (TREE_TYPE (b
));
289 tree low_bits
, high_bits
, b_low
, result_low
, signs
;
291 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
292 low_bits
= build_replicated_const (word_type
, width
, max
>> 1);
293 high_bits
= build_replicated_const (word_type
, width
, max
& ~(max
>> 1));
295 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
297 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
298 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, b
);
299 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
300 result_low
= gimplify_build2 (gsi
, MINUS_EXPR
, word_type
, high_bits
, b_low
);
301 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
304 /* Expand a vector operation to scalars, by using many operations
305 whose type is the vector type's inner type. */
307 expand_vector_piecewise (gimple_stmt_iterator
*gsi
, elem_op_func f
,
308 tree type
, tree inner_type
,
309 tree a
, tree b
, enum tree_code code
,
310 bool parallel_p
, tree ret_type
= NULL_TREE
)
312 vec
<constructor_elt
, va_gc
> *v
;
313 tree part_width
= TYPE_SIZE (inner_type
);
314 tree index
= bitsize_int (0);
315 int nunits
= nunits_for_known_piecewise_op (type
);
316 int delta
= tree_to_uhwi (part_width
) / vector_element_bits (type
);
318 location_t loc
= gimple_location (gsi_stmt (*gsi
));
320 if (ret_type
|| !parallel_p
)
321 warning_at (loc
, OPT_Wvector_operation_performance
,
322 "vector operation will be expanded piecewise");
324 warning_at (loc
, OPT_Wvector_operation_performance
,
325 "vector operation will be expanded in parallel");
329 vec_alloc (v
, (nunits
+ delta
- 1) / delta
);
330 bool constant_p
= true;
331 for (i
= 0; i
< nunits
;
332 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
334 tree result
= f (gsi
, inner_type
, a
, b
, index
, part_width
, code
,
336 if (!CONSTANT_CLASS_P (result
))
338 constructor_elt ce
= {NULL_TREE
, result
};
343 return build_vector_from_ctor (ret_type
, v
);
345 return build_constructor (ret_type
, v
);
348 /* Expand a vector operation to scalars with the freedom to use
349 a scalar integer type, or to use a different size for the items
350 in the vector type. */
352 expand_vector_parallel (gimple_stmt_iterator
*gsi
, elem_op_func f
, tree type
,
353 tree a
, tree b
, enum tree_code code
)
355 tree result
, compute_type
;
356 int n_words
= tree_to_uhwi (TYPE_SIZE_UNIT (type
)) / UNITS_PER_WORD
;
357 location_t loc
= gimple_location (gsi_stmt (*gsi
));
359 /* We have three strategies. If the type is already correct, just do
360 the operation an element at a time. Else, if the vector is wider than
361 one word, do it a word at a time; finally, if the vector is smaller
362 than one word, do it as a scalar. */
363 if (TYPE_MODE (TREE_TYPE (type
)) == word_mode
)
364 return expand_vector_piecewise (gsi
, f
,
365 type
, TREE_TYPE (type
),
367 else if (n_words
> 1)
369 tree word_type
= build_word_mode_vector_type (n_words
);
370 result
= expand_vector_piecewise (gsi
, f
,
371 word_type
, TREE_TYPE (word_type
),
373 result
= force_gimple_operand_gsi (gsi
, result
, true, NULL
, true,
378 /* Use a single scalar operation with a mode no wider than word_mode. */
380 = int_mode_for_size (tree_to_uhwi (TYPE_SIZE (type
)), 0).require ();
381 compute_type
= lang_hooks
.types
.type_for_mode (mode
, 1);
382 result
= f (gsi
, compute_type
, a
, b
, bitsize_zero_node
,
383 TYPE_SIZE (compute_type
), code
, type
);
384 warning_at (loc
, OPT_Wvector_operation_performance
,
385 "vector operation will be expanded with a "
386 "single scalar operation");
392 /* Expand a vector operation to scalars; for integer types we can use
393 special bit twiddling tricks to do the sums a word at a time, using
394 function F_PARALLEL instead of F. These tricks are done only if
395 they can process at least four items, that is, only if the vector
396 holds at least four items and if a word can hold four items. */
398 expand_vector_addition (gimple_stmt_iterator
*gsi
,
399 elem_op_func f
, elem_op_func f_parallel
,
400 tree type
, tree a
, tree b
, enum tree_code code
)
402 int parts_per_word
= BITS_PER_WORD
/ vector_element_bits (type
);
404 if (INTEGRAL_TYPE_P (TREE_TYPE (type
))
405 && parts_per_word
>= 4
406 && nunits_for_known_piecewise_op (type
) >= 4)
407 return expand_vector_parallel (gsi
, f_parallel
,
410 return expand_vector_piecewise (gsi
, f
,
411 type
, TREE_TYPE (type
),
416 expand_vector_condition (gimple_stmt_iterator
*gsi
, bitmap dce_ssa_names
);
418 /* Try to expand vector comparison expression OP0 CODE OP1 by
419 querying optab if the following expression:
420 VEC_COND_EXPR< OP0 CODE OP1, {-1,...}, {0,...}>
423 expand_vector_comparison (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
424 tree op1
, enum tree_code code
,
425 bitmap dce_ssa_names
)
427 tree lhs
= gimple_assign_lhs (gsi_stmt (*gsi
));
429 imm_use_iterator iterator
;
430 bool vec_cond_expr_only
= true;
432 /* As seen in PR95830, we should not expand comparisons that are only
433 feeding a VEC_COND_EXPR statement. */
434 auto_vec
<gimple
*> uses
;
435 FOR_EACH_IMM_USE_FAST (use_p
, iterator
, lhs
)
436 uses
.safe_push (USE_STMT (use_p
));
438 for (unsigned i
= 0; i
< uses
.length (); i
++)
440 gassign
*use
= dyn_cast
<gassign
*> (uses
[i
]);
442 && gimple_assign_rhs_code (use
) == VEC_COND_EXPR
443 && gimple_assign_rhs1 (use
) == lhs
)
445 gimple_stmt_iterator it
= gsi_for_stmt (use
);
446 if (!expand_vector_condition (&it
, dce_ssa_names
))
448 vec_cond_expr_only
= false;
454 vec_cond_expr_only
= false;
459 if (!uses
.is_empty () && vec_cond_expr_only
)
463 if (!expand_vec_cmp_expr_p (TREE_TYPE (op0
), type
, code
))
465 if (VECTOR_BOOLEAN_TYPE_P (type
)
466 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
467 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type
)),
468 TYPE_VECTOR_SUBPARTS (type
)
469 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
470 (TREE_TYPE (type
)))))
472 tree inner_type
= TREE_TYPE (TREE_TYPE (op0
));
473 tree part_width
= vector_element_bits_tree (TREE_TYPE (op0
));
474 tree index
= bitsize_int (0);
475 int nunits
= nunits_for_known_piecewise_op (TREE_TYPE (op0
));
476 int prec
= GET_MODE_PRECISION (SCALAR_TYPE_MODE (type
));
477 tree ret_type
= build_nonstandard_integer_type (prec
, 1);
478 tree ret_inner_type
= boolean_type_node
;
480 location_t loc
= gimple_location (gsi_stmt (*gsi
));
481 t
= build_zero_cst (ret_type
);
483 if (TYPE_PRECISION (ret_inner_type
) != 1)
484 ret_inner_type
= build_nonstandard_integer_type (1, 1);
485 warning_at (loc
, OPT_Wvector_operation_performance
,
486 "vector operation will be expanded piecewise");
487 for (i
= 0; i
< nunits
;
488 i
++, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
490 tree a
= tree_vec_extract (gsi
, inner_type
, op0
, part_width
,
492 tree b
= tree_vec_extract (gsi
, inner_type
, op1
, part_width
,
494 tree result
= gimplify_build2 (gsi
, code
, ret_inner_type
, a
, b
);
495 t
= gimplify_build3 (gsi
, BIT_INSERT_EXPR
, ret_type
, t
, result
,
498 t
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, type
, t
);
501 t
= expand_vector_piecewise (gsi
, do_compare
, type
,
502 TREE_TYPE (TREE_TYPE (op0
)), op0
, op1
,
511 /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
512 of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
513 the result if successful, otherwise return NULL_TREE. */
515 add_rshift (gimple_stmt_iterator
*gsi
, tree type
, tree op0
, int *shiftcnts
)
518 unsigned int i
, nunits
= nunits_for_known_piecewise_op (type
);
519 bool scalar_shift
= true;
521 for (i
= 1; i
< nunits
; i
++)
523 if (shiftcnts
[i
] != shiftcnts
[0])
524 scalar_shift
= false;
527 if (scalar_shift
&& shiftcnts
[0] == 0)
532 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_scalar
);
533 if (op
!= unknown_optab
534 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
535 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
536 build_int_cst (NULL_TREE
, shiftcnts
[0]));
539 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
540 if (op
!= unknown_optab
541 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
543 tree_vector_builder
vec (type
, nunits
, 1);
544 for (i
= 0; i
< nunits
; i
++)
545 vec
.quick_push (build_int_cst (TREE_TYPE (type
), shiftcnts
[i
]));
546 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
, vec
.build ());
552 /* Try to expand integer vector division by constant using
553 widening multiply, shifts and additions. */
555 expand_vector_divmod (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
556 tree op1
, enum tree_code code
)
558 bool use_pow2
= true;
559 bool has_vector_shift
= true;
560 bool use_abs_op1
= false;
561 int mode
= -1, this_mode
;
562 int pre_shift
= -1, post_shift
;
563 unsigned int nunits
= nunits_for_known_piecewise_op (type
);
564 int *shifts
= XALLOCAVEC (int, nunits
* 4);
565 int *pre_shifts
= shifts
+ nunits
;
566 int *post_shifts
= pre_shifts
+ nunits
;
567 int *shift_temps
= post_shifts
+ nunits
;
568 unsigned HOST_WIDE_INT
*mulc
= XALLOCAVEC (unsigned HOST_WIDE_INT
, nunits
);
569 int prec
= TYPE_PRECISION (TREE_TYPE (type
));
572 signop sign_p
= TYPE_SIGN (TREE_TYPE (type
));
573 unsigned HOST_WIDE_INT mask
= GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type
)));
574 tree cur_op
, mulcst
, tem
;
577 if (prec
> HOST_BITS_PER_WIDE_INT
)
580 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
581 if (op
== unknown_optab
582 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
583 has_vector_shift
= false;
585 /* Analysis phase. Determine if all op1 elements are either power
586 of two and it is possible to expand it using shifts (or for remainder
587 using masking). Additionally compute the multiplicative constants
588 and pre and post shifts if the division is to be expanded using
589 widening or high part multiplication plus shifts. */
590 for (i
= 0; i
< nunits
; i
++)
592 tree cst
= VECTOR_CST_ELT (op1
, i
);
593 unsigned HOST_WIDE_INT ml
;
595 if (TREE_CODE (cst
) != INTEGER_CST
|| integer_zerop (cst
))
601 && (!integer_pow2p (cst
) || tree_int_cst_sgn (cst
) != 1))
605 shifts
[i
] = tree_log2 (cst
);
606 if (shifts
[i
] != shifts
[0]
607 && code
== TRUNC_DIV_EXPR
608 && !has_vector_shift
)
613 if (sign_p
== UNSIGNED
)
615 unsigned HOST_WIDE_INT mh
;
616 unsigned HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
) & mask
;
618 if (d
>= (HOST_WIDE_INT_1U
<< (prec
- 1)))
619 /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
628 /* Find a suitable multiplier and right shift count
629 instead of multiplying with D. */
630 mh
= choose_multiplier (d
, prec
, prec
, &ml
, &post_shift
, &dummy_int
);
632 /* If the suggested multiplier is more than SIZE bits, we can
633 do better for even divisors, using an initial right shift. */
634 if ((mh
!= 0 && (d
& 1) == 0)
635 || (!has_vector_shift
&& pre_shift
!= -1))
637 if (has_vector_shift
)
638 pre_shift
= ctz_or_zero (d
);
639 else if (pre_shift
== -1)
642 for (j
= 0; j
< nunits
; j
++)
644 tree cst2
= VECTOR_CST_ELT (op1
, j
);
645 unsigned HOST_WIDE_INT d2
;
648 if (!tree_fits_uhwi_p (cst2
))
650 d2
= tree_to_uhwi (cst2
) & mask
;
653 this_pre_shift
= floor_log2 (d2
& -d2
);
654 if (pre_shift
== -1 || this_pre_shift
< pre_shift
)
655 pre_shift
= this_pre_shift
;
657 if (i
!= 0 && pre_shift
!= 0)
667 if ((d
>> pre_shift
) <= 1)
672 mh
= choose_multiplier (d
>> pre_shift
, prec
,
674 &ml
, &post_shift
, &dummy_int
);
676 pre_shifts
[i
] = pre_shift
;
686 HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
);
687 unsigned HOST_WIDE_INT abs_d
;
692 /* Since d might be INT_MIN, we have to cast to
693 unsigned HOST_WIDE_INT before negating to avoid
694 undefined signed overflow. */
696 ? (unsigned HOST_WIDE_INT
) d
697 : - (unsigned HOST_WIDE_INT
) d
);
699 /* n rem d = n rem -d */
700 if (code
== TRUNC_MOD_EXPR
&& d
< 0)
705 if (abs_d
== HOST_WIDE_INT_1U
<< (prec
- 1))
707 /* This case is not handled correctly below. */
717 choose_multiplier (abs_d
, prec
, prec
- 1, &ml
,
718 &post_shift
, &dummy_int
);
719 if (ml
>= HOST_WIDE_INT_1U
<< (prec
- 1))
721 this_mode
= 4 + (d
< 0);
722 ml
|= HOST_WIDE_INT_M1U
<< (prec
- 1);
725 this_mode
= 2 + (d
< 0);
728 post_shifts
[i
] = post_shift
;
729 if ((i
&& !has_vector_shift
&& post_shifts
[0] != post_shift
)
730 || post_shift
>= prec
731 || pre_shifts
[i
] >= prec
)
736 else if (mode
!= this_mode
)
742 tree addend
= NULL_TREE
;
743 if (sign_p
== SIGNED
)
747 /* Both division and remainder sequences need
748 op0 < 0 ? mask : 0 computed. It can be either computed as
749 (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
750 if none of the shifts is 0, or as the conditional. */
751 for (i
= 0; i
< nunits
; i
++)
755 = build_vector_type (build_nonstandard_integer_type (prec
, 1),
757 if (i
== nunits
&& TYPE_MODE (uns_type
) == TYPE_MODE (type
))
759 for (i
= 0; i
< nunits
; i
++)
760 shift_temps
[i
] = prec
- 1;
761 cur_op
= add_rshift (gsi
, type
, op0
, shift_temps
);
762 if (cur_op
!= NULL_TREE
)
764 cur_op
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
766 for (i
= 0; i
< nunits
; i
++)
767 shift_temps
[i
] = prec
- shifts
[i
];
768 cur_op
= add_rshift (gsi
, uns_type
, cur_op
, shift_temps
);
769 if (cur_op
!= NULL_TREE
)
770 addend
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
774 if (addend
== NULL_TREE
775 && expand_vec_cond_expr_p (type
, type
, LT_EXPR
))
777 tree zero
, cst
, mask_type
, mask
;
780 mask_type
= truth_type_for (type
);
781 zero
= build_zero_cst (type
);
782 mask
= make_ssa_name (mask_type
);
783 cond
= gimple_build_assign (mask
, LT_EXPR
, op0
, zero
);
784 gsi_insert_before (gsi
, cond
, GSI_SAME_STMT
);
785 tree_vector_builder
vec (type
, nunits
, 1);
786 for (i
= 0; i
< nunits
; i
++)
787 vec
.quick_push (build_int_cst (TREE_TYPE (type
),
791 addend
= make_ssa_name (type
);
793 = gimple_build_assign (addend
, VEC_COND_EXPR
, mask
, cst
, zero
);
794 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
797 if (code
== TRUNC_DIV_EXPR
)
799 if (sign_p
== UNSIGNED
)
801 /* q = op0 >> shift; */
802 cur_op
= add_rshift (gsi
, type
, op0
, shifts
);
803 if (cur_op
!= NULL_TREE
)
806 else if (addend
!= NULL_TREE
)
808 /* t1 = op0 + addend;
810 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
811 if (op
!= unknown_optab
812 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
814 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
, addend
);
815 cur_op
= add_rshift (gsi
, type
, cur_op
, shifts
);
816 if (cur_op
!= NULL_TREE
)
824 tree_vector_builder
vec (type
, nunits
, 1);
825 for (i
= 0; i
< nunits
; i
++)
826 vec
.quick_push (build_int_cst (TREE_TYPE (type
),
830 op
= optab_for_tree_code (BIT_AND_EXPR
, type
, optab_default
);
831 if (op
!= unknown_optab
832 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
834 if (sign_p
== UNSIGNED
)
835 /* r = op0 & mask; */
836 return gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, op0
, mask
);
837 else if (addend
!= NULL_TREE
)
839 /* t1 = op0 + addend;
842 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
843 if (op
!= unknown_optab
844 && optab_handler (op
, TYPE_MODE (type
))
847 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
,
849 cur_op
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
,
851 op
= optab_for_tree_code (MINUS_EXPR
, type
,
853 if (op
!= unknown_optab
854 && optab_handler (op
, TYPE_MODE (type
))
856 return gimplify_build2 (gsi
, MINUS_EXPR
, type
,
864 if (mode
== -2 || BYTES_BIG_ENDIAN
!= WORDS_BIG_ENDIAN
)
867 if (!can_mult_highpart_p (TYPE_MODE (type
), TYPE_UNSIGNED (type
)))
875 gcc_assert (sign_p
== UNSIGNED
);
876 /* t1 = oprnd0 >> pre_shift;
878 q = t2 >> post_shift; */
879 cur_op
= add_rshift (gsi
, type
, cur_op
, pre_shifts
);
880 if (cur_op
== NULL_TREE
)
884 gcc_assert (sign_p
== UNSIGNED
);
885 for (i
= 0; i
< nunits
; i
++)
895 gcc_assert (sign_p
== SIGNED
);
896 for (i
= 0; i
< nunits
; i
++)
897 shift_temps
[i
] = prec
- 1;
903 tree_vector_builder
vec (type
, nunits
, 1);
904 for (i
= 0; i
< nunits
; i
++)
905 vec
.quick_push (build_int_cst (TREE_TYPE (type
), mulc
[i
]));
906 mulcst
= vec
.build ();
908 cur_op
= gimplify_build2 (gsi
, MULT_HIGHPART_EXPR
, type
, cur_op
, mulcst
);
913 /* t1 = oprnd0 >> pre_shift;
915 q = t2 >> post_shift; */
916 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
919 /* t1 = oprnd0 h* ml;
923 q = t4 >> (post_shift - 1); */
924 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
925 if (op
== unknown_optab
926 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
928 tem
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, cur_op
);
929 tem
= add_rshift (gsi
, type
, tem
, shift_temps
);
930 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
931 if (op
== unknown_optab
932 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
934 tem
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, tem
);
935 cur_op
= add_rshift (gsi
, type
, tem
, post_shifts
);
936 if (cur_op
== NULL_TREE
)
943 /* t1 = oprnd0 h* ml;
944 t2 = t1; [ iff (mode & 2) != 0 ]
945 t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
946 t3 = t2 >> post_shift;
947 t4 = oprnd0 >> (prec - 1);
948 q = t3 - t4; [ iff (mode & 1) == 0 ]
949 q = t4 - t3; [ iff (mode & 1) != 0 ] */
952 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
953 if (op
== unknown_optab
954 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
956 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, op0
);
958 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
959 if (cur_op
== NULL_TREE
)
961 tem
= add_rshift (gsi
, type
, op0
, shift_temps
);
962 if (tem
== NULL_TREE
)
964 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
965 if (op
== unknown_optab
966 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
969 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, cur_op
, tem
);
971 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, tem
, cur_op
);
977 if (code
== TRUNC_DIV_EXPR
)
980 /* We divided. Now finish by:
983 op
= optab_for_tree_code (MULT_EXPR
, type
, optab_default
);
984 if (op
== unknown_optab
985 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
989 tree_vector_builder elts
;
990 if (!elts
.new_unary_operation (type
, op1
, false))
992 unsigned int count
= elts
.encoded_nelts ();
993 for (unsigned int i
= 0; i
< count
; ++i
)
995 tree elem1
= VECTOR_CST_ELT (op1
, i
);
997 tree elt
= const_unop (ABS_EXPR
, TREE_TYPE (elem1
), elem1
);
998 if (elt
== NULL_TREE
)
1000 elts
.quick_push (elt
);
1002 op1
= elts
.build ();
1004 tem
= gimplify_build2 (gsi
, MULT_EXPR
, type
, cur_op
, op1
);
1005 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
1006 if (op
== unknown_optab
1007 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
1009 return gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, tem
);
1012 /* Expand a vector condition to scalars, by using many conditions
1013 on the vector's elements. */
1016 expand_vector_condition (gimple_stmt_iterator
*gsi
, bitmap dce_ssa_names
)
1018 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1019 tree type
= TREE_TYPE (gimple_assign_lhs (stmt
));
1020 tree a
= gimple_assign_rhs1 (stmt
);
1022 tree a2
= NULL_TREE
;
1023 bool a_is_comparison
= false;
1024 bool a_is_scalar_bitmask
= false;
1025 tree b
= gimple_assign_rhs2 (stmt
);
1026 tree c
= gimple_assign_rhs3 (stmt
);
1027 vec
<constructor_elt
, va_gc
> *v
;
1029 tree inner_type
= TREE_TYPE (type
);
1030 tree width
= vector_element_bits_tree (type
);
1031 tree cond_type
= TREE_TYPE (TREE_TYPE (a
));
1032 tree comp_inner_type
= cond_type
;
1033 tree index
= bitsize_int (0);
1034 tree comp_width
= width
;
1035 tree comp_index
= index
;
1036 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1037 tree_code code
= TREE_CODE (a
);
1038 gassign
*assign
= NULL
;
1040 if (code
== SSA_NAME
)
1042 assign
= dyn_cast
<gassign
*> (SSA_NAME_DEF_STMT (a
));
1044 && TREE_CODE_CLASS (gimple_assign_rhs_code (assign
)) == tcc_comparison
)
1046 a_is_comparison
= true;
1047 a1
= gimple_assign_rhs1 (assign
);
1048 a2
= gimple_assign_rhs2 (assign
);
1049 code
= gimple_assign_rhs_code (assign
);
1050 comp_inner_type
= TREE_TYPE (TREE_TYPE (a1
));
1051 comp_width
= vector_element_bits_tree (TREE_TYPE (a1
));
1055 if (expand_vec_cond_expr_p (type
, TREE_TYPE (a1
), code
)
1056 || (integer_all_onesp (b
) && integer_zerop (c
)
1057 && expand_vec_cmp_expr_p (type
, TREE_TYPE (a1
), code
)))
1059 gcc_assert (TREE_CODE (a
) == SSA_NAME
|| TREE_CODE (a
) == VECTOR_CST
);
1063 /* Handle vector boolean types with bitmasks. If there is a comparison
1064 and we can expand the comparison into the vector boolean bitmask,
1065 or otherwise if it is compatible with type, we can transform
1066 vbfld_1 = x_2 < y_3 ? vbfld_4 : vbfld_5;
1069 tmp_7 = tmp_6 & vbfld_4;
1071 tmp_9 = tmp_8 & vbfld_5;
1072 vbfld_1 = tmp_7 | tmp_9;
1073 Similarly for vbfld_10 instead of x_2 < y_3. */
1074 if (VECTOR_BOOLEAN_TYPE_P (type
)
1075 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
1076 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type
)),
1077 TYPE_VECTOR_SUBPARTS (type
)
1078 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (type
))))
1080 ? useless_type_conversion_p (type
, TREE_TYPE (a
))
1081 : expand_vec_cmp_expr_p (TREE_TYPE (a1
), type
, TREE_CODE (a
))))
1083 if (a_is_comparison
)
1084 a
= gimplify_build2 (gsi
, code
, type
, a1
, a2
);
1085 a1
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, a
, b
);
1086 a2
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, type
, a
);
1087 a2
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, a2
, c
);
1088 a
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, type
, a1
, a2
);
1089 gimple_assign_set_rhs_from_tree (gsi
, a
);
1090 update_stmt (gsi_stmt (*gsi
));
1094 /* TODO: try and find a smaller vector type. */
1096 warning_at (loc
, OPT_Wvector_operation_performance
,
1097 "vector condition will be expanded piecewise");
1099 if (!a_is_comparison
1100 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
))
1101 && SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (a
)))
1102 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (a
))),
1103 TYPE_VECTOR_SUBPARTS (TREE_TYPE (a
))
1104 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
1105 (TREE_TYPE (TREE_TYPE (a
))))))
1107 a_is_scalar_bitmask
= true;
1108 int prec
= GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (a
)));
1109 tree atype
= build_nonstandard_integer_type (prec
, 1);
1110 a
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, atype
, a
);
1113 int nunits
= nunits_for_known_piecewise_op (type
);
1114 vec_alloc (v
, nunits
);
1115 bool constant_p
= true;
1116 for (int i
= 0; i
< nunits
; i
++)
1119 tree bb
= tree_vec_extract (gsi
, inner_type
, b
, width
, index
);
1120 tree cc
= tree_vec_extract (gsi
, inner_type
, c
, width
, index
);
1121 if (a_is_comparison
)
1123 tree aa1
= tree_vec_extract (gsi
, comp_inner_type
, a1
,
1124 comp_width
, comp_index
);
1125 tree aa2
= tree_vec_extract (gsi
, comp_inner_type
, a2
,
1126 comp_width
, comp_index
);
1127 aa
= build2 (code
, cond_type
, aa1
, aa2
);
1129 else if (a_is_scalar_bitmask
)
1131 wide_int w
= wi::set_bit_in_zero (i
, TYPE_PRECISION (TREE_TYPE (a
)));
1132 result
= gimplify_build2 (gsi
, BIT_AND_EXPR
, TREE_TYPE (a
),
1133 a
, wide_int_to_tree (TREE_TYPE (a
), w
));
1134 aa
= build2 (NE_EXPR
, boolean_type_node
, result
,
1135 build_zero_cst (TREE_TYPE (a
)));
1138 aa
= tree_vec_extract (gsi
, cond_type
, a
, width
, index
);
1139 result
= gimplify_build3 (gsi
, COND_EXPR
, inner_type
, aa
, bb
, cc
);
1140 if (!CONSTANT_CLASS_P (result
))
1142 constructor_elt ce
= {NULL_TREE
, result
};
1144 index
= int_const_binop (PLUS_EXPR
, index
, width
);
1145 if (width
== comp_width
)
1148 comp_index
= int_const_binop (PLUS_EXPR
, comp_index
, comp_width
);
1152 constr
= build_vector_from_ctor (type
, v
);
1154 constr
= build_constructor (type
, v
);
1155 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1156 update_stmt (gsi_stmt (*gsi
));
1158 if (a_is_comparison
)
1159 bitmap_set_bit (dce_ssa_names
,
1160 SSA_NAME_VERSION (gimple_assign_lhs (assign
)));
1166 expand_vector_operation (gimple_stmt_iterator
*gsi
, tree type
, tree compute_type
,
1167 gassign
*assign
, enum tree_code code
,
1168 bitmap dce_ssa_names
)
1170 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1172 /* If the compute mode is not a vector mode (hence we are not decomposing
1173 a BLKmode vector to smaller, hardware-supported vectors), we may want
1174 to expand the operations in parallel. */
1175 if (!VECTOR_MODE_P (compute_mode
))
1180 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
1181 return expand_vector_addition (gsi
, do_binop
, do_plus_minus
, type
,
1182 gimple_assign_rhs1 (assign
),
1183 gimple_assign_rhs2 (assign
), code
);
1187 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
1188 return expand_vector_addition (gsi
, do_unop
, do_negate
, type
,
1189 gimple_assign_rhs1 (assign
),
1196 return expand_vector_parallel (gsi
, do_binop
, type
,
1197 gimple_assign_rhs1 (assign
),
1198 gimple_assign_rhs2 (assign
), code
);
1201 return expand_vector_parallel (gsi
, do_unop
, type
,
1202 gimple_assign_rhs1 (assign
),
1217 case UNORDERED_EXPR
:
1219 tree rhs1
= gimple_assign_rhs1 (assign
);
1220 tree rhs2
= gimple_assign_rhs2 (assign
);
1222 return expand_vector_comparison (gsi
, type
, rhs1
, rhs2
, code
,
1226 case TRUNC_DIV_EXPR
:
1227 case TRUNC_MOD_EXPR
:
1229 tree rhs1
= gimple_assign_rhs1 (assign
);
1230 tree rhs2
= gimple_assign_rhs2 (assign
);
1234 || !VECTOR_INTEGER_TYPE_P (type
)
1235 || TREE_CODE (rhs2
) != VECTOR_CST
1236 || !VECTOR_MODE_P (TYPE_MODE (type
)))
1239 ret
= expand_vector_divmod (gsi
, type
, rhs1
, rhs2
, code
);
1240 if (ret
!= NULL_TREE
)
1249 if (TREE_CODE_CLASS (code
) == tcc_unary
)
1250 return expand_vector_piecewise (gsi
, do_unop
, type
, compute_type
,
1251 gimple_assign_rhs1 (assign
),
1252 NULL_TREE
, code
, false);
1254 return expand_vector_piecewise (gsi
, do_binop
, type
, compute_type
,
1255 gimple_assign_rhs1 (assign
),
1256 gimple_assign_rhs2 (assign
), code
, false);
1260 a_5 = { b_7, b_7 + 3, b_7 + 6, b_7 + 9 };
1262 _9 = { b_7, b_7, b_7, b_7 };
1263 a_5 = _9 + { 0, 3, 6, 9 };
1264 because vector splat operation is usually more efficient
1265 than piecewise initialization of the vector. */
1268 optimize_vector_constructor (gimple_stmt_iterator
*gsi
)
1270 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1271 tree lhs
= gimple_assign_lhs (stmt
);
1272 tree rhs
= gimple_assign_rhs1 (stmt
);
1273 tree type
= TREE_TYPE (rhs
);
1275 unsigned HOST_WIDE_INT nelts
;
1276 bool all_same
= true;
1277 constructor_elt
*elt
;
1279 tree base
= NULL_TREE
;
1282 if (!TYPE_VECTOR_SUBPARTS (type
).is_constant (&nelts
)
1284 || CONSTRUCTOR_NELTS (rhs
) != nelts
)
1286 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
1287 if (op
== unknown_optab
1288 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
1290 FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (rhs
), i
, elt
)
1291 if (TREE_CODE (elt
->value
) != SSA_NAME
1292 || TREE_CODE (TREE_TYPE (elt
->value
)) == VECTOR_TYPE
)
1296 tree this_base
= elt
->value
;
1297 if (this_base
!= CONSTRUCTOR_ELT (rhs
, 0)->value
)
1299 for (j
= 0; j
< nelts
+ 1; j
++)
1301 g
= SSA_NAME_DEF_STMT (this_base
);
1302 if (is_gimple_assign (g
)
1303 && gimple_assign_rhs_code (g
) == PLUS_EXPR
1304 && TREE_CODE (gimple_assign_rhs2 (g
)) == INTEGER_CST
1305 && TREE_CODE (gimple_assign_rhs1 (g
)) == SSA_NAME
1306 && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g
)))
1307 this_base
= gimple_assign_rhs1 (g
);
1313 else if (this_base
!= base
)
1318 tree_vector_builder
cst (type
, nelts
, 1);
1319 for (i
= 0; i
< nelts
; i
++)
1321 tree this_base
= CONSTRUCTOR_ELT (rhs
, i
)->value
;
1322 tree elt
= build_zero_cst (TREE_TYPE (base
));
1323 while (this_base
!= base
)
1325 g
= SSA_NAME_DEF_STMT (this_base
);
1326 elt
= fold_binary (PLUS_EXPR
, TREE_TYPE (base
),
1327 elt
, gimple_assign_rhs2 (g
));
1328 if (elt
== NULL_TREE
1329 || TREE_CODE (elt
) != INTEGER_CST
1330 || TREE_OVERFLOW (elt
))
1332 this_base
= gimple_assign_rhs1 (g
);
1334 cst
.quick_push (elt
);
1336 for (i
= 0; i
< nelts
; i
++)
1337 CONSTRUCTOR_ELT (rhs
, i
)->value
= base
;
1338 g
= gimple_build_assign (make_ssa_name (type
), rhs
);
1339 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1340 g
= gimple_build_assign (lhs
, PLUS_EXPR
, gimple_assign_lhs (g
),
1342 gsi_replace (gsi
, g
, false);
1345 /* Return a type for the widest vector mode whose components are of type
1346 TYPE, or NULL_TREE if none is found. */
1349 type_for_widest_vector_mode (tree type
, optab op
)
1351 machine_mode inner_mode
= TYPE_MODE (type
);
1352 machine_mode best_mode
= VOIDmode
, mode
;
1353 poly_int64 best_nunits
= 0;
1355 if (SCALAR_FLOAT_MODE_P (inner_mode
))
1356 mode
= MIN_MODE_VECTOR_FLOAT
;
1357 else if (SCALAR_FRACT_MODE_P (inner_mode
))
1358 mode
= MIN_MODE_VECTOR_FRACT
;
1359 else if (SCALAR_UFRACT_MODE_P (inner_mode
))
1360 mode
= MIN_MODE_VECTOR_UFRACT
;
1361 else if (SCALAR_ACCUM_MODE_P (inner_mode
))
1362 mode
= MIN_MODE_VECTOR_ACCUM
;
1363 else if (SCALAR_UACCUM_MODE_P (inner_mode
))
1364 mode
= MIN_MODE_VECTOR_UACCUM
;
1365 else if (inner_mode
== BImode
)
1366 mode
= MIN_MODE_VECTOR_BOOL
;
1368 mode
= MIN_MODE_VECTOR_INT
;
1370 FOR_EACH_MODE_FROM (mode
, mode
)
1371 if (GET_MODE_INNER (mode
) == inner_mode
1372 && maybe_gt (GET_MODE_NUNITS (mode
), best_nunits
)
1373 && optab_handler (op
, mode
) != CODE_FOR_nothing
)
1374 best_mode
= mode
, best_nunits
= GET_MODE_NUNITS (mode
);
1376 if (best_mode
== VOIDmode
)
1379 return build_vector_type_for_mode (type
, best_mode
);
1383 /* Build a reference to the element of the vector VECT. Function
1384 returns either the element itself, either BIT_FIELD_REF, or an
1385 ARRAY_REF expression.
1387 GSI is required to insert temporary variables while building a
1388 refernece to the element of the vector VECT.
1390 PTMPVEC is a pointer to the temporary variable for caching
1391 purposes. In case when PTMPVEC is NULL new temporary variable
1394 vector_element (gimple_stmt_iterator
*gsi
, tree vect
, tree idx
, tree
*ptmpvec
)
1396 tree vect_type
, vect_elt_type
;
1400 bool need_asgn
= true;
1401 unsigned int elements
;
1403 vect_type
= TREE_TYPE (vect
);
1404 vect_elt_type
= TREE_TYPE (vect_type
);
1405 elements
= nunits_for_known_piecewise_op (vect_type
);
1407 if (TREE_CODE (idx
) == INTEGER_CST
)
1409 unsigned HOST_WIDE_INT index
;
1411 /* Given that we're about to compute a binary modulus,
1412 we don't care about the high bits of the value. */
1413 index
= TREE_INT_CST_LOW (idx
);
1414 if (!tree_fits_uhwi_p (idx
) || index
>= elements
)
1416 index
&= elements
- 1;
1417 idx
= build_int_cst (TREE_TYPE (idx
), index
);
1420 /* When lowering a vector statement sequence do some easy
1421 simplification by looking through intermediate vector results. */
1422 if (TREE_CODE (vect
) == SSA_NAME
)
1424 gimple
*def_stmt
= SSA_NAME_DEF_STMT (vect
);
1425 if (is_gimple_assign (def_stmt
)
1426 && (gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
1427 || gimple_assign_rhs_code (def_stmt
) == CONSTRUCTOR
))
1428 vect
= gimple_assign_rhs1 (def_stmt
);
1431 if (TREE_CODE (vect
) == VECTOR_CST
)
1432 return VECTOR_CST_ELT (vect
, index
);
1433 else if (TREE_CODE (vect
) == CONSTRUCTOR
1434 && (CONSTRUCTOR_NELTS (vect
) == 0
1435 || TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (vect
, 0)->value
))
1438 if (index
< CONSTRUCTOR_NELTS (vect
))
1439 return CONSTRUCTOR_ELT (vect
, index
)->value
;
1440 return build_zero_cst (vect_elt_type
);
1444 tree size
= vector_element_bits_tree (vect_type
);
1445 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
, bitsize_int (index
),
1447 return fold_build3 (BIT_FIELD_REF
, vect_elt_type
, vect
, size
, pos
);
1452 tmpvec
= create_tmp_var (vect_type
, "vectmp");
1454 tmpvec
= *ptmpvec
= create_tmp_var (vect_type
, "vectmp");
1463 TREE_ADDRESSABLE (tmpvec
) = 1;
1464 asgn
= gimple_build_assign (tmpvec
, vect
);
1465 gsi_insert_before (gsi
, asgn
, GSI_SAME_STMT
);
1468 arraytype
= build_array_type_nelts (vect_elt_type
, elements
);
1469 return build4 (ARRAY_REF
, vect_elt_type
,
1470 build1 (VIEW_CONVERT_EXPR
, arraytype
, tmpvec
),
1471 idx
, NULL_TREE
, NULL_TREE
);
1474 /* Check if VEC_PERM_EXPR within the given setting is supported
1475 by hardware, or lower it piecewise.
1477 When VEC_PERM_EXPR has the same first and second operands:
1478 VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1479 {v0[mask[0]], v0[mask[1]], ...}
1480 MASK and V0 must have the same number of elements.
1482 Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1483 {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1484 V0 and V1 must have the same type. MASK, V0, V1 must have the
1485 same number of arguments. */
1488 lower_vec_perm (gimple_stmt_iterator
*gsi
)
1490 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1491 tree mask
= gimple_assign_rhs3 (stmt
);
1492 tree vec0
= gimple_assign_rhs1 (stmt
);
1493 tree vec1
= gimple_assign_rhs2 (stmt
);
1494 tree vect_type
= TREE_TYPE (vec0
);
1495 tree mask_type
= TREE_TYPE (mask
);
1496 tree vect_elt_type
= TREE_TYPE (vect_type
);
1497 tree mask_elt_type
= TREE_TYPE (mask_type
);
1498 unsigned HOST_WIDE_INT elements
;
1499 vec
<constructor_elt
, va_gc
> *v
;
1500 tree constr
, t
, si
, i_val
;
1501 tree vec0tmp
= NULL_TREE
, vec1tmp
= NULL_TREE
, masktmp
= NULL_TREE
;
1502 bool two_operand_p
= !operand_equal_p (vec0
, vec1
, 0);
1503 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1506 if (!TYPE_VECTOR_SUBPARTS (vect_type
).is_constant (&elements
))
1509 if (TREE_CODE (mask
) == SSA_NAME
)
1511 gimple
*def_stmt
= SSA_NAME_DEF_STMT (mask
);
1512 if (is_gimple_assign (def_stmt
)
1513 && gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
)
1514 mask
= gimple_assign_rhs1 (def_stmt
);
1517 vec_perm_builder sel_int
;
1519 if (TREE_CODE (mask
) == VECTOR_CST
1520 && tree_to_vec_perm_builder (&sel_int
, mask
))
1522 vec_perm_indices
indices (sel_int
, 2, elements
);
1523 if (can_vec_perm_const_p (TYPE_MODE (vect_type
), indices
))
1525 gimple_assign_set_rhs3 (stmt
, mask
);
1529 /* Also detect vec_shr pattern - VEC_PERM_EXPR with zero
1530 vector as VEC1 and a right element shift MASK. */
1531 if (optab_handler (vec_shr_optab
, TYPE_MODE (vect_type
))
1533 && TREE_CODE (vec1
) == VECTOR_CST
1534 && initializer_zerop (vec1
)
1535 && maybe_ne (indices
[0], 0)
1536 && known_lt (poly_uint64 (indices
[0]), elements
))
1538 bool ok_p
= indices
.series_p (0, 1, indices
[0], 1);
1541 for (i
= 1; i
< elements
; ++i
)
1543 poly_uint64 actual
= indices
[i
];
1544 poly_uint64 expected
= i
+ indices
[0];
1545 /* Indices into the second vector are all equivalent. */
1546 if (maybe_lt (actual
, elements
)
1547 ? maybe_ne (actual
, expected
)
1548 : maybe_lt (expected
, elements
))
1551 ok_p
= i
== elements
;
1555 gimple_assign_set_rhs3 (stmt
, mask
);
1560 /* And similarly vec_shl pattern. */
1561 if (optab_handler (vec_shl_optab
, TYPE_MODE (vect_type
))
1563 && TREE_CODE (vec0
) == VECTOR_CST
1564 && initializer_zerop (vec0
))
1566 unsigned int first
= 0;
1567 for (i
= 0; i
< elements
; ++i
)
1568 if (known_eq (poly_uint64 (indices
[i
]), elements
))
1570 if (i
== 0 || first
)
1575 ? maybe_ne (poly_uint64 (indices
[i
]),
1576 elements
+ i
- first
)
1577 : maybe_ge (poly_uint64 (indices
[i
]), elements
))
1579 if (first
&& i
== elements
)
1581 gimple_assign_set_rhs3 (stmt
, mask
);
1587 else if (can_vec_perm_var_p (TYPE_MODE (vect_type
)))
1590 warning_at (loc
, OPT_Wvector_operation_performance
,
1591 "vector shuffling operation will be expanded piecewise");
1593 vec_alloc (v
, elements
);
1594 bool constant_p
= true;
1595 for (i
= 0; i
< elements
; i
++)
1598 i_val
= vector_element (gsi
, mask
, si
, &masktmp
);
1600 if (TREE_CODE (i_val
) == INTEGER_CST
)
1602 unsigned HOST_WIDE_INT index
;
1604 index
= TREE_INT_CST_LOW (i_val
);
1605 if (!tree_fits_uhwi_p (i_val
) || index
>= elements
)
1606 i_val
= build_int_cst (mask_elt_type
, index
& (elements
- 1));
1608 if (two_operand_p
&& (index
& elements
) != 0)
1609 t
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1611 t
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1613 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
1614 true, GSI_SAME_STMT
);
1618 tree cond
= NULL_TREE
, v0_val
;
1622 cond
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1623 build_int_cst (mask_elt_type
, elements
));
1624 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1625 true, GSI_SAME_STMT
);
1628 i_val
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1629 build_int_cst (mask_elt_type
, elements
- 1));
1630 i_val
= force_gimple_operand_gsi (gsi
, i_val
, true, NULL_TREE
,
1631 true, GSI_SAME_STMT
);
1633 v0_val
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1634 v0_val
= force_gimple_operand_gsi (gsi
, v0_val
, true, NULL_TREE
,
1635 true, GSI_SAME_STMT
);
1641 v1_val
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1642 v1_val
= force_gimple_operand_gsi (gsi
, v1_val
, true, NULL_TREE
,
1643 true, GSI_SAME_STMT
);
1645 cond
= fold_build2 (EQ_EXPR
, boolean_type_node
,
1646 cond
, build_zero_cst (mask_elt_type
));
1647 cond
= fold_build3 (COND_EXPR
, vect_elt_type
,
1648 cond
, v0_val
, v1_val
);
1649 t
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1650 true, GSI_SAME_STMT
);
1656 if (!CONSTANT_CLASS_P (t
))
1658 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, t
);
1662 constr
= build_vector_from_ctor (vect_type
, v
);
1664 constr
= build_constructor (vect_type
, v
);
1665 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1666 update_stmt (gsi_stmt (*gsi
));
1669 /* If OP is a uniform vector return the element it is a splat from. */
1672 ssa_uniform_vector_p (tree op
)
1674 if (TREE_CODE (op
) == VECTOR_CST
1675 || TREE_CODE (op
) == VEC_DUPLICATE_EXPR
1676 || TREE_CODE (op
) == CONSTRUCTOR
)
1677 return uniform_vector_p (op
);
1678 if (TREE_CODE (op
) == SSA_NAME
)
1680 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
1681 if (gimple_assign_single_p (def_stmt
))
1682 return uniform_vector_p (gimple_assign_rhs1 (def_stmt
));
1687 /* Return type in which CODE operation with optab OP can be
1691 get_compute_type (enum tree_code code
, optab op
, tree type
)
1693 /* For very wide vectors, try using a smaller vector mode. */
1694 tree compute_type
= type
;
1696 && (!VECTOR_MODE_P (TYPE_MODE (type
))
1697 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
))
1699 tree vector_compute_type
1700 = type_for_widest_vector_mode (TREE_TYPE (type
), op
);
1701 if (vector_compute_type
!= NULL_TREE
1702 && subparts_gt (compute_type
, vector_compute_type
)
1703 && maybe_ne (TYPE_VECTOR_SUBPARTS (vector_compute_type
), 1U)
1704 && (optab_handler (op
, TYPE_MODE (vector_compute_type
))
1705 != CODE_FOR_nothing
))
1706 compute_type
= vector_compute_type
;
1709 /* If we are breaking a BLKmode vector into smaller pieces,
1710 type_for_widest_vector_mode has already looked into the optab,
1711 so skip these checks. */
1712 if (compute_type
== type
)
1714 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1715 if (VECTOR_MODE_P (compute_mode
))
1717 if (op
&& optab_handler (op
, compute_mode
) != CODE_FOR_nothing
)
1718 return compute_type
;
1719 if (code
== MULT_HIGHPART_EXPR
1720 && can_mult_highpart_p (compute_mode
,
1721 TYPE_UNSIGNED (compute_type
)))
1722 return compute_type
;
1724 /* There is no operation in hardware, so fall back to scalars. */
1725 compute_type
= TREE_TYPE (type
);
1728 return compute_type
;
1732 do_cond (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
1733 tree bitpos
, tree bitsize
, enum tree_code code
,
1734 tree type ATTRIBUTE_UNUSED
)
1736 if (TREE_CODE (TREE_TYPE (a
)) == VECTOR_TYPE
)
1737 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1738 if (TREE_CODE (TREE_TYPE (b
)) == VECTOR_TYPE
)
1739 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
1740 tree cond
= gimple_assign_rhs1 (gsi_stmt (*gsi
));
1741 return gimplify_build3 (gsi
, code
, inner_type
, unshare_expr (cond
), a
, b
);
1744 /* Expand a vector COND_EXPR to scalars, piecewise. */
1746 expand_vector_scalar_condition (gimple_stmt_iterator
*gsi
)
1748 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1749 tree lhs
= gimple_assign_lhs (stmt
);
1750 tree type
= TREE_TYPE (lhs
);
1751 tree compute_type
= get_compute_type (COND_EXPR
, mov_optab
, type
);
1752 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1753 gcc_assert (compute_mode
!= BLKmode
);
1754 tree rhs2
= gimple_assign_rhs2 (stmt
);
1755 tree rhs3
= gimple_assign_rhs3 (stmt
);
1758 /* If the compute mode is not a vector mode (hence we are not decomposing
1759 a BLKmode vector to smaller, hardware-supported vectors), we may want
1760 to expand the operations in parallel. */
1761 if (!VECTOR_MODE_P (compute_mode
))
1762 new_rhs
= expand_vector_parallel (gsi
, do_cond
, type
, rhs2
, rhs3
,
1765 new_rhs
= expand_vector_piecewise (gsi
, do_cond
, type
, compute_type
,
1766 rhs2
, rhs3
, COND_EXPR
, false);
1767 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
1768 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
1771 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1772 way to do it is change expand_vector_operation and its callees to
1773 return a tree_code, RHS1 and RHS2 instead of a tree. */
1774 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
1775 update_stmt (gsi_stmt (*gsi
));
1778 /* Callback for expand_vector_piecewise to do VEC_CONVERT ifn call
1779 lowering. If INNER_TYPE is not a vector type, this is a scalar
1783 do_vec_conversion (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
1784 tree decl
, tree bitpos
, tree bitsize
,
1785 enum tree_code code
, tree type
)
1787 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1788 if (!VECTOR_TYPE_P (inner_type
))
1789 return gimplify_build1 (gsi
, code
, TREE_TYPE (type
), a
);
1790 if (code
== CALL_EXPR
)
1792 gimple
*g
= gimple_build_call (decl
, 1, a
);
1793 tree lhs
= make_ssa_name (TREE_TYPE (TREE_TYPE (decl
)));
1794 gimple_call_set_lhs (g
, lhs
);
1795 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1800 tree outer_type
= build_vector_type (TREE_TYPE (type
),
1801 TYPE_VECTOR_SUBPARTS (inner_type
));
1802 return gimplify_build1 (gsi
, code
, outer_type
, a
);
1806 /* Similarly, but for narrowing conversion. */
1809 do_vec_narrow_conversion (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
1810 tree
, tree bitpos
, tree
, enum tree_code code
,
1813 tree itype
= build_vector_type (TREE_TYPE (inner_type
),
1814 exact_div (TYPE_VECTOR_SUBPARTS (inner_type
),
1816 tree b
= tree_vec_extract (gsi
, itype
, a
, TYPE_SIZE (itype
), bitpos
);
1817 tree c
= tree_vec_extract (gsi
, itype
, a
, TYPE_SIZE (itype
),
1818 int_const_binop (PLUS_EXPR
, bitpos
,
1819 TYPE_SIZE (itype
)));
1820 tree outer_type
= build_vector_type (TREE_TYPE (type
),
1821 TYPE_VECTOR_SUBPARTS (inner_type
));
1822 return gimplify_build2 (gsi
, code
, outer_type
, b
, c
);
1825 /* Expand VEC_CONVERT ifn call. */
1828 expand_vector_conversion (gimple_stmt_iterator
*gsi
)
1830 gimple
*stmt
= gsi_stmt (*gsi
);
1832 tree lhs
= gimple_call_lhs (stmt
);
1833 if (lhs
== NULL_TREE
)
1835 g
= gimple_build_nop ();
1836 gsi_replace (gsi
, g
, false);
1839 tree arg
= gimple_call_arg (stmt
, 0);
1840 tree ret_type
= TREE_TYPE (lhs
);
1841 tree arg_type
= TREE_TYPE (arg
);
1842 tree new_rhs
, compute_type
= TREE_TYPE (arg_type
);
1843 enum tree_code code
= NOP_EXPR
;
1844 enum tree_code code1
= ERROR_MARK
;
1845 enum { NARROW
, NONE
, WIDEN
} modifier
= NONE
;
1846 optab optab1
= unknown_optab
;
1848 gcc_checking_assert (VECTOR_TYPE_P (ret_type
) && VECTOR_TYPE_P (arg_type
));
1849 if (INTEGRAL_TYPE_P (TREE_TYPE (ret_type
))
1850 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg_type
)))
1851 code
= FIX_TRUNC_EXPR
;
1852 else if (INTEGRAL_TYPE_P (TREE_TYPE (arg_type
))
1853 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (ret_type
)))
1855 unsigned int ret_elt_bits
= vector_element_bits (ret_type
);
1856 unsigned int arg_elt_bits
= vector_element_bits (arg_type
);
1857 if (ret_elt_bits
< arg_elt_bits
)
1859 else if (ret_elt_bits
> arg_elt_bits
)
1862 if (modifier
== NONE
&& (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
))
1864 if (supportable_convert_operation (code
, ret_type
, arg_type
, &code1
))
1866 g
= gimple_build_assign (lhs
, code1
, arg
);
1867 gsi_replace (gsi
, g
, false);
1870 /* Can't use get_compute_type here, as supportable_convert_operation
1871 doesn't necessarily use an optab and needs two arguments. */
1872 tree vec_compute_type
1873 = type_for_widest_vector_mode (TREE_TYPE (arg_type
), mov_optab
);
1874 if (vec_compute_type
1875 && VECTOR_MODE_P (TYPE_MODE (vec_compute_type
))
1876 && subparts_gt (arg_type
, vec_compute_type
))
1878 unsigned HOST_WIDE_INT nelts
1879 = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vec_compute_type
));
1882 tree ret1_type
= build_vector_type (TREE_TYPE (ret_type
), nelts
);
1883 tree arg1_type
= build_vector_type (TREE_TYPE (arg_type
), nelts
);
1884 if (supportable_convert_operation (code
, ret1_type
, arg1_type
,
1887 new_rhs
= expand_vector_piecewise (gsi
, do_vec_conversion
,
1888 ret_type
, arg1_type
, arg
,
1889 NULL_TREE
, code1
, false);
1890 g
= gimple_build_assign (lhs
, new_rhs
);
1891 gsi_replace (gsi
, g
, false);
1898 else if (modifier
== NARROW
)
1903 code1
= VEC_PACK_TRUNC_EXPR
;
1904 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1906 case FIX_TRUNC_EXPR
:
1907 code1
= VEC_PACK_FIX_TRUNC_EXPR
;
1908 /* The signedness is determined from output operand. */
1909 optab1
= optab_for_tree_code (code1
, ret_type
, optab_default
);
1912 code1
= VEC_PACK_FLOAT_EXPR
;
1913 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1920 compute_type
= get_compute_type (code1
, optab1
, arg_type
);
1921 enum insn_code icode1
;
1922 if (VECTOR_TYPE_P (compute_type
)
1923 && ((icode1
= optab_handler (optab1
, TYPE_MODE (compute_type
)))
1924 != CODE_FOR_nothing
)
1925 && VECTOR_MODE_P (insn_data
[icode1
].operand
[0].mode
))
1928 = build_vector_type (TREE_TYPE (ret_type
),
1929 TYPE_VECTOR_SUBPARTS (compute_type
) * 2);
1930 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (cretd_type
))
1932 if (compute_type
== arg_type
)
1934 new_rhs
= gimplify_build2 (gsi
, code1
, cretd_type
,
1935 arg
, build_zero_cst (arg_type
));
1936 new_rhs
= tree_vec_extract (gsi
, ret_type
, new_rhs
,
1937 TYPE_SIZE (ret_type
),
1939 g
= gimple_build_assign (lhs
, new_rhs
);
1940 gsi_replace (gsi
, g
, false);
1944 = build_vector_type (TREE_TYPE (compute_type
),
1945 TYPE_VECTOR_SUBPARTS (compute_type
) * 2);
1946 if (TYPE_MAIN_VARIANT (dcompute_type
)
1947 == TYPE_MAIN_VARIANT (arg_type
))
1948 new_rhs
= do_vec_narrow_conversion (gsi
, dcompute_type
, arg
,
1949 NULL_TREE
, bitsize_int (0),
1953 new_rhs
= expand_vector_piecewise (gsi
,
1954 do_vec_narrow_conversion
,
1955 arg_type
, dcompute_type
,
1956 arg
, NULL_TREE
, code1
,
1958 g
= gimple_build_assign (lhs
, new_rhs
);
1959 gsi_replace (gsi
, g
, false);
1964 else if (modifier
== WIDEN
)
1966 enum tree_code code2
= ERROR_MARK
;
1967 optab optab2
= unknown_optab
;
1971 code1
= VEC_UNPACK_LO_EXPR
;
1972 code2
= VEC_UNPACK_HI_EXPR
;
1974 case FIX_TRUNC_EXPR
:
1975 code1
= VEC_UNPACK_FIX_TRUNC_LO_EXPR
;
1976 code2
= VEC_UNPACK_FIX_TRUNC_HI_EXPR
;
1979 code1
= VEC_UNPACK_FLOAT_LO_EXPR
;
1980 code2
= VEC_UNPACK_FLOAT_HI_EXPR
;
1985 if (BYTES_BIG_ENDIAN
)
1986 std::swap (code1
, code2
);
1988 if (code
== FIX_TRUNC_EXPR
)
1990 /* The signedness is determined from output operand. */
1991 optab1
= optab_for_tree_code (code1
, ret_type
, optab_default
);
1992 optab2
= optab_for_tree_code (code2
, ret_type
, optab_default
);
1996 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1997 optab2
= optab_for_tree_code (code2
, arg_type
, optab_default
);
2000 if (optab1
&& optab2
)
2001 compute_type
= get_compute_type (code1
, optab1
, arg_type
);
2003 enum insn_code icode1
, icode2
;
2004 if (VECTOR_TYPE_P (compute_type
)
2005 && ((icode1
= optab_handler (optab1
, TYPE_MODE (compute_type
)))
2006 != CODE_FOR_nothing
)
2007 && ((icode2
= optab_handler (optab2
, TYPE_MODE (compute_type
)))
2008 != CODE_FOR_nothing
)
2009 && VECTOR_MODE_P (insn_data
[icode1
].operand
[0].mode
)
2010 && (insn_data
[icode1
].operand
[0].mode
2011 == insn_data
[icode2
].operand
[0].mode
))
2014 = exact_div (TYPE_VECTOR_SUBPARTS (compute_type
), 2);
2015 tree cretd_type
= build_vector_type (TREE_TYPE (ret_type
), nunits
);
2016 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (cretd_type
))
2018 vec
<constructor_elt
, va_gc
> *v
;
2019 tree part_width
= TYPE_SIZE (compute_type
);
2020 tree index
= bitsize_int (0);
2021 int nunits
= nunits_for_known_piecewise_op (arg_type
);
2022 int delta
= tree_to_uhwi (part_width
) / arg_elt_bits
;
2024 location_t loc
= gimple_location (gsi_stmt (*gsi
));
2026 if (compute_type
!= arg_type
)
2027 warning_at (loc
, OPT_Wvector_operation_performance
,
2028 "vector operation will be expanded piecewise");
2035 vec_alloc (v
, (nunits
+ delta
- 1) / delta
* 2);
2036 bool constant_p
= true;
2037 for (i
= 0; i
< nunits
;
2038 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
,
2042 if (compute_type
!= arg_type
)
2043 a
= tree_vec_extract (gsi
, compute_type
, a
, part_width
,
2045 tree result
= gimplify_build1 (gsi
, code1
, cretd_type
, a
);
2046 constructor_elt ce
= { NULL_TREE
, result
};
2047 if (!CONSTANT_CLASS_P (ce
.value
))
2050 ce
.value
= gimplify_build1 (gsi
, code2
, cretd_type
, a
);
2051 if (!CONSTANT_CLASS_P (ce
.value
))
2057 new_rhs
= build_vector_from_ctor (ret_type
, v
);
2059 new_rhs
= build_constructor (ret_type
, v
);
2060 g
= gimple_build_assign (lhs
, new_rhs
);
2061 gsi_replace (gsi
, g
, false);
2067 new_rhs
= expand_vector_piecewise (gsi
, do_vec_conversion
, arg_type
,
2068 TREE_TYPE (arg_type
), arg
,
2069 NULL_TREE
, code
, false, ret_type
);
2070 g
= gimple_build_assign (lhs
, new_rhs
);
2071 gsi_replace (gsi
, g
, false);
2074 /* Process one statement. If we identify a vector operation, expand it. */
2077 expand_vector_operations_1 (gimple_stmt_iterator
*gsi
,
2078 bitmap dce_ssa_names
)
2080 tree lhs
, rhs1
, rhs2
= NULL
, type
, compute_type
= NULL_TREE
;
2081 enum tree_code code
;
2082 optab op
= unknown_optab
;
2083 enum gimple_rhs_class rhs_class
;
2086 /* Only consider code == GIMPLE_ASSIGN. */
2087 gassign
*stmt
= dyn_cast
<gassign
*> (gsi_stmt (*gsi
));
2090 if (gimple_call_internal_p (gsi_stmt (*gsi
), IFN_VEC_CONVERT
))
2091 expand_vector_conversion (gsi
);
2095 code
= gimple_assign_rhs_code (stmt
);
2096 rhs_class
= get_gimple_rhs_class (code
);
2097 lhs
= gimple_assign_lhs (stmt
);
2099 if (code
== VEC_PERM_EXPR
)
2101 lower_vec_perm (gsi
);
2105 if (code
== VEC_COND_EXPR
)
2107 expand_vector_condition (gsi
, dce_ssa_names
);
2111 if (code
== COND_EXPR
2112 && TREE_CODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == VECTOR_TYPE
2113 && TYPE_MODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == BLKmode
)
2115 expand_vector_scalar_condition (gsi
);
2119 if (code
== CONSTRUCTOR
2120 && TREE_CODE (lhs
) == SSA_NAME
2121 && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (lhs
)))
2122 && !gimple_clobber_p (stmt
)
2125 optimize_vector_constructor (gsi
);
2129 if (rhs_class
!= GIMPLE_UNARY_RHS
&& rhs_class
!= GIMPLE_BINARY_RHS
)
2132 rhs1
= gimple_assign_rhs1 (stmt
);
2133 if (rhs_class
== GIMPLE_BINARY_RHS
)
2134 rhs2
= gimple_assign_rhs2 (stmt
);
2136 type
= TREE_TYPE (lhs
);
2137 if (!VECTOR_TYPE_P (type
)
2138 || !VECTOR_TYPE_P (TREE_TYPE (rhs1
)))
2141 /* A scalar operation pretending to be a vector one. */
2142 if (VECTOR_BOOLEAN_TYPE_P (type
)
2143 && !VECTOR_MODE_P (TYPE_MODE (type
))
2144 && TYPE_MODE (type
) != BLKmode
2145 && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt
)) != tcc_comparison
2146 || (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1
))
2147 && !VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (rhs1
)))
2148 && TYPE_MODE (TREE_TYPE (rhs1
)) != BLKmode
)))
2151 /* If the vector operation is operating on all same vector elements
2152 implement it with a scalar operation and a splat if the target
2153 supports the scalar operation. */
2154 tree srhs1
, srhs2
= NULL_TREE
;
2155 if ((srhs1
= ssa_uniform_vector_p (rhs1
)) != NULL_TREE
2156 && (rhs2
== NULL_TREE
2157 || (! VECTOR_TYPE_P (TREE_TYPE (rhs2
))
2159 || (srhs2
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
2160 /* As we query direct optabs restrict to non-convert operations. */
2161 && TYPE_MODE (TREE_TYPE (type
)) == TYPE_MODE (TREE_TYPE (srhs1
)))
2163 op
= optab_for_tree_code (code
, TREE_TYPE (type
), optab_scalar
);
2164 if (op
>= FIRST_NORM_OPTAB
&& op
<= LAST_NORM_OPTAB
2165 && optab_handler (op
, TYPE_MODE (TREE_TYPE (type
))) != CODE_FOR_nothing
)
2167 tree stype
= TREE_TYPE (TREE_TYPE (lhs
));
2168 tree slhs
= (rhs2
!= NULL_TREE
)
2169 ? gimplify_build2 (gsi
, code
, stype
, srhs1
, srhs2
)
2170 : gimplify_build1 (gsi
, code
, stype
, srhs1
);
2171 gimple_assign_set_rhs_from_tree (gsi
,
2172 build_vector_from_val (type
, slhs
));
2178 if (CONVERT_EXPR_CODE_P (code
)
2179 || code
== FLOAT_EXPR
2180 || code
== FIX_TRUNC_EXPR
2181 || code
== VIEW_CONVERT_EXPR
)
2184 /* The signedness is determined from input argument. */
2185 if (code
== VEC_UNPACK_FLOAT_HI_EXPR
2186 || code
== VEC_UNPACK_FLOAT_LO_EXPR
2187 || code
== VEC_PACK_FLOAT_EXPR
)
2189 /* We do not know how to scalarize those. */
2193 /* For widening/narrowing vector operations, the relevant type is of the
2194 arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
2195 calculated in the same way above. */
2196 if (code
== WIDEN_SUM_EXPR
2197 || code
== VEC_WIDEN_PLUS_HI_EXPR
2198 || code
== VEC_WIDEN_PLUS_LO_EXPR
2199 || code
== VEC_WIDEN_MINUS_HI_EXPR
2200 || code
== VEC_WIDEN_MINUS_LO_EXPR
2201 || code
== VEC_WIDEN_MULT_HI_EXPR
2202 || code
== VEC_WIDEN_MULT_LO_EXPR
2203 || code
== VEC_WIDEN_MULT_EVEN_EXPR
2204 || code
== VEC_WIDEN_MULT_ODD_EXPR
2205 || code
== VEC_UNPACK_HI_EXPR
2206 || code
== VEC_UNPACK_LO_EXPR
2207 || code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
2208 || code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
2209 || code
== VEC_PACK_TRUNC_EXPR
2210 || code
== VEC_PACK_SAT_EXPR
2211 || code
== VEC_PACK_FIX_TRUNC_EXPR
2212 || code
== VEC_WIDEN_LSHIFT_HI_EXPR
2213 || code
== VEC_WIDEN_LSHIFT_LO_EXPR
)
2215 /* We do not know how to scalarize those. */
2219 /* Choose between vector shift/rotate by vector and vector shift/rotate by
2221 if (code
== LSHIFT_EXPR
2222 || code
== RSHIFT_EXPR
2223 || code
== LROTATE_EXPR
2224 || code
== RROTATE_EXPR
)
2228 /* Check whether we have vector <op> {x,x,x,x} where x
2229 could be a scalar variable or a constant. Transform
2230 vector <op> {x,x,x,x} ==> vector <op> scalar. */
2231 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2235 if ((first
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
2237 gimple_assign_set_rhs2 (stmt
, first
);
2243 opv
= optab_for_tree_code (code
, type
, optab_vector
);
2244 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2248 op
= optab_for_tree_code (code
, type
, optab_scalar
);
2250 compute_type
= get_compute_type (code
, op
, type
);
2251 if (compute_type
== type
)
2253 /* The rtl expander will expand vector/scalar as vector/vector
2254 if necessary. Pick one with wider vector type. */
2255 tree compute_vtype
= get_compute_type (code
, opv
, type
);
2256 if (subparts_gt (compute_vtype
, compute_type
))
2258 compute_type
= compute_vtype
;
2263 if (code
== LROTATE_EXPR
|| code
== RROTATE_EXPR
)
2265 if (compute_type
== NULL_TREE
)
2266 compute_type
= get_compute_type (code
, op
, type
);
2267 if (compute_type
== type
)
2269 /* Before splitting vector rotates into scalar rotates,
2270 see if we can't use vector shifts and BIT_IOR_EXPR
2271 instead. For vector by vector rotates we'd also
2272 need to check BIT_AND_EXPR and NEGATE_EXPR, punt there
2273 for now, fold doesn't seem to create such rotates anyway. */
2274 if (compute_type
== TREE_TYPE (type
)
2275 && !VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2277 optab oplv
= vashl_optab
, opl
= ashl_optab
;
2278 optab oprv
= vlshr_optab
, opr
= lshr_optab
, opo
= ior_optab
;
2279 tree compute_lvtype
= get_compute_type (LSHIFT_EXPR
, oplv
, type
);
2280 tree compute_rvtype
= get_compute_type (RSHIFT_EXPR
, oprv
, type
);
2281 tree compute_otype
= get_compute_type (BIT_IOR_EXPR
, opo
, type
);
2282 tree compute_ltype
= get_compute_type (LSHIFT_EXPR
, opl
, type
);
2283 tree compute_rtype
= get_compute_type (RSHIFT_EXPR
, opr
, type
);
2284 /* The rtl expander will expand vector/scalar as vector/vector
2285 if necessary. Pick one with wider vector type. */
2286 if (subparts_gt (compute_lvtype
, compute_ltype
))
2288 compute_ltype
= compute_lvtype
;
2291 if (subparts_gt (compute_rvtype
, compute_rtype
))
2293 compute_rtype
= compute_rvtype
;
2296 /* Pick the narrowest type from LSHIFT_EXPR, RSHIFT_EXPR and
2298 compute_type
= compute_ltype
;
2299 if (subparts_gt (compute_type
, compute_rtype
))
2300 compute_type
= compute_rtype
;
2301 if (subparts_gt (compute_type
, compute_otype
))
2302 compute_type
= compute_otype
;
2303 /* Verify all 3 operations can be performed in that type. */
2304 if (compute_type
!= TREE_TYPE (type
))
2306 if (optab_handler (opl
, TYPE_MODE (compute_type
))
2308 || optab_handler (opr
, TYPE_MODE (compute_type
))
2310 || optab_handler (opo
, TYPE_MODE (compute_type
))
2311 == CODE_FOR_nothing
)
2312 compute_type
= TREE_TYPE (type
);
2318 op
= optab_for_tree_code (code
, type
, optab_default
);
2320 /* Optabs will try converting a negation into a subtraction, so
2321 look for it as well. TODO: negation of floating-point vectors
2322 might be turned into an exclusive OR toggling the sign bit. */
2323 if (op
== unknown_optab
2324 && code
== NEGATE_EXPR
2325 && INTEGRAL_TYPE_P (TREE_TYPE (type
)))
2326 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
2328 if (compute_type
== NULL_TREE
)
2329 compute_type
= get_compute_type (code
, op
, type
);
2330 if (compute_type
== type
)
2333 new_rhs
= expand_vector_operation (gsi
, type
, compute_type
, stmt
, code
,
2336 /* Leave expression untouched for later expansion. */
2337 if (new_rhs
== NULL_TREE
)
2340 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
2341 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
2344 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
2345 way to do it is change expand_vector_operation and its callees to
2346 return a tree_code, RHS1 and RHS2 instead of a tree. */
2347 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
2348 update_stmt (gsi_stmt (*gsi
));
2351 /* Use this to lower vector operations introduced by the vectorizer,
2352 if it may need the bit-twiddling tricks implemented in this file. */
2355 expand_vector_operations (void)
2357 gimple_stmt_iterator gsi
;
2359 bool cfg_changed
= false;
2361 auto_bitmap dce_ssa_names
;
2363 FOR_EACH_BB_FN (bb
, cfun
)
2365 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
2367 expand_vector_operations_1 (&gsi
, dce_ssa_names
);
2368 /* ??? If we do not cleanup EH then we will ICE in
2369 verification. But in reality we have created wrong-code
2370 as we did not properly transition EH info and edges to
2371 the piecewise computations. */
2372 if (maybe_clean_eh_stmt (gsi_stmt (gsi
))
2373 && gimple_purge_dead_eh_edges (bb
))
2378 simple_dce_from_worklist (dce_ssa_names
);
2380 return cfg_changed
? TODO_cleanup_cfg
: 0;
2385 const pass_data pass_data_lower_vector
=
2387 GIMPLE_PASS
, /* type */
2388 "veclower", /* name */
2389 OPTGROUP_VEC
, /* optinfo_flags */
2390 TV_NONE
, /* tv_id */
2391 PROP_cfg
, /* properties_required */
2392 PROP_gimple_lvec
, /* properties_provided */
2393 0, /* properties_destroyed */
2394 0, /* todo_flags_start */
2395 TODO_update_ssa
, /* todo_flags_finish */
2398 class pass_lower_vector
: public gimple_opt_pass
2401 pass_lower_vector (gcc::context
*ctxt
)
2402 : gimple_opt_pass (pass_data_lower_vector
, ctxt
)
2405 /* opt_pass methods: */
2406 virtual bool gate (function
*fun
)
2408 return !(fun
->curr_properties
& PROP_gimple_lvec
);
2411 virtual unsigned int execute (function
*)
2413 return expand_vector_operations ();
2416 }; // class pass_lower_vector
2421 make_pass_lower_vector (gcc::context
*ctxt
)
2423 return new pass_lower_vector (ctxt
);
2428 const pass_data pass_data_lower_vector_ssa
=
2430 GIMPLE_PASS
, /* type */
2431 "veclower2", /* name */
2432 OPTGROUP_VEC
, /* optinfo_flags */
2433 TV_NONE
, /* tv_id */
2434 PROP_cfg
, /* properties_required */
2435 PROP_gimple_lvec
, /* properties_provided */
2436 0, /* properties_destroyed */
2437 0, /* todo_flags_start */
2439 | TODO_cleanup_cfg
), /* todo_flags_finish */
2442 class pass_lower_vector_ssa
: public gimple_opt_pass
2445 pass_lower_vector_ssa (gcc::context
*ctxt
)
2446 : gimple_opt_pass (pass_data_lower_vector_ssa
, ctxt
)
2449 /* opt_pass methods: */
2450 opt_pass
* clone () { return new pass_lower_vector_ssa (m_ctxt
); }
2451 virtual unsigned int execute (function
*)
2453 return expand_vector_operations ();
2456 }; // class pass_lower_vector_ssa
2461 make_pass_lower_vector_ssa (gcc::context
*ctxt
)
2463 return new pass_lower_vector_ssa (ctxt
);
2466 #include "gt-tree-vect-generic.h"