1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2023 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
45 #include "optabs-tree.h"
47 #include "internal-fn.h"
48 #include "langhooks.h"
52 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
54 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
55 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
57 static rtx
emit_conditional_move_1 (rtx
, rtx
, rtx
, rtx
, machine_mode
);
59 /* Debug facility for use in GDB. */
60 void debug_optab_libfuncs (void);
62 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
63 the result of operation CODE applied to OP0 (and OP1 if it is a binary
64 operation). OP0_MODE is OP0's mode.
66 If the last insn does not set TARGET, don't do anything, but return 1.
68 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
69 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
70 try again, ensuring that TARGET is not one of the operands. */
73 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
,
74 rtx op1
, machine_mode op0_mode
)
80 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
82 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
83 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
84 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
85 && GET_RTX_CLASS (code
) != RTX_COMPARE
86 && GET_RTX_CLASS (code
) != RTX_UNARY
)
89 if (GET_CODE (target
) == ZERO_EXTRACT
)
92 for (last_insn
= insns
;
93 NEXT_INSN (last_insn
) != NULL_RTX
;
94 last_insn
= NEXT_INSN (last_insn
))
97 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
98 a value changing in the insn, so the note would be invalid for CSE. */
99 if (reg_overlap_mentioned_p (target
, op0
)
100 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
103 && (rtx_equal_p (target
, op0
)
104 || (op1
&& rtx_equal_p (target
, op1
))))
106 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
107 over expanding it as temp = MEM op X, MEM = temp. If the target
108 supports MEM = MEM op X instructions, it is sometimes too hard
109 to reconstruct that form later, especially if X is also a memory,
110 and due to multiple occurrences of addresses the address might
111 be forced into register unnecessarily.
112 Note that not emitting the REG_EQUIV note might inhibit
113 CSE in some cases. */
114 set
= single_set (last_insn
);
116 && GET_CODE (SET_SRC (set
)) == code
117 && MEM_P (SET_DEST (set
))
118 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
119 || (op1
&& rtx_equal_p (SET_DEST (set
),
120 XEXP (SET_SRC (set
), 1)))))
126 set
= set_for_reg_notes (last_insn
);
130 if (! rtx_equal_p (SET_DEST (set
), target
)
131 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
132 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
133 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
136 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
146 if (op0_mode
!= VOIDmode
&& GET_MODE (target
) != op0_mode
)
148 note
= gen_rtx_fmt_e (code
, op0_mode
, copy_rtx (op0
));
149 if (GET_MODE_UNIT_SIZE (op0_mode
)
150 > GET_MODE_UNIT_SIZE (GET_MODE (target
)))
151 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
154 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
160 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
164 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
166 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
171 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
172 for a widening operation would be. In most cases this would be OP0, but if
173 that's a constant it'll be VOIDmode, which isn't useful. */
176 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
178 machine_mode m0
= GET_MODE (op0
);
179 machine_mode m1
= GET_MODE (op1
);
182 if (m0
== VOIDmode
&& m1
== VOIDmode
)
184 else if (m0
== VOIDmode
|| GET_MODE_UNIT_SIZE (m0
) < GET_MODE_UNIT_SIZE (m1
))
189 if (GET_MODE_UNIT_SIZE (result
) > GET_MODE_UNIT_SIZE (to_mode
))
195 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
196 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
197 not actually do a sign-extend or zero-extend, but can leave the
198 higher-order bits of the result rtx undefined, for example, in the case
199 of logical operations, but not right shifts. */
202 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
203 int unsignedp
, int no_extend
)
206 scalar_int_mode int_mode
;
208 /* If we don't have to extend and this is a constant, return it. */
209 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
212 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
213 extend since it will be more efficient to do so unless the signedness of
214 a promoted object differs from our extension. */
216 || !is_a
<scalar_int_mode
> (mode
, &int_mode
)
217 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
218 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
219 return convert_modes (mode
, oldmode
, op
, unsignedp
);
221 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
223 if (GET_MODE_SIZE (int_mode
) <= UNITS_PER_WORD
)
224 return gen_lowpart (int_mode
, force_reg (GET_MODE (op
), op
));
226 /* Otherwise, get an object of MODE, clobber it, and set the low-order
229 result
= gen_reg_rtx (int_mode
);
230 emit_clobber (result
);
231 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
235 /* Expand vector widening operations.
237 There are two different classes of operations handled here:
238 1) Operations whose result is wider than all the arguments to the operation.
239 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
240 In this case OP0 and optionally OP1 would be initialized,
241 but WIDE_OP wouldn't (not relevant for this case).
242 2) Operations whose result is of the same size as the last argument to the
243 operation, but wider than all the other arguments to the operation.
244 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
245 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
247 E.g, when called to expand the following operations, this is how
248 the arguments will be initialized:
250 widening-sum 2 oprnd0 - oprnd1
251 widening-dot-product 3 oprnd0 oprnd1 oprnd2
252 widening-mult 2 oprnd0 oprnd1 -
253 type-promotion (vec-unpack) 1 oprnd0 - - */
256 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
257 rtx target
, int unsignedp
)
259 class expand_operand eops
[4];
260 tree oprnd0
, oprnd1
, oprnd2
;
261 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
262 optab widen_pattern_optab
;
263 enum insn_code icode
;
264 int nops
= TREE_CODE_LENGTH (ops
->code
);
269 oprnd1
= nops
>= 2 ? ops
->op1
: NULL_TREE
;
270 oprnd2
= nops
>= 3 ? ops
->op2
: NULL_TREE
;
272 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
273 if (ops
->code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
274 || ops
->code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
)
275 /* The sign is from the result type rather than operand's type
278 = optab_for_tree_code (ops
->code
, ops
->type
, optab_default
);
279 else if ((ops
->code
== VEC_UNPACK_HI_EXPR
280 || ops
->code
== VEC_UNPACK_LO_EXPR
)
281 && VECTOR_BOOLEAN_TYPE_P (ops
->type
)
282 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0
))
283 && TYPE_MODE (ops
->type
) == TYPE_MODE (TREE_TYPE (oprnd0
))
284 && SCALAR_INT_MODE_P (TYPE_MODE (ops
->type
)))
286 /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
287 the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
288 vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
289 the pattern number of elements in the wider vector. */
291 = (ops
->code
== VEC_UNPACK_HI_EXPR
292 ? vec_unpacks_sbool_hi_optab
: vec_unpacks_sbool_lo_optab
);
295 else if (ops
->code
== DOT_PROD_EXPR
)
297 enum optab_subtype subtype
= optab_default
;
298 signop sign1
= TYPE_SIGN (TREE_TYPE (oprnd0
));
299 signop sign2
= TYPE_SIGN (TREE_TYPE (oprnd1
));
302 else if (sign1
== SIGNED
&& sign2
== UNSIGNED
)
304 subtype
= optab_vector_mixed_sign
;
305 /* Same as optab_vector_mixed_sign but flip the operands. */
306 std::swap (op0
, op1
);
308 else if (sign1
== UNSIGNED
&& sign2
== SIGNED
)
309 subtype
= optab_vector_mixed_sign
;
314 = optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), subtype
);
318 = optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
319 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
320 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
321 icode
= find_widening_optab_handler (widen_pattern_optab
,
322 TYPE_MODE (TREE_TYPE (ops
->op2
)),
325 icode
= optab_handler (widen_pattern_optab
, tmode0
);
326 gcc_assert (icode
!= CODE_FOR_nothing
);
329 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
333 op1
= GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0
)).to_constant ());
337 /* The last operand is of a wider mode than the rest of the operands. */
342 gcc_assert (tmode1
== tmode0
);
344 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
348 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
349 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
351 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
353 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
354 expand_insn (icode
, op
, eops
);
355 return eops
[0].value
;
358 /* Generate code to perform an operation specified by TERNARY_OPTAB
359 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
361 UNSIGNEDP is for the case where we have to widen the operands
362 to perform the operation. It says to use zero-extension.
364 If TARGET is nonzero, the value
365 is generated there, if it is convenient to do so.
366 In all cases an rtx is returned for the locus of the value;
367 this may or may not be TARGET. */
370 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
371 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
373 class expand_operand ops
[4];
374 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
376 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
378 create_output_operand (&ops
[0], target
, mode
);
379 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
380 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
381 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
382 expand_insn (icode
, 4, ops
);
387 /* Like expand_binop, but return a constant rtx if the result can be
388 calculated at compile time. The arguments and return value are
389 otherwise the same as for expand_binop. */
392 simplify_expand_binop (machine_mode mode
, optab binoptab
,
393 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
394 enum optab_methods methods
)
396 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
398 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
404 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
407 /* Like simplify_expand_binop, but always put the result in TARGET.
408 Return true if the expansion succeeded. */
411 force_expand_binop (machine_mode mode
, optab binoptab
,
412 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
413 enum optab_methods methods
)
415 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
416 target
, unsignedp
, methods
);
420 emit_move_insn (target
, x
);
424 /* Create a new vector value in VMODE with all elements set to OP. The
425 mode of OP must be the element mode of VMODE. If OP is a constant,
426 then the return value will be a constant. */
429 expand_vector_broadcast (machine_mode vmode
, rtx op
)
434 gcc_checking_assert (VECTOR_MODE_P (vmode
));
436 if (valid_for_const_vector_p (vmode
, op
))
437 return gen_const_vec_duplicate (vmode
, op
);
439 insn_code icode
= optab_handler (vec_duplicate_optab
, vmode
);
440 if (icode
!= CODE_FOR_nothing
)
442 class expand_operand ops
[2];
443 create_output_operand (&ops
[0], NULL_RTX
, vmode
);
444 create_input_operand (&ops
[1], op
, GET_MODE (op
));
445 expand_insn (icode
, 2, ops
);
449 if (!GET_MODE_NUNITS (vmode
).is_constant (&n
))
452 /* ??? If the target doesn't have a vec_init, then we have no easy way
453 of performing this operation. Most of this sort of generic support
454 is hidden away in the vector lowering support in gimple. */
455 icode
= convert_optab_handler (vec_init_optab
, vmode
,
456 GET_MODE_INNER (vmode
));
457 if (icode
== CODE_FOR_nothing
)
460 vec
= rtvec_alloc (n
);
461 for (int i
= 0; i
< n
; ++i
)
462 RTVEC_ELT (vec
, i
) = op
;
463 rtx ret
= gen_reg_rtx (vmode
);
464 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
469 /* This subroutine of expand_doubleword_shift handles the cases in which
470 the effective shift value is >= BITS_PER_WORD. The arguments and return
471 value are the same as for the parent routine, except that SUPERWORD_OP1
472 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
473 INTO_TARGET may be null if the caller has decided to calculate it. */
476 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
477 rtx outof_target
, rtx into_target
,
478 int unsignedp
, enum optab_methods methods
)
480 if (into_target
!= 0)
481 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
482 into_target
, unsignedp
, methods
))
485 if (outof_target
!= 0)
487 /* For a signed right shift, we must fill OUTOF_TARGET with copies
488 of the sign bit, otherwise we must fill it with zeros. */
489 if (binoptab
!= ashr_optab
)
490 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
492 if (!force_expand_binop (word_mode
, binoptab
, outof_input
,
493 gen_int_shift_amount (word_mode
,
495 outof_target
, unsignedp
, methods
))
501 /* This subroutine of expand_doubleword_shift handles the cases in which
502 the effective shift value is < BITS_PER_WORD. The arguments and return
503 value are the same as for the parent routine. */
506 expand_subword_shift (scalar_int_mode op1_mode
, optab binoptab
,
507 rtx outof_input
, rtx into_input
, rtx op1
,
508 rtx outof_target
, rtx into_target
,
509 int unsignedp
, enum optab_methods methods
,
510 unsigned HOST_WIDE_INT shift_mask
)
512 optab reverse_unsigned_shift
, unsigned_shift
;
515 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
516 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
518 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
519 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
520 the opposite direction to BINOPTAB. */
521 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
523 carries
= outof_input
;
524 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
525 op1_mode
), op1_mode
);
526 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
531 /* We must avoid shifting by BITS_PER_WORD bits since that is either
532 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
533 has unknown behavior. Do a single shift first, then shift by the
534 remainder. It's OK to use ~OP1 as the remainder if shift counts
535 are truncated to the mode size. */
536 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
537 outof_input
, const1_rtx
, 0, unsignedp
, methods
);
538 if (shift_mask
== BITS_PER_WORD
- 1)
540 tmp
= immed_wide_int_const
541 (wi::minus_one (GET_MODE_PRECISION (op1_mode
)), op1_mode
);
542 tmp
= simplify_expand_binop (op1_mode
, xor_optab
, op1
, tmp
,
547 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
548 op1_mode
), op1_mode
);
549 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
553 if (tmp
== 0 || carries
== 0)
555 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
556 carries
, tmp
, 0, unsignedp
, methods
);
560 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
561 so the result can go directly into INTO_TARGET if convenient. */
562 tmp
= expand_binop (word_mode
, unsigned_shift
, into_input
, op1
,
563 into_target
, unsignedp
, methods
);
567 /* Now OR in the bits carried over from OUTOF_INPUT. */
568 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
569 into_target
, unsignedp
, methods
))
572 /* Use a standard word_mode shift for the out-of half. */
573 if (outof_target
!= 0)
574 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
575 outof_target
, unsignedp
, methods
))
582 /* Try implementing expand_doubleword_shift using conditional moves.
583 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
584 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
585 are the shift counts to use in the former and latter case. All other
586 arguments are the same as the parent routine. */
589 expand_doubleword_shift_condmove (scalar_int_mode op1_mode
, optab binoptab
,
590 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
591 rtx outof_input
, rtx into_input
,
592 rtx subword_op1
, rtx superword_op1
,
593 rtx outof_target
, rtx into_target
,
594 int unsignedp
, enum optab_methods methods
,
595 unsigned HOST_WIDE_INT shift_mask
)
597 rtx outof_superword
, into_superword
;
599 /* Put the superword version of the output into OUTOF_SUPERWORD and
601 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
602 if (outof_target
!= 0 && subword_op1
== superword_op1
)
604 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
605 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
606 into_superword
= outof_target
;
607 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
608 outof_superword
, 0, unsignedp
, methods
))
613 into_superword
= gen_reg_rtx (word_mode
);
614 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
615 outof_superword
, into_superword
,
620 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
621 if (!expand_subword_shift (op1_mode
, binoptab
,
622 outof_input
, into_input
, subword_op1
,
623 outof_target
, into_target
,
624 unsignedp
, methods
, shift_mask
))
627 /* Select between them. Do the INTO half first because INTO_SUPERWORD
628 might be the current value of OUTOF_TARGET. */
629 if (!emit_conditional_move (into_target
, { cmp_code
, cmp1
, cmp2
, op1_mode
},
630 into_target
, into_superword
, word_mode
, false))
633 if (outof_target
!= 0)
634 if (!emit_conditional_move (outof_target
,
635 { cmp_code
, cmp1
, cmp2
, op1_mode
},
636 outof_target
, outof_superword
,
643 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
644 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
645 input operand; the shift moves bits in the direction OUTOF_INPUT->
646 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
647 of the target. OP1 is the shift count and OP1_MODE is its mode.
648 If OP1 is constant, it will have been truncated as appropriate
649 and is known to be nonzero.
651 If SHIFT_MASK is zero, the result of word shifts is undefined when the
652 shift count is outside the range [0, BITS_PER_WORD). This routine must
653 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
655 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
656 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
657 fill with zeros or sign bits as appropriate.
659 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
660 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
661 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
662 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
665 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
666 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
667 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
668 function wants to calculate it itself.
670 Return true if the shift could be successfully synthesized. */
673 expand_doubleword_shift (scalar_int_mode op1_mode
, optab binoptab
,
674 rtx outof_input
, rtx into_input
, rtx op1
,
675 rtx outof_target
, rtx into_target
,
676 int unsignedp
, enum optab_methods methods
,
677 unsigned HOST_WIDE_INT shift_mask
)
679 rtx superword_op1
, tmp
, cmp1
, cmp2
;
680 enum rtx_code cmp_code
;
682 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
683 fill the result with sign or zero bits as appropriate. If so, the value
684 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
685 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
686 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
688 This isn't worthwhile for constant shifts since the optimizers will
689 cope better with in-range shift counts. */
690 if (shift_mask
>= BITS_PER_WORD
692 && !CONSTANT_P (op1
))
694 if (!expand_doubleword_shift (op1_mode
, binoptab
,
695 outof_input
, into_input
, op1
,
697 unsignedp
, methods
, shift_mask
))
699 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
700 outof_target
, unsignedp
, methods
))
705 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
706 is true when the effective shift value is less than BITS_PER_WORD.
707 Set SUPERWORD_OP1 to the shift count that should be used to shift
708 OUTOF_INPUT into INTO_TARGET when the condition is false. */
709 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
710 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
712 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
713 is a subword shift count. */
714 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
716 cmp2
= CONST0_RTX (op1_mode
);
722 /* Set CMP1 to OP1 - BITS_PER_WORD. */
723 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
725 cmp2
= CONST0_RTX (op1_mode
);
727 superword_op1
= cmp1
;
732 /* If we can compute the condition at compile time, pick the
733 appropriate subroutine. */
734 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
735 if (tmp
!= 0 && CONST_INT_P (tmp
))
737 if (tmp
== const0_rtx
)
738 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
739 outof_target
, into_target
,
742 return expand_subword_shift (op1_mode
, binoptab
,
743 outof_input
, into_input
, op1
,
744 outof_target
, into_target
,
745 unsignedp
, methods
, shift_mask
);
748 /* Try using conditional moves to generate straight-line code. */
749 if (HAVE_conditional_move
)
751 rtx_insn
*start
= get_last_insn ();
752 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
753 cmp_code
, cmp1
, cmp2
,
754 outof_input
, into_input
,
756 outof_target
, into_target
,
757 unsignedp
, methods
, shift_mask
))
759 delete_insns_since (start
);
762 /* As a last resort, use branches to select the correct alternative. */
763 rtx_code_label
*subword_label
= gen_label_rtx ();
764 rtx_code_label
*done_label
= gen_label_rtx ();
767 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
769 profile_probability::uninitialized ());
772 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
773 outof_target
, into_target
,
777 emit_jump_insn (targetm
.gen_jump (done_label
));
779 emit_label (subword_label
);
781 if (!expand_subword_shift (op1_mode
, binoptab
,
782 outof_input
, into_input
, op1
,
783 outof_target
, into_target
,
784 unsignedp
, methods
, shift_mask
))
787 emit_label (done_label
);
791 /* Subroutine of expand_binop. Perform a double word multiplication of
792 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
793 as the target's word_mode. This function return NULL_RTX if anything
794 goes wrong, in which case it may have already emitted instructions
795 which need to be deleted.
797 If we want to multiply two two-word values and have normal and widening
798 multiplies of single-word values, we can do this with three smaller
801 The multiplication proceeds as follows:
802 _______________________
803 [__op0_high_|__op0_low__]
804 _______________________
805 * [__op1_high_|__op1_low__]
806 _______________________________________________
807 _______________________
808 (1) [__op0_low__*__op1_low__]
809 _______________________
810 (2a) [__op0_low__*__op1_high_]
811 _______________________
812 (2b) [__op0_high_*__op1_low__]
813 _______________________
814 (3) [__op0_high_*__op1_high_]
817 This gives a 4-word result. Since we are only interested in the
818 lower 2 words, partial result (3) and the upper words of (2a) and
819 (2b) don't need to be calculated. Hence (2a) and (2b) can be
820 calculated using non-widening multiplication.
822 (1), however, needs to be calculated with an unsigned widening
823 multiplication. If this operation is not directly supported we
824 try using a signed widening multiplication and adjust the result.
825 This adjustment works as follows:
827 If both operands are positive then no adjustment is needed.
829 If the operands have different signs, for example op0_low < 0 and
830 op1_low >= 0, the instruction treats the most significant bit of
831 op0_low as a sign bit instead of a bit with significance
832 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
833 with 2**BITS_PER_WORD - op0_low, and two's complements the
834 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
837 Similarly, if both operands are negative, we need to add
838 (op0_low + op1_low) * 2**BITS_PER_WORD.
840 We use a trick to adjust quickly. We logically shift op0_low right
841 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
842 op0_high (op1_high) before it is used to calculate 2b (2a). If no
843 logical shift exists, we do an arithmetic right shift and subtract
847 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
848 bool umulp
, enum optab_methods methods
)
850 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
851 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
852 rtx wordm1
= (umulp
? NULL_RTX
853 : gen_int_shift_amount (word_mode
, BITS_PER_WORD
- 1));
854 rtx product
, adjust
, product_high
, temp
;
856 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
857 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
858 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
859 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
861 /* If we're using an unsigned multiply to directly compute the product
862 of the low-order words of the operands and perform any required
863 adjustments of the operands, we begin by trying two more multiplications
864 and then computing the appropriate sum.
866 We have checked above that the required addition is provided.
867 Full-word addition will normally always succeed, especially if
868 it is provided at all, so we don't worry about its failure. The
869 multiplication may well fail, however, so we do handle that. */
873 /* ??? This could be done with emit_store_flag where available. */
874 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
875 NULL_RTX
, 1, methods
);
877 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
878 NULL_RTX
, 0, OPTAB_DIRECT
);
881 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
882 NULL_RTX
, 0, methods
);
885 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
886 NULL_RTX
, 0, OPTAB_DIRECT
);
893 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
894 NULL_RTX
, 0, OPTAB_DIRECT
);
898 /* OP0_HIGH should now be dead. */
902 /* ??? This could be done with emit_store_flag where available. */
903 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
904 NULL_RTX
, 1, methods
);
906 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
907 NULL_RTX
, 0, OPTAB_DIRECT
);
910 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
911 NULL_RTX
, 0, methods
);
914 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
915 NULL_RTX
, 0, OPTAB_DIRECT
);
922 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
923 NULL_RTX
, 0, OPTAB_DIRECT
);
927 /* OP1_HIGH should now be dead. */
929 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
930 NULL_RTX
, 0, OPTAB_DIRECT
);
932 if (target
&& !REG_P (target
))
935 /* *_widen_optab needs to determine operand mode, make sure at least
936 one operand has non-VOID mode. */
937 if (GET_MODE (op0_low
) == VOIDmode
&& GET_MODE (op1_low
) == VOIDmode
)
938 op0_low
= force_reg (word_mode
, op0_low
);
941 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
942 target
, 1, OPTAB_DIRECT
);
944 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
945 target
, 1, OPTAB_DIRECT
);
950 product_high
= operand_subword (product
, high
, 1, mode
);
951 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
952 NULL_RTX
, 0, OPTAB_DIRECT
);
953 emit_move_insn (product_high
, adjust
);
957 /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
958 constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
959 (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
960 computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
961 + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
962 depends on the bit value, if 2, then carry from the addition needs to be
963 added too, i.e. like:
964 sum += __builtin_add_overflow (low, high, &sum)
966 Optimize signed double-word OP0 % OP1 similarly, just apply some correction
967 factor to the sum before doing unsigned remainder, in the form of
968 sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
969 then perform unsigned
970 remainder = sum % OP1;
972 remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
975 expand_doubleword_mod (machine_mode mode
, rtx op0
, rtx op1
, bool unsignedp
)
977 if (INTVAL (op1
) <= 1 || (INTVAL (op1
) & 1) == 0)
980 rtx_insn
*last
= get_last_insn ();
981 for (int bit
= BITS_PER_WORD
; bit
>= BITS_PER_WORD
/ 2; bit
--)
983 wide_int w
= wi::shifted_mask (bit
, 1, false, 2 * BITS_PER_WORD
);
984 if (wi::ne_p (wi::umod_trunc (w
, INTVAL (op1
)), 1))
986 rtx sum
= NULL_RTX
, mask
= NULL_RTX
;
987 if (bit
== BITS_PER_WORD
)
989 /* For signed modulo we need to add correction to the sum
990 and that might again overflow. */
993 if (optab_handler (uaddv4_optab
, word_mode
) == CODE_FOR_nothing
)
995 tree wtype
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
996 if (wtype
== NULL_TREE
)
998 tree ctype
= build_complex_type (wtype
);
999 if (TYPE_MODE (ctype
) != GET_MODE_COMPLEX_MODE (word_mode
))
1001 machine_mode cmode
= TYPE_MODE (ctype
);
1002 rtx op00
= operand_subword_force (op0
, 0, mode
);
1003 rtx op01
= operand_subword_force (op0
, 1, mode
);
1004 rtx cres
= gen_rtx_CONCAT (cmode
, gen_reg_rtx (word_mode
),
1005 gen_reg_rtx (word_mode
));
1006 tree lhs
= make_tree (ctype
, cres
);
1007 tree arg0
= make_tree (wtype
, op00
);
1008 tree arg1
= make_tree (wtype
, op01
);
1009 expand_addsub_overflow (UNKNOWN_LOCATION
, PLUS_EXPR
, lhs
, arg0
,
1010 arg1
, true, true, true, false, NULL
);
1011 sum
= expand_simple_binop (word_mode
, PLUS
, XEXP (cres
, 0),
1012 XEXP (cres
, 1), NULL_RTX
, 1,
1014 if (sum
== NULL_RTX
)
1019 /* Code below uses GEN_INT, so we need the masks to be representable
1020 in HOST_WIDE_INTs. */
1021 if (bit
>= HOST_BITS_PER_WIDE_INT
)
1023 /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1024 overflow. Consider 64-bit -1ULL for word size 32, if we add
1025 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1026 if (bit
== BITS_PER_WORD
- 1)
1029 int count
= (2 * BITS_PER_WORD
+ bit
- 1) / bit
;
1030 rtx sum_corr
= NULL_RTX
;
1034 /* For signed modulo, compute it as unsigned modulo of
1035 sum with a correction added to it if OP0 is negative,
1036 such that the result can be computed as unsigned
1037 remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1038 w
= wi::min_value (2 * BITS_PER_WORD
, SIGNED
);
1039 wide_int wmod1
= wi::umod_trunc (w
, INTVAL (op1
));
1040 wide_int wmod2
= wi::smod_trunc (w
, INTVAL (op1
));
1041 /* wmod2 == -wmod1. */
1042 wmod2
= wmod2
+ (INTVAL (op1
) - 1);
1043 if (wi::ne_p (wmod1
, wmod2
))
1045 wide_int wcorr
= wmod2
- wmod1
;
1047 wcorr
= wcorr
+ INTVAL (op1
);
1048 /* Now verify if the count sums can't overflow, and punt
1050 w
= wi::mask (bit
, false, 2 * BITS_PER_WORD
);
1051 w
= w
* (count
- 1);
1052 w
= w
+ wi::mask (2 * BITS_PER_WORD
- (count
- 1) * bit
,
1053 false, 2 * BITS_PER_WORD
);
1055 w
= wi::lrshift (w
, BITS_PER_WORD
);
1056 if (wi::ne_p (w
, 0))
1059 mask
= operand_subword_force (op0
, WORDS_BIG_ENDIAN
? 0 : 1,
1061 mask
= expand_simple_binop (word_mode
, ASHIFTRT
, mask
,
1062 GEN_INT (BITS_PER_WORD
- 1),
1063 NULL_RTX
, 0, OPTAB_DIRECT
);
1064 if (mask
== NULL_RTX
)
1066 sum_corr
= immed_wide_int_const (wcorr
, word_mode
);
1067 sum_corr
= expand_simple_binop (word_mode
, AND
, mask
,
1068 sum_corr
, NULL_RTX
, 1,
1070 if (sum_corr
== NULL_RTX
)
1075 for (int i
= 0; i
< count
; i
++)
1079 v
= expand_simple_binop (mode
, LSHIFTRT
, v
, GEN_INT (i
* bit
),
1080 NULL_RTX
, 1, OPTAB_DIRECT
);
1083 v
= lowpart_subreg (word_mode
, v
, mode
);
1087 v
= expand_simple_binop (word_mode
, AND
, v
,
1088 GEN_INT ((HOST_WIDE_INT_1U
<< bit
)
1093 if (sum
== NULL_RTX
)
1096 sum
= expand_simple_binop (word_mode
, PLUS
, sum
, v
, NULL_RTX
,
1098 if (sum
== NULL_RTX
)
1103 sum
= expand_simple_binop (word_mode
, PLUS
, sum
, sum_corr
,
1104 NULL_RTX
, 1, OPTAB_DIRECT
);
1105 if (sum
== NULL_RTX
)
1109 rtx remainder
= expand_divmod (1, TRUNC_MOD_EXPR
, word_mode
, sum
,
1110 gen_int_mode (INTVAL (op1
), word_mode
),
1111 NULL_RTX
, 1, OPTAB_DIRECT
);
1112 if (remainder
== NULL_RTX
)
1117 if (mask
== NULL_RTX
)
1119 mask
= operand_subword_force (op0
, WORDS_BIG_ENDIAN
? 0 : 1,
1121 mask
= expand_simple_binop (word_mode
, ASHIFTRT
, mask
,
1122 GEN_INT (BITS_PER_WORD
- 1),
1123 NULL_RTX
, 0, OPTAB_DIRECT
);
1124 if (mask
== NULL_RTX
)
1127 mask
= expand_simple_binop (word_mode
, AND
, mask
,
1128 gen_int_mode (1 - INTVAL (op1
),
1130 NULL_RTX
, 1, OPTAB_DIRECT
);
1131 if (mask
== NULL_RTX
)
1133 remainder
= expand_simple_binop (word_mode
, PLUS
, remainder
,
1134 mask
, NULL_RTX
, 1, OPTAB_DIRECT
);
1135 if (remainder
== NULL_RTX
)
1139 remainder
= convert_modes (mode
, word_mode
, remainder
, unsignedp
);
1140 /* Punt if we need any library calls. */
1142 last
= NEXT_INSN (last
);
1144 last
= get_insns ();
1145 for (; last
; last
= NEXT_INSN (last
))
1153 /* Similarly to the above function, but compute both quotient and remainder.
1154 Quotient can be computed from the remainder as:
1155 rem = op0 % op1; // Handled using expand_doubleword_mod
1156 quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1157 // 2 * BITS_PER_WORD
1159 We can also handle cases where op1 is a multiple of power of two constant
1160 and constant handled by expand_doubleword_mod.
1161 op11 = 1 << __builtin_ctz (op1);
1163 rem1 = op0 % op12; // Handled using expand_doubleword_mod
1164 quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1165 // 2 * BITS_PER_WORD
1166 rem = (quot1 % op11) * op12 + rem1;
1167 quot = quot1 / op11; */
1170 expand_doubleword_divmod (machine_mode mode
, rtx op0
, rtx op1
, rtx
*rem
,
1175 /* Negative dividend should have been optimized into positive,
1176 similarly modulo by 1 and modulo by power of two is optimized
1178 if (INTVAL (op1
) <= 1 || pow2p_hwi (INTVAL (op1
)))
1181 rtx op11
= const1_rtx
;
1183 if ((INTVAL (op1
) & 1) == 0)
1185 int bit
= ctz_hwi (INTVAL (op1
));
1186 op11
= GEN_INT (HOST_WIDE_INT_1
<< bit
);
1187 op12
= GEN_INT (INTVAL (op1
) >> bit
);
1190 rtx rem1
= expand_doubleword_mod (mode
, op0
, op12
, unsignedp
);
1191 if (rem1
== NULL_RTX
)
1194 int prec
= 2 * BITS_PER_WORD
;
1195 wide_int a
= wide_int::from (INTVAL (op12
), prec
+ 1, UNSIGNED
);
1196 wide_int b
= wi::shifted_mask (prec
, 1, false, prec
+ 1);
1197 wide_int m
= wide_int::from (wi::mod_inv (a
, b
), prec
, UNSIGNED
);
1198 rtx inv
= immed_wide_int_const (m
, mode
);
1200 rtx_insn
*last
= get_last_insn ();
1201 rtx quot1
= expand_simple_binop (mode
, MINUS
, op0
, rem1
,
1202 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1203 if (quot1
== NULL_RTX
)
1206 quot1
= expand_simple_binop (mode
, MULT
, quot1
, inv
,
1207 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1208 if (quot1
== NULL_RTX
)
1211 if (op11
!= const1_rtx
)
1213 rtx rem2
= expand_divmod (1, TRUNC_MOD_EXPR
, mode
, quot1
, op11
,
1214 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1215 if (rem2
== NULL_RTX
)
1218 rem2
= expand_simple_binop (mode
, MULT
, rem2
, op12
, NULL_RTX
,
1219 unsignedp
, OPTAB_DIRECT
);
1220 if (rem2
== NULL_RTX
)
1223 rem2
= expand_simple_binop (mode
, PLUS
, rem2
, rem1
, NULL_RTX
,
1224 unsignedp
, OPTAB_DIRECT
);
1225 if (rem2
== NULL_RTX
)
1228 rtx quot2
= expand_divmod (0, TRUNC_DIV_EXPR
, mode
, quot1
, op11
,
1229 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1230 if (quot2
== NULL_RTX
)
1237 /* Punt if we need any library calls. */
1239 last
= NEXT_INSN (last
);
1241 last
= get_insns ();
1242 for (; last
; last
= NEXT_INSN (last
))
1250 /* Wrapper around expand_binop which takes an rtx code to specify
1251 the operation to perform, not an optab pointer. All other
1252 arguments are the same. */
1254 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
1255 rtx op1
, rtx target
, int unsignedp
,
1256 enum optab_methods methods
)
1258 optab binop
= code_to_optab (code
);
1261 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
1264 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1265 binop. Order them according to commutative_operand_precedence and, if
1266 possible, try to put TARGET or a pseudo first. */
1268 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
1270 int op0_prec
= commutative_operand_precedence (op0
);
1271 int op1_prec
= commutative_operand_precedence (op1
);
1273 if (op0_prec
< op1_prec
)
1276 if (op0_prec
> op1_prec
)
1279 /* With equal precedence, both orders are ok, but it is better if the
1280 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1281 if (target
== 0 || REG_P (target
))
1282 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
1284 return rtx_equal_p (op1
, target
);
1287 /* Return true if BINOPTAB implements a shift operation. */
1290 shift_optab_p (optab binoptab
)
1292 switch (optab_to_code (binoptab
))
1308 /* Return true if BINOPTAB implements a commutative binary operation. */
1311 commutative_optab_p (optab binoptab
)
1313 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
1314 || binoptab
== smul_widen_optab
1315 || binoptab
== umul_widen_optab
1316 || binoptab
== smul_highpart_optab
1317 || binoptab
== umul_highpart_optab
1318 || binoptab
== vec_widen_sadd_optab
1319 || binoptab
== vec_widen_uadd_optab
1320 || binoptab
== vec_widen_sadd_hi_optab
1321 || binoptab
== vec_widen_sadd_lo_optab
1322 || binoptab
== vec_widen_uadd_hi_optab
1323 || binoptab
== vec_widen_uadd_lo_optab
1324 || binoptab
== vec_widen_sadd_even_optab
1325 || binoptab
== vec_widen_sadd_odd_optab
1326 || binoptab
== vec_widen_uadd_even_optab
1327 || binoptab
== vec_widen_uadd_odd_optab
);
1330 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1331 optimizing, and if the operand is a constant that costs more than
1332 1 instruction, force the constant into a register and return that
1333 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1336 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
1337 int opn
, rtx x
, bool unsignedp
)
1339 bool speed
= optimize_insn_for_speed_p ();
1341 if (mode
!= VOIDmode
1344 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
1345 > set_src_cost (x
, mode
, speed
)))
1347 if (CONST_INT_P (x
))
1349 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
1350 if (intval
!= INTVAL (x
))
1351 x
= GEN_INT (intval
);
1354 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
1355 x
= force_reg (mode
, x
);
1360 /* Helper function for expand_binop: handle the case where there
1361 is an insn ICODE that directly implements the indicated operation.
1362 Returns null if this is not possible. */
1364 expand_binop_directly (enum insn_code icode
, machine_mode mode
, optab binoptab
,
1366 rtx target
, int unsignedp
, enum optab_methods methods
,
1369 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
1370 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
1371 machine_mode mode0
, mode1
, tmp_mode
;
1372 class expand_operand ops
[3];
1375 rtx xop0
= op0
, xop1
= op1
;
1376 bool canonicalize_op1
= false;
1378 /* If it is a commutative operator and the modes would match
1379 if we would swap the operands, we can save the conversions. */
1380 commutative_p
= commutative_optab_p (binoptab
);
1382 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1383 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode0
)
1384 std::swap (xop0
, xop1
);
1386 /* If we are optimizing, force expensive constants into a register. */
1387 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1388 if (!shift_optab_p (binoptab
))
1389 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1391 /* Shifts and rotates often use a different mode for op1 from op0;
1392 for VOIDmode constants we don't know the mode, so force it
1393 to be canonicalized using convert_modes. */
1394 canonicalize_op1
= true;
1396 /* In case the insn wants input operands in modes different from
1397 those of the actual operands, convert the operands. It would
1398 seem that we don't need to convert CONST_INTs, but we do, so
1399 that they're properly zero-extended, sign-extended or truncated
1402 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1403 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1405 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1409 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1410 ? GET_MODE (xop1
) : mode
);
1411 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1413 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1417 /* If operation is commutative,
1418 try to make the first operand a register.
1419 Even better, try to make it the same as the target.
1420 Also try to make the last operand a constant. */
1422 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1423 std::swap (xop0
, xop1
);
1425 /* Now, if insn's predicates don't allow our operands, put them into
1428 if (binoptab
== vec_pack_trunc_optab
1429 || binoptab
== vec_pack_usat_optab
1430 || binoptab
== vec_pack_ssat_optab
1431 || binoptab
== vec_pack_ufix_trunc_optab
1432 || binoptab
== vec_pack_sfix_trunc_optab
1433 || binoptab
== vec_packu_float_optab
1434 || binoptab
== vec_packs_float_optab
)
1436 /* The mode of the result is different then the mode of the
1438 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1439 if (VECTOR_MODE_P (mode
)
1440 && maybe_ne (GET_MODE_NUNITS (tmp_mode
), 2 * GET_MODE_NUNITS (mode
)))
1442 delete_insns_since (last
);
1449 create_output_operand (&ops
[0], target
, tmp_mode
);
1450 create_input_operand (&ops
[1], xop0
, mode0
);
1451 create_input_operand (&ops
[2], xop1
, mode1
);
1452 pat
= maybe_gen_insn (icode
, 3, ops
);
1455 /* If PAT is composed of more than one insn, try to add an appropriate
1456 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1457 operand, call expand_binop again, this time without a target. */
1458 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1459 && ! add_equal_note (pat
, ops
[0].value
,
1460 optab_to_code (binoptab
),
1461 ops
[1].value
, ops
[2].value
, mode0
))
1463 delete_insns_since (last
);
1464 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1465 unsignedp
, methods
);
1469 return ops
[0].value
;
1471 delete_insns_since (last
);
1475 /* Generate code to perform an operation specified by BINOPTAB
1476 on operands OP0 and OP1, with result having machine-mode MODE.
1478 UNSIGNEDP is for the case where we have to widen the operands
1479 to perform the operation. It says to use zero-extension.
1481 If TARGET is nonzero, the value
1482 is generated there, if it is convenient to do so.
1483 In all cases an rtx is returned for the locus of the value;
1484 this may or may not be TARGET. */
1487 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1488 rtx target
, int unsignedp
, enum optab_methods methods
)
1490 enum optab_methods next_methods
1491 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1492 ? OPTAB_WIDEN
: methods
);
1493 enum mode_class mclass
;
1494 enum insn_code icode
;
1495 machine_mode wider_mode
;
1496 scalar_int_mode int_mode
;
1499 rtx_insn
*entry_last
= get_last_insn ();
1502 mclass
= GET_MODE_CLASS (mode
);
1504 /* If subtracting an integer constant, convert this into an addition of
1505 the negated constant. */
1507 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1509 op1
= negate_rtx (mode
, op1
);
1510 binoptab
= add_optab
;
1512 /* For shifts, constant invalid op1 might be expanded from different
1513 mode than MODE. As those are invalid, force them to a register
1514 to avoid further problems during expansion. */
1515 else if (CONST_INT_P (op1
)
1516 && shift_optab_p (binoptab
)
1517 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1519 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1520 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1523 /* Record where to delete back to if we backtrack. */
1524 last
= get_last_insn ();
1526 /* If we can do it with a three-operand insn, do so. */
1528 if (methods
!= OPTAB_MUST_WIDEN
)
1530 if (convert_optab_p (binoptab
))
1532 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
1533 icode
= find_widening_optab_handler (binoptab
, mode
, from_mode
);
1536 icode
= optab_handler (binoptab
, mode
);
1537 if (icode
!= CODE_FOR_nothing
)
1539 temp
= expand_binop_directly (icode
, mode
, binoptab
, op0
, op1
,
1540 target
, unsignedp
, methods
, last
);
1546 /* If we were trying to rotate, and that didn't work, try rotating
1547 the other direction before falling back to shifts and bitwise-or. */
1548 if (((binoptab
== rotl_optab
1549 && (icode
= optab_handler (rotr_optab
, mode
)) != CODE_FOR_nothing
)
1550 || (binoptab
== rotr_optab
1551 && (icode
= optab_handler (rotl_optab
, mode
)) != CODE_FOR_nothing
))
1552 && is_int_mode (mode
, &int_mode
))
1554 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1556 unsigned int bits
= GET_MODE_PRECISION (int_mode
);
1558 if (CONST_INT_P (op1
))
1559 newop1
= gen_int_shift_amount (int_mode
, bits
- INTVAL (op1
));
1560 else if (targetm
.shift_truncation_mask (int_mode
) == bits
- 1)
1561 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1563 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1564 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1565 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1567 temp
= expand_binop_directly (icode
, int_mode
, otheroptab
, op0
, newop1
,
1568 target
, unsignedp
, methods
, last
);
1573 /* If this is a multiply, see if we can do a widening operation that
1574 takes operands of this mode and makes a wider mode. */
1576 if (binoptab
== smul_optab
1577 && GET_MODE_2XWIDER_MODE (mode
).exists (&wider_mode
)
1578 && (convert_optab_handler ((unsignedp
1580 : smul_widen_optab
),
1581 wider_mode
, mode
) != CODE_FOR_nothing
))
1583 /* *_widen_optab needs to determine operand mode, make sure at least
1584 one operand has non-VOID mode. */
1585 if (GET_MODE (op0
) == VOIDmode
&& GET_MODE (op1
) == VOIDmode
)
1586 op0
= force_reg (mode
, op0
);
1587 temp
= expand_binop (wider_mode
,
1588 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1589 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1593 if (GET_MODE_CLASS (mode
) == MODE_INT
1594 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1595 return gen_lowpart (mode
, temp
);
1597 return convert_to_mode (mode
, temp
, unsignedp
);
1601 /* If this is a vector shift by a scalar, see if we can do a vector
1602 shift by a vector. If so, broadcast the scalar into a vector. */
1603 if (mclass
== MODE_VECTOR_INT
)
1605 optab otheroptab
= unknown_optab
;
1607 if (binoptab
== ashl_optab
)
1608 otheroptab
= vashl_optab
;
1609 else if (binoptab
== ashr_optab
)
1610 otheroptab
= vashr_optab
;
1611 else if (binoptab
== lshr_optab
)
1612 otheroptab
= vlshr_optab
;
1613 else if (binoptab
== rotl_optab
)
1614 otheroptab
= vrotl_optab
;
1615 else if (binoptab
== rotr_optab
)
1616 otheroptab
= vrotr_optab
;
1619 && (icode
= optab_handler (otheroptab
, mode
)) != CODE_FOR_nothing
)
1621 /* The scalar may have been extended to be too wide. Truncate
1622 it back to the proper size to fit in the broadcast vector. */
1623 scalar_mode inner_mode
= GET_MODE_INNER (mode
);
1624 if (!CONST_INT_P (op1
)
1625 && (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (op1
)))
1626 > GET_MODE_BITSIZE (inner_mode
)))
1627 op1
= force_reg (inner_mode
,
1628 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1630 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1633 temp
= expand_binop_directly (icode
, mode
, otheroptab
, op0
, vop1
,
1634 target
, unsignedp
, methods
, last
);
1641 /* Look for a wider mode of the same class for which we think we
1642 can open-code the operation. Check for a widening multiply at the
1643 wider mode as well. */
1645 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1646 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1647 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1649 machine_mode next_mode
;
1650 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1651 || (binoptab
== smul_optab
1652 && GET_MODE_WIDER_MODE (wider_mode
).exists (&next_mode
)
1653 && (find_widening_optab_handler ((unsignedp
1655 : smul_widen_optab
),
1657 != CODE_FOR_nothing
)))
1659 rtx xop0
= op0
, xop1
= op1
;
1662 /* For certain integer operations, we need not actually extend
1663 the narrow operands, as long as we will truncate
1664 the results to the same narrowness. */
1666 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1667 || binoptab
== xor_optab
1668 || binoptab
== add_optab
|| binoptab
== sub_optab
1669 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1670 && mclass
== MODE_INT
)
1673 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1675 if (binoptab
!= ashl_optab
)
1676 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1680 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1682 /* The second operand of a shift must always be extended. */
1683 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1684 no_extend
&& binoptab
!= ashl_optab
);
1686 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1687 unsignedp
, OPTAB_DIRECT
);
1690 if (mclass
!= MODE_INT
1691 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1694 target
= gen_reg_rtx (mode
);
1695 convert_move (target
, temp
, 0);
1699 return gen_lowpart (mode
, temp
);
1702 delete_insns_since (last
);
1706 /* If operation is commutative,
1707 try to make the first operand a register.
1708 Even better, try to make it the same as the target.
1709 Also try to make the last operand a constant. */
1710 if (commutative_optab_p (binoptab
)
1711 && swap_commutative_operands_with_target (target
, op0
, op1
))
1712 std::swap (op0
, op1
);
1714 /* These can be done a word at a time. */
1715 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1716 && is_int_mode (mode
, &int_mode
)
1717 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
1718 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1723 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1724 won't be accurate, so use a new target. */
1728 || reg_overlap_mentioned_p (target
, op0
)
1729 || reg_overlap_mentioned_p (target
, op1
)
1730 || !valid_multiword_target_p (target
))
1731 target
= gen_reg_rtx (int_mode
);
1735 /* Do the actual arithmetic. */
1736 machine_mode op0_mode
= GET_MODE (op0
);
1737 machine_mode op1_mode
= GET_MODE (op1
);
1738 if (op0_mode
== VOIDmode
)
1739 op0_mode
= int_mode
;
1740 if (op1_mode
== VOIDmode
)
1741 op1_mode
= int_mode
;
1742 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
1744 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
1745 rtx x
= expand_binop (word_mode
, binoptab
,
1746 operand_subword_force (op0
, i
, op0_mode
),
1747 operand_subword_force (op1
, i
, op1_mode
),
1748 target_piece
, unsignedp
, next_methods
);
1753 if (target_piece
!= x
)
1754 emit_move_insn (target_piece
, x
);
1757 insns
= get_insns ();
1760 if (i
== GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
)
1767 /* Synthesize double word shifts from single word shifts. */
1768 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1769 || binoptab
== ashr_optab
)
1770 && is_int_mode (mode
, &int_mode
)
1771 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1772 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1773 && GET_MODE_PRECISION (int_mode
) == GET_MODE_BITSIZE (int_mode
)
1774 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1775 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1776 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1778 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1779 scalar_int_mode op1_mode
;
1781 double_shift_mask
= targetm
.shift_truncation_mask (int_mode
);
1782 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1783 op1_mode
= (GET_MODE (op1
) != VOIDmode
1784 ? as_a
<scalar_int_mode
> (GET_MODE (op1
))
1787 /* Apply the truncation to constant shifts. */
1788 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1789 op1
= gen_int_mode (INTVAL (op1
) & double_shift_mask
, op1_mode
);
1791 if (op1
== CONST0_RTX (op1_mode
))
1794 /* Make sure that this is a combination that expand_doubleword_shift
1795 can handle. See the comments there for details. */
1796 if (double_shift_mask
== 0
1797 || (shift_mask
== BITS_PER_WORD
- 1
1798 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1801 rtx into_target
, outof_target
;
1802 rtx into_input
, outof_input
;
1803 int left_shift
, outof_word
;
1805 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1806 won't be accurate, so use a new target. */
1810 || reg_overlap_mentioned_p (target
, op0
)
1811 || reg_overlap_mentioned_p (target
, op1
)
1812 || !valid_multiword_target_p (target
))
1813 target
= gen_reg_rtx (int_mode
);
1817 /* OUTOF_* is the word we are shifting bits away from, and
1818 INTO_* is the word that we are shifting bits towards, thus
1819 they differ depending on the direction of the shift and
1820 WORDS_BIG_ENDIAN. */
1822 left_shift
= binoptab
== ashl_optab
;
1823 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1825 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1826 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1828 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1829 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1831 if (expand_doubleword_shift (op1_mode
, binoptab
,
1832 outof_input
, into_input
, op1
,
1833 outof_target
, into_target
,
1834 unsignedp
, next_methods
, shift_mask
))
1836 insns
= get_insns ();
1846 /* Synthesize double word rotates from single word shifts. */
1847 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1848 && is_int_mode (mode
, &int_mode
)
1849 && CONST_INT_P (op1
)
1850 && GET_MODE_PRECISION (int_mode
) == 2 * BITS_PER_WORD
1851 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1852 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1855 rtx into_target
, outof_target
;
1856 rtx into_input
, outof_input
;
1858 int shift_count
, left_shift
, outof_word
;
1860 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1861 won't be accurate, so use a new target. Do this also if target is not
1862 a REG, first because having a register instead may open optimization
1863 opportunities, and second because if target and op0 happen to be MEMs
1864 designating the same location, we would risk clobbering it too early
1865 in the code sequence we generate below. */
1870 || reg_overlap_mentioned_p (target
, op0
)
1871 || reg_overlap_mentioned_p (target
, op1
)
1872 || !valid_multiword_target_p (target
))
1873 target
= gen_reg_rtx (int_mode
);
1877 shift_count
= INTVAL (op1
);
1879 /* OUTOF_* is the word we are shifting bits away from, and
1880 INTO_* is the word that we are shifting bits towards, thus
1881 they differ depending on the direction of the shift and
1882 WORDS_BIG_ENDIAN. */
1884 left_shift
= (binoptab
== rotl_optab
);
1885 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1887 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1888 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1890 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1891 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1893 if (shift_count
== BITS_PER_WORD
)
1895 /* This is just a word swap. */
1896 emit_move_insn (outof_target
, into_input
);
1897 emit_move_insn (into_target
, outof_input
);
1902 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1903 HOST_WIDE_INT first_shift_count
, second_shift_count
;
1904 optab reverse_unsigned_shift
, unsigned_shift
;
1906 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1907 ? lshr_optab
: ashl_optab
);
1909 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1910 ? ashl_optab
: lshr_optab
);
1912 if (shift_count
> BITS_PER_WORD
)
1914 first_shift_count
= shift_count
- BITS_PER_WORD
;
1915 second_shift_count
= 2 * BITS_PER_WORD
- shift_count
;
1919 first_shift_count
= BITS_PER_WORD
- shift_count
;
1920 second_shift_count
= shift_count
;
1922 rtx first_shift_count_rtx
1923 = gen_int_shift_amount (word_mode
, first_shift_count
);
1924 rtx second_shift_count_rtx
1925 = gen_int_shift_amount (word_mode
, second_shift_count
);
1927 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1928 outof_input
, first_shift_count_rtx
,
1929 NULL_RTX
, unsignedp
, next_methods
);
1930 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1931 into_input
, second_shift_count_rtx
,
1932 NULL_RTX
, unsignedp
, next_methods
);
1934 if (into_temp1
!= 0 && into_temp2
!= 0)
1935 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1936 into_target
, unsignedp
, next_methods
);
1940 if (inter
!= 0 && inter
!= into_target
)
1941 emit_move_insn (into_target
, inter
);
1943 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1944 into_input
, first_shift_count_rtx
,
1945 NULL_RTX
, unsignedp
, next_methods
);
1946 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1947 outof_input
, second_shift_count_rtx
,
1948 NULL_RTX
, unsignedp
, next_methods
);
1950 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1951 inter
= expand_binop (word_mode
, ior_optab
,
1952 outof_temp1
, outof_temp2
,
1953 outof_target
, unsignedp
, next_methods
);
1955 if (inter
!= 0 && inter
!= outof_target
)
1956 emit_move_insn (outof_target
, inter
);
1959 insns
= get_insns ();
1969 /* These can be done a word at a time by propagating carries. */
1970 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1971 && is_int_mode (mode
, &int_mode
)
1972 && GET_MODE_SIZE (int_mode
) >= 2 * UNITS_PER_WORD
1973 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1976 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1977 const unsigned int nwords
= GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
;
1978 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1979 rtx xop0
, xop1
, xtarget
;
1981 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1982 value is one of those, use it. Otherwise, use 1 since it is the
1983 one easiest to get. */
1984 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1985 int normalizep
= STORE_FLAG_VALUE
;
1990 /* Prepare the operands. */
1991 xop0
= force_reg (int_mode
, op0
);
1992 xop1
= force_reg (int_mode
, op1
);
1994 xtarget
= gen_reg_rtx (int_mode
);
1996 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1999 /* Indicate for flow that the entire target reg is being set. */
2001 emit_clobber (xtarget
);
2003 /* Do the actual arithmetic. */
2004 for (i
= 0; i
< nwords
; i
++)
2006 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
2007 rtx target_piece
= operand_subword (xtarget
, index
, 1, int_mode
);
2008 rtx op0_piece
= operand_subword_force (xop0
, index
, int_mode
);
2009 rtx op1_piece
= operand_subword_force (xop1
, index
, int_mode
);
2012 /* Main add/subtract of the input operands. */
2013 x
= expand_binop (word_mode
, binoptab
,
2014 op0_piece
, op1_piece
,
2015 target_piece
, unsignedp
, next_methods
);
2021 /* Store carry from main add/subtract. */
2022 carry_out
= gen_reg_rtx (word_mode
);
2023 carry_out
= emit_store_flag_force (carry_out
,
2024 (binoptab
== add_optab
2027 word_mode
, 1, normalizep
);
2034 /* Add/subtract previous carry to main result. */
2035 newx
= expand_binop (word_mode
,
2036 normalizep
== 1 ? binoptab
: otheroptab
,
2038 NULL_RTX
, 1, next_methods
);
2042 /* Get out carry from adding/subtracting carry in. */
2043 rtx carry_tmp
= gen_reg_rtx (word_mode
);
2044 carry_tmp
= emit_store_flag_force (carry_tmp
,
2045 (binoptab
== add_optab
2048 word_mode
, 1, normalizep
);
2050 /* Logical-ior the two poss. carry together. */
2051 carry_out
= expand_binop (word_mode
, ior_optab
,
2052 carry_out
, carry_tmp
,
2053 carry_out
, 0, next_methods
);
2057 emit_move_insn (target_piece
, newx
);
2061 if (x
!= target_piece
)
2062 emit_move_insn (target_piece
, x
);
2065 carry_in
= carry_out
;
2068 if (i
== GET_MODE_BITSIZE (int_mode
) / (unsigned) BITS_PER_WORD
)
2070 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
2071 || ! rtx_equal_p (target
, xtarget
))
2073 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
2075 set_dst_reg_note (temp
, REG_EQUAL
,
2076 gen_rtx_fmt_ee (optab_to_code (binoptab
),
2077 int_mode
, copy_rtx (xop0
),
2088 delete_insns_since (last
);
2091 /* Attempt to synthesize double word multiplies using a sequence of word
2092 mode multiplications. We first attempt to generate a sequence using a
2093 more efficient unsigned widening multiply, and if that fails we then
2094 try using a signed widening multiply. */
2096 if (binoptab
== smul_optab
2097 && is_int_mode (mode
, &int_mode
)
2098 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2099 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
2100 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
2102 rtx product
= NULL_RTX
;
2103 if (convert_optab_handler (umul_widen_optab
, int_mode
, word_mode
)
2104 != CODE_FOR_nothing
)
2106 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
2109 delete_insns_since (last
);
2112 if (product
== NULL_RTX
2113 && (convert_optab_handler (smul_widen_optab
, int_mode
, word_mode
)
2114 != CODE_FOR_nothing
))
2116 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
2119 delete_insns_since (last
);
2122 if (product
!= NULL_RTX
)
2124 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
2126 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
2128 set_dst_reg_note (move
,
2130 gen_rtx_fmt_ee (MULT
, int_mode
,
2133 target
? target
: product
);
2139 /* Attempt to synthetize double word modulo by constant divisor. */
2140 if ((binoptab
== umod_optab
2141 || binoptab
== smod_optab
2142 || binoptab
== udiv_optab
2143 || binoptab
== sdiv_optab
)
2145 && CONST_INT_P (op1
)
2146 && is_int_mode (mode
, &int_mode
)
2147 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2148 && optab_handler ((binoptab
== umod_optab
|| binoptab
== udiv_optab
)
2149 ? udivmod_optab
: sdivmod_optab
,
2150 int_mode
) == CODE_FOR_nothing
2151 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
2152 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
2153 && optimize_insn_for_speed_p ())
2156 if ((binoptab
== umod_optab
|| binoptab
== smod_optab
)
2157 && (INTVAL (op1
) & 1) == 0)
2158 res
= expand_doubleword_mod (int_mode
, op0
, op1
,
2159 binoptab
== umod_optab
);
2162 rtx quot
= expand_doubleword_divmod (int_mode
, op0
, op1
, &res
,
2163 binoptab
== umod_optab
2164 || binoptab
== udiv_optab
);
2165 if (quot
== NULL_RTX
)
2167 else if (binoptab
== udiv_optab
|| binoptab
== sdiv_optab
)
2170 if (res
!= NULL_RTX
)
2172 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
2174 rtx_insn
*move
= emit_move_insn (target
? target
: res
,
2176 set_dst_reg_note (move
, REG_EQUAL
,
2177 gen_rtx_fmt_ee (optab_to_code (binoptab
),
2178 int_mode
, copy_rtx (op0
), op1
),
2179 target
? target
: res
);
2184 delete_insns_since (last
);
2187 /* It can't be open-coded in this mode.
2188 Use a library call if one is available and caller says that's ok. */
2190 libfunc
= optab_libfunc (binoptab
, mode
);
2192 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
2196 machine_mode op1_mode
= mode
;
2201 if (shift_optab_p (binoptab
))
2203 op1_mode
= targetm
.libgcc_shift_count_mode ();
2204 /* Specify unsigned here,
2205 since negative shift counts are meaningless. */
2206 op1x
= convert_to_mode (op1_mode
, op1
, 1);
2209 if (GET_MODE (op0
) != VOIDmode
2210 && GET_MODE (op0
) != mode
)
2211 op0
= convert_to_mode (mode
, op0
, unsignedp
);
2213 /* Pass 1 for NO_QUEUE so we don't lose any increments
2214 if the libcall is cse'd or moved. */
2215 value
= emit_library_call_value (libfunc
,
2216 NULL_RTX
, LCT_CONST
, mode
,
2217 op0
, mode
, op1x
, op1_mode
);
2219 insns
= get_insns ();
2222 bool trapv
= trapv_binoptab_p (binoptab
);
2223 target
= gen_reg_rtx (mode
);
2224 emit_libcall_block_1 (insns
, target
, value
,
2226 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
2227 mode
, op0
, op1
), trapv
);
2232 delete_insns_since (last
);
2234 /* It can't be done in this mode. Can we do it in a wider mode? */
2236 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
2237 || methods
== OPTAB_MUST_WIDEN
))
2239 /* Caller says, don't even try. */
2240 delete_insns_since (entry_last
);
2244 /* Compute the value of METHODS to pass to recursive calls.
2245 Don't allow widening to be tried recursively. */
2247 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
2249 /* Look for a wider mode of the same class for which it appears we can do
2252 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2254 /* This code doesn't make sense for conversion optabs, since we
2255 wouldn't then want to extend the operands to be the same size
2257 gcc_assert (!convert_optab_p (binoptab
));
2258 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2260 if (optab_handler (binoptab
, wider_mode
)
2261 || (methods
== OPTAB_LIB
2262 && optab_libfunc (binoptab
, wider_mode
)))
2264 rtx xop0
= op0
, xop1
= op1
;
2267 /* For certain integer operations, we need not actually extend
2268 the narrow operands, as long as we will truncate
2269 the results to the same narrowness. */
2271 if ((binoptab
== ior_optab
|| binoptab
== and_optab
2272 || binoptab
== xor_optab
2273 || binoptab
== add_optab
|| binoptab
== sub_optab
2274 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
2275 && mclass
== MODE_INT
)
2278 xop0
= widen_operand (xop0
, wider_mode
, mode
,
2279 unsignedp
, no_extend
);
2281 /* The second operand of a shift must always be extended. */
2282 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
2283 no_extend
&& binoptab
!= ashl_optab
);
2285 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
2286 unsignedp
, methods
);
2289 if (mclass
!= MODE_INT
2290 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2293 target
= gen_reg_rtx (mode
);
2294 convert_move (target
, temp
, 0);
2298 return gen_lowpart (mode
, temp
);
2301 delete_insns_since (last
);
2306 delete_insns_since (entry_last
);
2310 /* Expand a binary operator which has both signed and unsigned forms.
2311 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2314 If we widen unsigned operands, we may use a signed wider operation instead
2315 of an unsigned wider operation, since the result would be the same. */
2318 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
2319 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
2320 enum optab_methods methods
)
2323 optab direct_optab
= unsignedp
? uoptab
: soptab
;
2326 /* Do it without widening, if possible. */
2327 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2328 unsignedp
, OPTAB_DIRECT
);
2329 if (temp
|| methods
== OPTAB_DIRECT
)
2332 /* Try widening to a signed int. Disable any direct use of any
2333 signed insn in the current mode. */
2334 save_enable
= swap_optab_enable (soptab
, mode
, false);
2336 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2337 unsignedp
, OPTAB_WIDEN
);
2339 /* For unsigned operands, try widening to an unsigned int. */
2340 if (!temp
&& unsignedp
)
2341 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2342 unsignedp
, OPTAB_WIDEN
);
2343 if (temp
|| methods
== OPTAB_WIDEN
)
2346 /* Use the right width libcall if that exists. */
2347 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2348 unsignedp
, OPTAB_LIB
);
2349 if (temp
|| methods
== OPTAB_LIB
)
2352 /* Must widen and use a libcall, use either signed or unsigned. */
2353 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2354 unsignedp
, methods
);
2355 if (!temp
&& unsignedp
)
2356 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2357 unsignedp
, methods
);
2360 /* Undo the fiddling above. */
2362 swap_optab_enable (soptab
, mode
, true);
2366 /* Generate code to perform an operation specified by UNOPPTAB
2367 on operand OP0, with two results to TARG0 and TARG1.
2368 We assume that the order of the operands for the instruction
2369 is TARG0, TARG1, OP0.
2371 Either TARG0 or TARG1 may be zero, but what that means is that
2372 the result is not actually wanted. We will generate it into
2373 a dummy pseudo-reg and discard it. They may not both be zero.
2375 Returns 1 if this operation can be performed; 0 if not. */
2378 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
2381 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2382 enum mode_class mclass
;
2383 machine_mode wider_mode
;
2384 rtx_insn
*entry_last
= get_last_insn ();
2387 mclass
= GET_MODE_CLASS (mode
);
2390 targ0
= gen_reg_rtx (mode
);
2392 targ1
= gen_reg_rtx (mode
);
2394 /* Record where to go back to if we fail. */
2395 last
= get_last_insn ();
2397 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2399 class expand_operand ops
[3];
2400 enum insn_code icode
= optab_handler (unoptab
, mode
);
2402 create_fixed_operand (&ops
[0], targ0
);
2403 create_fixed_operand (&ops
[1], targ1
);
2404 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
2405 if (maybe_expand_insn (icode
, 3, ops
))
2409 /* It can't be done in this mode. Can we do it in a wider mode? */
2411 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2413 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2415 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2417 rtx t0
= gen_reg_rtx (wider_mode
);
2418 rtx t1
= gen_reg_rtx (wider_mode
);
2419 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2421 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
2423 convert_move (targ0
, t0
, unsignedp
);
2424 convert_move (targ1
, t1
, unsignedp
);
2428 delete_insns_since (last
);
2433 delete_insns_since (entry_last
);
2437 /* Generate code to perform an operation specified by BINOPTAB
2438 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2439 We assume that the order of the operands for the instruction
2440 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2441 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2443 Either TARG0 or TARG1 may be zero, but what that means is that
2444 the result is not actually wanted. We will generate it into
2445 a dummy pseudo-reg and discard it. They may not both be zero.
2447 Returns 1 if this operation can be performed; 0 if not. */
2450 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
2453 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2454 enum mode_class mclass
;
2455 machine_mode wider_mode
;
2456 rtx_insn
*entry_last
= get_last_insn ();
2459 mclass
= GET_MODE_CLASS (mode
);
2462 targ0
= gen_reg_rtx (mode
);
2464 targ1
= gen_reg_rtx (mode
);
2466 /* Record where to go back to if we fail. */
2467 last
= get_last_insn ();
2469 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2471 class expand_operand ops
[4];
2472 enum insn_code icode
= optab_handler (binoptab
, mode
);
2473 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2474 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2475 rtx xop0
= op0
, xop1
= op1
;
2477 /* If we are optimizing, force expensive constants into a register. */
2478 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2479 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2481 create_fixed_operand (&ops
[0], targ0
);
2482 create_convert_operand_from (&ops
[1], xop0
, mode
, unsignedp
);
2483 create_convert_operand_from (&ops
[2], xop1
, mode
, unsignedp
);
2484 create_fixed_operand (&ops
[3], targ1
);
2485 if (maybe_expand_insn (icode
, 4, ops
))
2487 delete_insns_since (last
);
2490 /* It can't be done in this mode. Can we do it in a wider mode? */
2492 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2494 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2496 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2498 rtx t0
= gen_reg_rtx (wider_mode
);
2499 rtx t1
= gen_reg_rtx (wider_mode
);
2500 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2501 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2503 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2506 convert_move (targ0
, t0
, unsignedp
);
2507 convert_move (targ1
, t1
, unsignedp
);
2511 delete_insns_since (last
);
2516 delete_insns_since (entry_last
);
2520 /* Expand the two-valued library call indicated by BINOPTAB, but
2521 preserve only one of the values. If TARG0 is non-NULL, the first
2522 value is placed into TARG0; otherwise the second value is placed
2523 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2524 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2525 This routine assumes that the value returned by the library call is
2526 as if the return value was of an integral mode twice as wide as the
2527 mode of OP0. Returns 1 if the call was successful. */
2530 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2531 rtx targ0
, rtx targ1
, enum rtx_code code
)
2534 machine_mode libval_mode
;
2539 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2540 gcc_assert (!targ0
!= !targ1
);
2542 mode
= GET_MODE (op0
);
2543 libfunc
= optab_libfunc (binoptab
, mode
);
2547 /* The value returned by the library function will have twice as
2548 many bits as the nominal MODE. */
2549 libval_mode
= smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode
));
2551 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2555 /* Get the part of VAL containing the value that we want. */
2556 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2557 targ0
? 0 : GET_MODE_SIZE (mode
));
2558 insns
= get_insns ();
2560 /* Move the into the desired location. */
2561 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2562 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2568 /* Wrapper around expand_unop which takes an rtx code to specify
2569 the operation to perform, not an optab pointer. All other
2570 arguments are the same. */
2572 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2573 rtx target
, int unsignedp
)
2575 optab unop
= code_to_optab (code
);
2578 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2584 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2586 A similar operation can be used for clrsb. UNOPTAB says which operation
2587 we are trying to expand. */
2589 widen_leading (scalar_int_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2591 opt_scalar_int_mode wider_mode_iter
;
2592 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2594 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2595 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2600 last
= get_last_insn ();
2603 target
= gen_reg_rtx (mode
);
2604 xop0
= widen_operand (op0
, wider_mode
, mode
,
2605 unoptab
!= clrsb_optab
, false);
2606 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2607 unoptab
!= clrsb_optab
);
2610 (wider_mode
, sub_optab
, temp
,
2611 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2612 - GET_MODE_PRECISION (mode
),
2614 target
, true, OPTAB_DIRECT
);
2616 delete_insns_since (last
);
2624 /* Attempt to emit (clrsb:mode op0) as
2625 (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2627 if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2629 (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2630 (ashr:mode op0 (const_int prec-1)))
2635 expand_clrsb_using_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2637 if (optimize_insn_for_size_p ()
2638 || optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2642 HOST_WIDE_INT val
= 0;
2643 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) != 2
2644 || val
!= GET_MODE_PRECISION (mode
))
2652 temp2
= expand_binop (mode
, ashl_optab
, op0
, const1_rtx
,
2653 NULL_RTX
, 0, OPTAB_DIRECT
);
2662 rtx temp
= expand_binop (mode
, ashr_optab
, op0
,
2663 GEN_INT (GET_MODE_PRECISION (mode
) - 1),
2664 NULL_RTX
, 0, OPTAB_DIRECT
);
2668 temp
= expand_binop (mode
, xor_optab
, temp2
, temp
, NULL_RTX
, 0,
2675 temp
= expand_binop (mode
, ior_optab
, temp
, const1_rtx
,
2676 NULL_RTX
, 0, OPTAB_DIRECT
);
2680 temp
= expand_unop_direct (mode
, clz_optab
, temp
, val
? NULL_RTX
: target
,
2686 temp
= expand_binop (mode
, add_optab
, temp
, constm1_rtx
,
2687 target
, 0, OPTAB_DIRECT
);
2692 rtx_insn
*seq
= get_insns ();
2695 add_equal_note (seq
, temp
, CLRSB
, op0
, NULL_RTX
, mode
);
2700 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2701 quantities, choosing which based on whether the high word is nonzero. */
2703 expand_doubleword_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2705 rtx xop0
= force_reg (mode
, op0
);
2706 rtx subhi
= gen_highpart (word_mode
, xop0
);
2707 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2708 rtx_code_label
*hi0_label
= gen_label_rtx ();
2709 rtx_code_label
*after_label
= gen_label_rtx ();
2713 /* If we were not given a target, use a word_mode register, not a
2714 'mode' register. The result will fit, and nobody is expecting
2715 anything bigger (the return type of __builtin_clz* is int). */
2717 target
= gen_reg_rtx (word_mode
);
2719 /* In any case, write to a word_mode scratch in both branches of the
2720 conditional, so we can ensure there is a single move insn setting
2721 'target' to tag a REG_EQUAL note on. */
2722 result
= gen_reg_rtx (word_mode
);
2726 /* If the high word is not equal to zero,
2727 then clz of the full value is clz of the high word. */
2728 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2729 word_mode
, true, hi0_label
);
2731 temp
= expand_unop_direct (word_mode
, clz_optab
, subhi
, result
, true);
2736 convert_move (result
, temp
, true);
2738 emit_jump_insn (targetm
.gen_jump (after_label
));
2741 /* Else clz of the full value is clz of the low word plus the number
2742 of bits in the high word. */
2743 emit_label (hi0_label
);
2745 temp
= expand_unop_direct (word_mode
, clz_optab
, sublo
, 0, true);
2748 temp
= expand_binop (word_mode
, add_optab
, temp
,
2749 gen_int_mode (GET_MODE_BITSIZE (word_mode
), word_mode
),
2750 result
, true, OPTAB_DIRECT
);
2754 convert_move (result
, temp
, true);
2756 emit_label (after_label
);
2757 convert_move (target
, result
, true);
2762 add_equal_note (seq
, target
, CLZ
, xop0
, NULL_RTX
, mode
);
2771 /* Try calculating popcount of a double-word quantity as two popcount's of
2772 word-sized quantities and summing up the results. */
2774 expand_doubleword_popcount (scalar_int_mode mode
, rtx op0
, rtx target
)
2781 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2782 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2784 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2785 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2793 /* If we were not given a target, use a word_mode register, not a
2794 'mode' register. The result will fit, and nobody is expecting
2795 anything bigger (the return type of __builtin_popcount* is int). */
2797 target
= gen_reg_rtx (word_mode
);
2799 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2804 add_equal_note (seq
, t
, POPCOUNT
, op0
, NULL_RTX
, mode
);
2812 (parity:narrow (low (x) ^ high (x))) */
2814 expand_doubleword_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2816 rtx t
= expand_binop (word_mode
, xor_optab
,
2817 operand_subword_force (op0
, 0, mode
),
2818 operand_subword_force (op0
, 1, mode
),
2819 NULL_RTX
, 0, OPTAB_DIRECT
);
2820 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2826 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2828 widen_bswap (scalar_int_mode mode
, rtx op0
, rtx target
)
2832 opt_scalar_int_mode wider_mode_iter
;
2834 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2835 if (optab_handler (bswap_optab
, wider_mode_iter
.require ())
2836 != CODE_FOR_nothing
)
2839 if (!wider_mode_iter
.exists ())
2842 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2843 last
= get_last_insn ();
2845 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2846 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2848 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2849 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2851 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2852 GET_MODE_BITSIZE (wider_mode
)
2853 - GET_MODE_BITSIZE (mode
),
2859 target
= gen_reg_rtx (mode
);
2860 emit_move_insn (target
, gen_lowpart (mode
, x
));
2863 delete_insns_since (last
);
2868 /* Try calculating bswap as two bswaps of two word-sized operands. */
2871 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2875 t1
= expand_unop (word_mode
, bswap_optab
,
2876 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2877 t0
= expand_unop (word_mode
, bswap_optab
,
2878 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2880 if (target
== 0 || !valid_multiword_target_p (target
))
2881 target
= gen_reg_rtx (mode
);
2883 emit_clobber (target
);
2884 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2885 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2890 /* Try calculating (parity x) as (and (popcount x) 1), where
2891 popcount can also be done in a wider mode. */
2893 expand_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2895 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2896 opt_scalar_int_mode wider_mode_iter
;
2897 FOR_EACH_MODE_FROM (wider_mode_iter
, mode
)
2899 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2900 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2905 last
= get_last_insn ();
2907 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2908 target
= gen_reg_rtx (wider_mode
);
2910 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2911 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2914 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2915 target
, true, OPTAB_DIRECT
);
2919 if (mclass
!= MODE_INT
2920 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2921 return convert_to_mode (mode
, temp
, 0);
2923 return gen_lowpart (mode
, temp
);
2926 delete_insns_since (last
);
2932 /* Try calculating ctz(x) as K - clz(x & -x) ,
2933 where K is GET_MODE_PRECISION(mode) - 1.
2935 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2936 don't have to worry about what the hardware does in that case. (If
2937 the clz instruction produces the usual value at 0, which is K, the
2938 result of this code sequence will be -1; expand_ffs, below, relies
2939 on this. It might be nice to have it be K instead, for consistency
2940 with the (very few) processors that provide a ctz with a defined
2941 value, but that would take one more instruction, and it would be
2942 less convenient for expand_ffs anyway. */
2945 expand_ctz (scalar_int_mode mode
, rtx op0
, rtx target
)
2950 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2955 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2957 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2958 true, OPTAB_DIRECT
);
2960 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2962 temp
= expand_binop (mode
, sub_optab
,
2963 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
2965 true, OPTAB_DIRECT
);
2975 add_equal_note (seq
, temp
, CTZ
, op0
, NULL_RTX
, mode
);
2981 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2982 else with the sequence used by expand_clz.
2984 The ffs builtin promises to return zero for a zero value and ctz/clz
2985 may have an undefined value in that case. If they do not give us a
2986 convenient value, we have to generate a test and branch. */
2988 expand_ffs (scalar_int_mode mode
, rtx op0
, rtx target
)
2990 HOST_WIDE_INT val
= 0;
2991 bool defined_at_zero
= false;
2995 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
2999 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
3003 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
3005 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
3008 temp
= expand_ctz (mode
, op0
, 0);
3012 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
3014 defined_at_zero
= true;
3015 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
3021 if (defined_at_zero
&& val
== -1)
3022 /* No correction needed at zero. */;
3025 /* We don't try to do anything clever with the situation found
3026 on some processors (eg Alpha) where ctz(0:mode) ==
3027 bitsize(mode). If someone can think of a way to send N to -1
3028 and leave alone all values in the range 0..N-1 (where N is a
3029 power of two), cheaper than this test-and-branch, please add it.
3031 The test-and-branch is done after the operation itself, in case
3032 the operation sets condition codes that can be recycled for this.
3033 (This is true on i386, for instance.) */
3035 rtx_code_label
*nonzero_label
= gen_label_rtx ();
3036 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
3037 mode
, true, nonzero_label
);
3039 convert_move (temp
, GEN_INT (-1), false);
3040 emit_label (nonzero_label
);
3043 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
3044 to produce a value in the range 0..bitsize. */
3045 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
3046 target
, false, OPTAB_DIRECT
);
3053 add_equal_note (seq
, temp
, FFS
, op0
, NULL_RTX
, mode
);
3062 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
3063 conditions, VAL may already be a SUBREG against which we cannot generate
3064 a further SUBREG. In this case, we expect forcing the value into a
3065 register will work around the situation. */
3068 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
3072 ret
= lowpart_subreg (omode
, val
, imode
);
3075 val
= force_reg (imode
, val
);
3076 ret
= lowpart_subreg (omode
, val
, imode
);
3077 gcc_assert (ret
!= NULL
);
3082 /* Expand a floating point absolute value or negation operation via a
3083 logical operation on the sign bit. */
3086 expand_absneg_bit (enum rtx_code code
, scalar_float_mode mode
,
3087 rtx op0
, rtx target
)
3089 const struct real_format
*fmt
;
3090 int bitpos
, word
, nwords
, i
;
3091 scalar_int_mode imode
;
3095 /* The format has to have a simple sign bit. */
3096 fmt
= REAL_MODE_FORMAT (mode
);
3100 bitpos
= fmt
->signbit_rw
;
3104 /* Don't create negative zeros if the format doesn't support them. */
3105 if (code
== NEG
&& !fmt
->has_signed_zero
)
3108 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3110 if (!int_mode_for_mode (mode
).exists (&imode
))
3119 if (FLOAT_WORDS_BIG_ENDIAN
)
3120 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3122 word
= bitpos
/ BITS_PER_WORD
;
3123 bitpos
= bitpos
% BITS_PER_WORD
;
3124 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3127 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3133 || reg_overlap_mentioned_p (target
, op0
)
3134 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3135 target
= gen_reg_rtx (mode
);
3141 for (i
= 0; i
< nwords
; ++i
)
3143 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3144 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3148 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
3150 immed_wide_int_const (mask
, imode
),
3151 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3152 if (temp
!= targ_piece
)
3153 emit_move_insn (targ_piece
, temp
);
3156 emit_move_insn (targ_piece
, op0_piece
);
3159 insns
= get_insns ();
3166 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
3167 gen_lowpart (imode
, op0
),
3168 immed_wide_int_const (mask
, imode
),
3169 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3170 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3172 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
3173 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
3180 /* As expand_unop, but will fail rather than attempt the operation in a
3181 different mode or with a libcall. */
3183 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
3186 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
3188 class expand_operand ops
[2];
3189 enum insn_code icode
= optab_handler (unoptab
, mode
);
3190 rtx_insn
*last
= get_last_insn ();
3193 create_output_operand (&ops
[0], target
, mode
);
3194 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
3195 pat
= maybe_gen_insn (icode
, 2, ops
);
3198 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3199 && ! add_equal_note (pat
, ops
[0].value
,
3200 optab_to_code (unoptab
),
3201 ops
[1].value
, NULL_RTX
, mode
))
3203 delete_insns_since (last
);
3204 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
3209 return ops
[0].value
;
3215 /* Generate code to perform an operation specified by UNOPTAB
3216 on operand OP0, with result having machine-mode MODE.
3218 UNSIGNEDP is for the case where we have to widen the operands
3219 to perform the operation. It says to use zero-extension.
3221 If TARGET is nonzero, the value
3222 is generated there, if it is convenient to do so.
3223 In all cases an rtx is returned for the locus of the value;
3224 this may or may not be TARGET. */
3227 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
3230 enum mode_class mclass
= GET_MODE_CLASS (mode
);
3231 machine_mode wider_mode
;
3232 scalar_int_mode int_mode
;
3233 scalar_float_mode float_mode
;
3237 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
3241 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3243 /* Widening (or narrowing) clz needs special treatment. */
3244 if (unoptab
== clz_optab
)
3246 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3248 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
3252 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3253 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3255 temp
= expand_doubleword_clz (int_mode
, op0
, target
);
3264 if (unoptab
== clrsb_optab
)
3266 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3268 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
3271 temp
= expand_clrsb_using_clz (int_mode
, op0
, target
);
3278 if (unoptab
== popcount_optab
3279 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3280 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3281 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3282 && optimize_insn_for_speed_p ())
3284 temp
= expand_doubleword_popcount (int_mode
, op0
, target
);
3289 if (unoptab
== parity_optab
3290 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3291 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3292 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3293 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
3294 && optimize_insn_for_speed_p ())
3296 temp
= expand_doubleword_parity (int_mode
, op0
, target
);
3301 /* Widening (or narrowing) bswap needs special treatment. */
3302 if (unoptab
== bswap_optab
)
3304 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3305 or ROTATERT. First try these directly; if this fails, then try the
3306 obvious pair of shifts with allowed widening, as this will probably
3307 be always more efficient than the other fallback methods. */
3313 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
3315 temp
= expand_binop (mode
, rotl_optab
, op0
,
3316 gen_int_shift_amount (mode
, 8),
3317 target
, unsignedp
, OPTAB_DIRECT
);
3322 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
3324 temp
= expand_binop (mode
, rotr_optab
, op0
,
3325 gen_int_shift_amount (mode
, 8),
3326 target
, unsignedp
, OPTAB_DIRECT
);
3331 last
= get_last_insn ();
3333 temp1
= expand_binop (mode
, ashl_optab
, op0
,
3334 gen_int_shift_amount (mode
, 8), NULL_RTX
,
3335 unsignedp
, OPTAB_WIDEN
);
3336 temp2
= expand_binop (mode
, lshr_optab
, op0
,
3337 gen_int_shift_amount (mode
, 8), NULL_RTX
,
3338 unsignedp
, OPTAB_WIDEN
);
3341 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
3342 unsignedp
, OPTAB_WIDEN
);
3347 delete_insns_since (last
);
3350 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3352 temp
= widen_bswap (int_mode
, op0
, target
);
3356 /* We do not provide a 128-bit bswap in libgcc so force the use of
3357 a double bswap for 64-bit targets. */
3358 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3359 && (UNITS_PER_WORD
== 8
3360 || optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
))
3362 temp
= expand_doubleword_bswap (mode
, op0
, target
);
3371 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3372 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3374 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
3377 rtx_insn
*last
= get_last_insn ();
3379 /* For certain operations, we need not actually extend
3380 the narrow operand, as long as we will truncate the
3381 results to the same narrowness. */
3383 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3384 (unoptab
== neg_optab
3385 || unoptab
== one_cmpl_optab
)
3386 && mclass
== MODE_INT
);
3388 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3393 if (mclass
!= MODE_INT
3394 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
3397 target
= gen_reg_rtx (mode
);
3398 convert_move (target
, temp
, 0);
3402 return gen_lowpart (mode
, temp
);
3405 delete_insns_since (last
);
3409 /* These can be done a word at a time. */
3410 if (unoptab
== one_cmpl_optab
3411 && is_int_mode (mode
, &int_mode
)
3412 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
3413 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3420 || reg_overlap_mentioned_p (target
, op0
)
3421 || !valid_multiword_target_p (target
))
3422 target
= gen_reg_rtx (int_mode
);
3426 /* Do the actual arithmetic. */
3427 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
3429 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
3430 rtx x
= expand_unop (word_mode
, unoptab
,
3431 operand_subword_force (op0
, i
, int_mode
),
3432 target_piece
, unsignedp
);
3434 if (target_piece
!= x
)
3435 emit_move_insn (target_piece
, x
);
3438 insns
= get_insns ();
3445 /* Emit ~op0 as op0 ^ -1. */
3446 if (unoptab
== one_cmpl_optab
3447 && (SCALAR_INT_MODE_P (mode
) || GET_MODE_CLASS (mode
) == MODE_VECTOR_INT
)
3448 && optab_handler (xor_optab
, mode
) != CODE_FOR_nothing
)
3450 temp
= expand_binop (mode
, xor_optab
, op0
, CONSTM1_RTX (mode
),
3451 target
, unsignedp
, OPTAB_DIRECT
);
3456 if (optab_to_code (unoptab
) == NEG
)
3458 /* Try negating floating point values by flipping the sign bit. */
3459 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3461 temp
= expand_absneg_bit (NEG
, float_mode
, op0
, target
);
3466 /* If there is no negation pattern, and we have no negative zero,
3467 try subtracting from zero. */
3468 if (!HONOR_SIGNED_ZEROS (mode
))
3470 temp
= expand_binop (mode
, (unoptab
== negv_optab
3471 ? subv_optab
: sub_optab
),
3472 CONST0_RTX (mode
), op0
, target
,
3473 unsignedp
, OPTAB_DIRECT
);
3479 /* Try calculating parity (x) as popcount (x) % 2. */
3480 if (unoptab
== parity_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3482 temp
= expand_parity (int_mode
, op0
, target
);
3487 /* Try implementing ffs (x) in terms of clz (x). */
3488 if (unoptab
== ffs_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3490 temp
= expand_ffs (int_mode
, op0
, target
);
3495 /* Try implementing ctz (x) in terms of clz (x). */
3496 if (unoptab
== ctz_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3498 temp
= expand_ctz (int_mode
, op0
, target
);
3504 /* Now try a library call in this mode. */
3505 libfunc
= optab_libfunc (unoptab
, mode
);
3511 machine_mode outmode
= mode
;
3513 /* All of these functions return small values. Thus we choose to
3514 have them return something that isn't a double-word. */
3515 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
3516 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
3517 || unoptab
== parity_optab
)
3519 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
3520 optab_libfunc (unoptab
, mode
)));
3524 /* Pass 1 for NO_QUEUE so we don't lose any increments
3525 if the libcall is cse'd or moved. */
3526 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
3528 insns
= get_insns ();
3531 target
= gen_reg_rtx (outmode
);
3532 bool trapv
= trapv_unoptab_p (unoptab
);
3534 eq_value
= NULL_RTX
;
3537 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
3538 if (GET_MODE_UNIT_SIZE (outmode
) < GET_MODE_UNIT_SIZE (mode
))
3539 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
3540 else if (GET_MODE_UNIT_SIZE (outmode
) > GET_MODE_UNIT_SIZE (mode
))
3541 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
3542 outmode
, eq_value
, mode
);
3544 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
3549 /* It can't be done in this mode. Can we do it in a wider mode? */
3551 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3553 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3555 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
3556 || optab_libfunc (unoptab
, wider_mode
))
3559 rtx_insn
*last
= get_last_insn ();
3561 /* For certain operations, we need not actually extend
3562 the narrow operand, as long as we will truncate the
3563 results to the same narrowness. */
3564 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3565 (unoptab
== neg_optab
3566 || unoptab
== one_cmpl_optab
3567 || unoptab
== bswap_optab
)
3568 && mclass
== MODE_INT
);
3570 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3573 /* If we are generating clz using wider mode, adjust the
3574 result. Similarly for clrsb. */
3575 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3578 scalar_int_mode wider_int_mode
3579 = as_a
<scalar_int_mode
> (wider_mode
);
3580 int_mode
= as_a
<scalar_int_mode
> (mode
);
3582 (wider_mode
, sub_optab
, temp
,
3583 gen_int_mode (GET_MODE_PRECISION (wider_int_mode
)
3584 - GET_MODE_PRECISION (int_mode
),
3586 target
, true, OPTAB_DIRECT
);
3589 /* Likewise for bswap. */
3590 if (unoptab
== bswap_optab
&& temp
!= 0)
3592 scalar_int_mode wider_int_mode
3593 = as_a
<scalar_int_mode
> (wider_mode
);
3594 int_mode
= as_a
<scalar_int_mode
> (mode
);
3595 gcc_assert (GET_MODE_PRECISION (wider_int_mode
)
3596 == GET_MODE_BITSIZE (wider_int_mode
)
3597 && GET_MODE_PRECISION (int_mode
)
3598 == GET_MODE_BITSIZE (int_mode
));
3600 temp
= expand_shift (RSHIFT_EXPR
, wider_int_mode
, temp
,
3601 GET_MODE_BITSIZE (wider_int_mode
)
3602 - GET_MODE_BITSIZE (int_mode
),
3608 if (mclass
!= MODE_INT
)
3611 target
= gen_reg_rtx (mode
);
3612 convert_move (target
, temp
, 0);
3616 return gen_lowpart (mode
, temp
);
3619 delete_insns_since (last
);
3624 /* One final attempt at implementing negation via subtraction,
3625 this time allowing widening of the operand. */
3626 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3629 temp
= expand_binop (mode
,
3630 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3631 CONST0_RTX (mode
), op0
,
3632 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3640 /* Emit code to compute the absolute value of OP0, with result to
3641 TARGET if convenient. (TARGET may be 0.) The return value says
3642 where the result actually is to be found.
3644 MODE is the mode of the operand; the mode of the result is
3645 different but can be deduced from MODE.
3650 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3651 int result_unsignedp
)
3655 if (GET_MODE_CLASS (mode
) != MODE_INT
3657 result_unsignedp
= 1;
3659 /* First try to do it with a special abs instruction. */
3660 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3665 /* For floating point modes, try clearing the sign bit. */
3666 scalar_float_mode float_mode
;
3667 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3669 temp
= expand_absneg_bit (ABS
, float_mode
, op0
, target
);
3674 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3675 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3676 && !HONOR_SIGNED_ZEROS (mode
))
3678 rtx_insn
*last
= get_last_insn ();
3680 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3683 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3689 delete_insns_since (last
);
3692 /* If this machine has expensive jumps, we can do integer absolute
3693 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3694 where W is the width of MODE. */
3696 scalar_int_mode int_mode
;
3697 if (is_int_mode (mode
, &int_mode
)
3698 && BRANCH_COST (optimize_insn_for_speed_p (),
3701 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3702 GET_MODE_PRECISION (int_mode
) - 1,
3705 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3708 temp
= expand_binop (int_mode
,
3709 result_unsignedp
? sub_optab
: subv_optab
,
3710 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3720 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3721 int result_unsignedp
, int safe
)
3724 rtx_code_label
*op1
;
3726 if (GET_MODE_CLASS (mode
) != MODE_INT
3728 result_unsignedp
= 1;
3730 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3734 /* If that does not win, use conditional jump and negate. */
3736 /* It is safe to use the target if it is the same
3737 as the source if this is also a pseudo register */
3738 if (op0
== target
&& REG_P (op0
)
3739 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3742 op1
= gen_label_rtx ();
3743 if (target
== 0 || ! safe
3744 || GET_MODE (target
) != mode
3745 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3747 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3748 target
= gen_reg_rtx (mode
);
3750 emit_move_insn (target
, op0
);
3753 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3754 NULL_RTX
, NULL
, op1
,
3755 profile_probability::uninitialized ());
3757 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3760 emit_move_insn (target
, op0
);
3766 /* Emit code to compute the one's complement absolute value of OP0
3767 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3768 (TARGET may be NULL_RTX.) The return value says where the result
3769 actually is to be found.
3771 MODE is the mode of the operand; the mode of the result is
3772 different but can be deduced from MODE. */
3775 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3779 /* Not applicable for floating point modes. */
3780 if (FLOAT_MODE_P (mode
))
3783 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3784 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3786 rtx_insn
*last
= get_last_insn ();
3788 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3790 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3796 delete_insns_since (last
);
3799 /* If this machine has expensive jumps, we can do one's complement
3800 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3802 scalar_int_mode int_mode
;
3803 if (is_int_mode (mode
, &int_mode
)
3804 && BRANCH_COST (optimize_insn_for_speed_p (),
3807 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3808 GET_MODE_PRECISION (int_mode
) - 1,
3811 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3821 /* A subroutine of expand_copysign, perform the copysign operation using the
3822 abs and neg primitives advertised to exist on the target. The assumption
3823 is that we have a split register file, and leaving op0 in fp registers,
3824 and not playing with subregs so much, will help the register allocator. */
3827 expand_copysign_absneg (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3828 int bitpos
, bool op0_is_abs
)
3830 scalar_int_mode imode
;
3831 enum insn_code icode
;
3833 rtx_code_label
*label
;
3838 /* Check if the back end provides an insn that handles signbit for the
3840 icode
= optab_handler (signbit_optab
, mode
);
3841 if (icode
!= CODE_FOR_nothing
)
3843 imode
= as_a
<scalar_int_mode
> (insn_data
[(int) icode
].operand
[0].mode
);
3844 sign
= gen_reg_rtx (imode
);
3845 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3849 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3851 if (!int_mode_for_mode (mode
).exists (&imode
))
3853 op1
= gen_lowpart (imode
, op1
);
3860 if (FLOAT_WORDS_BIG_ENDIAN
)
3861 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3863 word
= bitpos
/ BITS_PER_WORD
;
3864 bitpos
= bitpos
% BITS_PER_WORD
;
3865 op1
= operand_subword_force (op1
, word
, mode
);
3868 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3869 sign
= expand_binop (imode
, and_optab
, op1
,
3870 immed_wide_int_const (mask
, imode
),
3871 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3876 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3883 if (target
== NULL_RTX
)
3884 target
= copy_to_reg (op0
);
3886 emit_move_insn (target
, op0
);
3889 label
= gen_label_rtx ();
3890 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3892 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3893 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3895 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3897 emit_move_insn (target
, op0
);
3905 /* A subroutine of expand_copysign, perform the entire copysign operation
3906 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3907 is true if op0 is known to have its sign bit clear. */
3910 expand_copysign_bit (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3911 int bitpos
, bool op0_is_abs
)
3913 scalar_int_mode imode
;
3914 int word
, nwords
, i
;
3918 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3920 if (!int_mode_for_mode (mode
).exists (&imode
))
3929 if (FLOAT_WORDS_BIG_ENDIAN
)
3930 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3932 word
= bitpos
/ BITS_PER_WORD
;
3933 bitpos
= bitpos
% BITS_PER_WORD
;
3934 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3937 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3942 || reg_overlap_mentioned_p (target
, op0
)
3943 || reg_overlap_mentioned_p (target
, op1
)
3944 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3945 target
= gen_reg_rtx (mode
);
3951 for (i
= 0; i
< nwords
; ++i
)
3953 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3954 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3960 = expand_binop (imode
, and_optab
, op0_piece
,
3961 immed_wide_int_const (~mask
, imode
),
3962 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3963 op1
= expand_binop (imode
, and_optab
,
3964 operand_subword_force (op1
, i
, mode
),
3965 immed_wide_int_const (mask
, imode
),
3966 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3968 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
3969 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3970 if (temp
!= targ_piece
)
3971 emit_move_insn (targ_piece
, temp
);
3974 emit_move_insn (targ_piece
, op0_piece
);
3977 insns
= get_insns ();
3984 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
3985 immed_wide_int_const (mask
, imode
),
3986 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3988 op0
= gen_lowpart (imode
, op0
);
3990 op0
= expand_binop (imode
, and_optab
, op0
,
3991 immed_wide_int_const (~mask
, imode
),
3992 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3994 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
3995 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3996 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
4002 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
4003 scalar floating point mode. Return NULL if we do not know how to
4004 expand the operation inline. */
4007 expand_copysign (rtx op0
, rtx op1
, rtx target
)
4009 scalar_float_mode mode
;
4010 const struct real_format
*fmt
;
4014 mode
= as_a
<scalar_float_mode
> (GET_MODE (op0
));
4015 gcc_assert (GET_MODE (op1
) == mode
);
4017 /* First try to do it with a special instruction. */
4018 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
4019 target
, 0, OPTAB_DIRECT
);
4023 fmt
= REAL_MODE_FORMAT (mode
);
4024 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
4028 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
4030 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
4031 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
4035 if (fmt
->signbit_ro
>= 0
4036 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
4037 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
4038 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
4040 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
4041 fmt
->signbit_ro
, op0_is_abs
);
4046 if (fmt
->signbit_rw
< 0)
4048 return expand_copysign_bit (mode
, op0
, op1
, target
,
4049 fmt
->signbit_rw
, op0_is_abs
);
4052 /* Generate an instruction whose insn-code is INSN_CODE,
4053 with two operands: an output TARGET and an input OP0.
4054 TARGET *must* be nonzero, and the output is always stored there.
4055 CODE is an rtx code such that (CODE OP0) is an rtx that describes
4056 the value that is stored into TARGET.
4058 Return false if expansion failed. */
4061 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
4064 class expand_operand ops
[2];
4067 create_output_operand (&ops
[0], target
, GET_MODE (target
));
4068 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
4069 pat
= maybe_gen_insn (icode
, 2, ops
);
4073 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
4075 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
,
4080 if (ops
[0].value
!= target
)
4081 emit_move_insn (target
, ops
[0].value
);
4084 /* Generate an instruction whose insn-code is INSN_CODE,
4085 with two operands: an output TARGET and an input OP0.
4086 TARGET *must* be nonzero, and the output is always stored there.
4087 CODE is an rtx code such that (CODE OP0) is an rtx that describes
4088 the value that is stored into TARGET. */
4091 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
4093 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
4097 struct no_conflict_data
4100 rtx_insn
*first
, *insn
;
4104 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
4105 the currently examined clobber / store has to stay in the list of
4106 insns that constitute the actual libcall block. */
4108 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
4110 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
4112 /* If this inns directly contributes to setting the target, it must stay. */
4113 if (reg_overlap_mentioned_p (p
->target
, dest
))
4114 p
->must_stay
= true;
4115 /* If we haven't committed to keeping any other insns in the list yet,
4116 there is nothing more to check. */
4117 else if (p
->insn
== p
->first
)
4119 /* If this insn sets / clobbers a register that feeds one of the insns
4120 already in the list, this insn has to stay too. */
4121 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
4122 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
4123 || reg_used_between_p (dest
, p
->first
, p
->insn
)
4124 /* Likewise if this insn depends on a register set by a previous
4125 insn in the list, or if it sets a result (presumably a hard
4126 register) that is set or clobbered by a previous insn.
4127 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4128 SET_DEST perform the former check on the address, and the latter
4129 check on the MEM. */
4130 || (GET_CODE (set
) == SET
4131 && (modified_in_p (SET_SRC (set
), p
->first
)
4132 || modified_in_p (SET_DEST (set
), p
->first
)
4133 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
4134 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
4135 p
->must_stay
= true;
4139 /* Emit code to make a call to a constant function or a library call.
4141 INSNS is a list containing all insns emitted in the call.
4142 These insns leave the result in RESULT. Our block is to copy RESULT
4143 to TARGET, which is logically equivalent to EQUIV.
4145 We first emit any insns that set a pseudo on the assumption that these are
4146 loading constants into registers; doing so allows them to be safely cse'ed
4147 between blocks. Then we emit all the other insns in the block, followed by
4148 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4149 note with an operand of EQUIV. */
4152 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
4153 bool equiv_may_trap
)
4155 rtx final_dest
= target
;
4156 rtx_insn
*next
, *last
, *insn
;
4158 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4159 into a MEM later. Protect the libcall block from this change. */
4160 if (! REG_P (target
) || REG_USERVAR_P (target
))
4161 target
= gen_reg_rtx (GET_MODE (target
));
4163 /* If we're using non-call exceptions, a libcall corresponding to an
4164 operation that may trap may also trap. */
4165 /* ??? See the comment in front of make_reg_eh_region_note. */
4166 if (cfun
->can_throw_non_call_exceptions
4167 && (equiv_may_trap
|| may_trap_p (equiv
)))
4169 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
4172 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
4175 int lp_nr
= INTVAL (XEXP (note
, 0));
4176 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
4177 remove_note (insn
, note
);
4183 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4184 reg note to indicate that this call cannot throw or execute a nonlocal
4185 goto (unless there is already a REG_EH_REGION note, in which case
4187 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
4189 make_reg_eh_region_note_nothrow_nononlocal (insn
);
4192 /* First emit all insns that set pseudos. Remove them from the list as
4193 we go. Avoid insns that set pseudos which were referenced in previous
4194 insns. These can be generated by move_by_pieces, for example,
4195 to update an address. Similarly, avoid insns that reference things
4196 set in previous insns. */
4198 for (insn
= insns
; insn
; insn
= next
)
4200 rtx set
= single_set (insn
);
4202 next
= NEXT_INSN (insn
);
4204 if (set
!= 0 && REG_P (SET_DEST (set
))
4205 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
4207 struct no_conflict_data data
;
4209 data
.target
= const0_rtx
;
4213 note_stores (insn
, no_conflict_move_test
, &data
);
4214 if (! data
.must_stay
)
4216 if (PREV_INSN (insn
))
4217 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
4222 SET_PREV_INSN (next
) = PREV_INSN (insn
);
4228 /* Some ports use a loop to copy large arguments onto the stack.
4229 Don't move anything outside such a loop. */
4234 /* Write the remaining insns followed by the final copy. */
4235 for (insn
= insns
; insn
; insn
= next
)
4237 next
= NEXT_INSN (insn
);
4242 last
= emit_move_insn (target
, result
);
4244 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
4246 if (final_dest
!= target
)
4247 emit_move_insn (final_dest
, target
);
4251 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
4253 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
4256 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
4257 PURPOSE describes how this comparison will be used. CODE is the rtx
4258 comparison code we will be using.
4260 ??? Actually, CODE is slightly weaker than that. A target is still
4261 required to implement all of the normal bcc operations, but not
4262 required to implement all (or any) of the unordered bcc operations. */
4265 can_compare_p (enum rtx_code code
, machine_mode mode
,
4266 enum can_compare_purpose purpose
)
4269 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
4272 enum insn_code icode
;
4274 if (purpose
== ccp_jump
4275 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
4276 && insn_operand_matches (icode
, 0, test
))
4278 if (purpose
== ccp_store_flag
4279 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
4280 && insn_operand_matches (icode
, 1, test
))
4282 if (purpose
== ccp_cmov
4283 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
4286 mode
= GET_MODE_WIDER_MODE (mode
).else_void ();
4287 PUT_MODE (test
, mode
);
4289 while (mode
!= VOIDmode
);
4294 /* Return whether RTL code CODE corresponds to an unsigned optab. */
4297 unsigned_optab_p (enum rtx_code code
)
4299 return code
== LTU
|| code
== LEU
|| code
== GTU
|| code
== GEU
;
4302 /* Return whether the backend-emitted comparison for code CODE, comparing
4303 operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4304 operand OPNO of pattern ICODE. */
4307 insn_predicate_matches_p (enum insn_code icode
, unsigned int opno
,
4308 enum rtx_code code
, machine_mode mask_mode
,
4309 machine_mode value_mode
)
4311 rtx reg1
= alloca_raw_REG (value_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4312 rtx reg2
= alloca_raw_REG (value_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4313 rtx test
= alloca_rtx_fmt_ee (code
, mask_mode
, reg1
, reg2
);
4314 return insn_operand_matches (icode
, opno
, test
);
4317 /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4318 for code CODE, comparing operands of mode VALUE_MODE and producing a result
4322 can_vec_cmp_compare_p (enum rtx_code code
, machine_mode value_mode
,
4323 machine_mode mask_mode
)
4325 enum insn_code icode
4326 = get_vec_cmp_icode (value_mode
, mask_mode
, unsigned_optab_p (code
));
4327 if (icode
== CODE_FOR_nothing
)
4330 return insn_predicate_matches_p (icode
, 1, code
, mask_mode
, value_mode
);
4333 /* Return whether the backend can emit a vector comparison (vcond/vcondu) for
4334 code CODE, comparing operands of mode CMP_OP_MODE and producing a result
4338 can_vcond_compare_p (enum rtx_code code
, machine_mode value_mode
,
4339 machine_mode cmp_op_mode
)
4341 enum insn_code icode
4342 = get_vcond_icode (value_mode
, cmp_op_mode
, unsigned_optab_p (code
));
4343 if (icode
== CODE_FOR_nothing
)
4346 return insn_predicate_matches_p (icode
, 3, code
, value_mode
, cmp_op_mode
);
4349 /* Return whether the backend can emit vector set instructions for inserting
4350 element into vector at variable index position. */
4353 can_vec_set_var_idx_p (machine_mode vec_mode
)
4355 if (!VECTOR_MODE_P (vec_mode
))
4358 machine_mode inner_mode
= GET_MODE_INNER (vec_mode
);
4360 rtx reg1
= alloca_raw_REG (vec_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4361 rtx reg2
= alloca_raw_REG (inner_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4363 enum insn_code icode
= optab_handler (vec_set_optab
, vec_mode
);
4365 const struct insn_data_d
*data
= &insn_data
[icode
];
4366 machine_mode idx_mode
= data
->operand
[2].mode
;
4368 rtx reg3
= alloca_raw_REG (idx_mode
, LAST_VIRTUAL_REGISTER
+ 3);
4370 return icode
!= CODE_FOR_nothing
&& insn_operand_matches (icode
, 0, reg1
)
4371 && insn_operand_matches (icode
, 1, reg2
)
4372 && insn_operand_matches (icode
, 2, reg3
);
4375 /* This function is called when we are going to emit a compare instruction that
4376 compares the values found in X and Y, using the rtl operator COMPARISON.
4378 If they have mode BLKmode, then SIZE specifies the size of both operands.
4380 UNSIGNEDP nonzero says that the operands are unsigned;
4381 this matters if they need to be widened (as given by METHODS).
4383 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4384 if we failed to produce one.
4386 *PMODE is the mode of the inputs (in case they are const_int).
4388 This function performs all the setup necessary so that the caller only has
4389 to emit a single comparison insn. This setup can involve doing a BLKmode
4390 comparison or emitting a library call to perform the comparison if no insn
4391 is available to handle it.
4392 The values which are passed in through pointers can be modified; the caller
4393 should perform the comparison on the modified values. Constant
4394 comparisons must have already been folded. */
4397 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4398 int unsignedp
, enum optab_methods methods
,
4399 rtx
*ptest
, machine_mode
*pmode
)
4401 machine_mode mode
= *pmode
;
4403 machine_mode cmp_mode
;
4405 /* The other methods are not needed. */
4406 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
4407 || methods
== OPTAB_LIB_WIDEN
);
4409 if (CONST_SCALAR_INT_P (y
))
4410 canonicalize_comparison (mode
, &comparison
, &y
);
4412 /* If we are optimizing, force expensive constants into a register. */
4413 if (CONSTANT_P (x
) && optimize
4414 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
4415 > COSTS_N_INSNS (1))
4416 && can_create_pseudo_p ())
4417 x
= force_reg (mode
, x
);
4419 if (CONSTANT_P (y
) && optimize
4420 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
4421 > COSTS_N_INSNS (1))
4422 && can_create_pseudo_p ())
4423 y
= force_reg (mode
, y
);
4425 /* Don't let both operands fail to indicate the mode. */
4426 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
4427 x
= force_reg (mode
, x
);
4428 if (mode
== VOIDmode
)
4429 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
4431 /* Handle all BLKmode compares. */
4433 if (mode
== BLKmode
)
4435 machine_mode result_mode
;
4436 enum insn_code cmp_code
;
4439 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
4443 /* Try to use a memory block compare insn - either cmpstr
4444 or cmpmem will do. */
4445 opt_scalar_int_mode cmp_mode_iter
;
4446 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
4448 scalar_int_mode cmp_mode
= cmp_mode_iter
.require ();
4449 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
4450 if (cmp_code
== CODE_FOR_nothing
)
4451 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
4452 if (cmp_code
== CODE_FOR_nothing
)
4453 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
4454 if (cmp_code
== CODE_FOR_nothing
)
4457 /* Must make sure the size fits the insn's mode. */
4458 if (CONST_INT_P (size
)
4459 ? UINTVAL (size
) > GET_MODE_MASK (cmp_mode
)
4460 : (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (size
)))
4461 > GET_MODE_BITSIZE (cmp_mode
)))
4464 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
4465 result
= gen_reg_rtx (result_mode
);
4466 size
= convert_to_mode (cmp_mode
, size
, 1);
4467 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
4469 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
4470 *pmode
= result_mode
;
4474 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
4477 /* Otherwise call a library function. */
4478 result
= emit_block_comp_via_libcall (x
, y
, size
);
4482 mode
= TYPE_MODE (integer_type_node
);
4483 methods
= OPTAB_LIB_WIDEN
;
4487 /* Don't allow operands to the compare to trap, as that can put the
4488 compare and branch in different basic blocks. */
4489 if (cfun
->can_throw_non_call_exceptions
)
4491 if (!can_create_pseudo_p () && (may_trap_p (x
) || may_trap_p (y
)))
4494 x
= copy_to_reg (x
);
4496 y
= copy_to_reg (y
);
4499 if (GET_MODE_CLASS (mode
) == MODE_CC
)
4501 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
4502 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4503 if (icode
!= CODE_FOR_nothing
4504 && insn_operand_matches (icode
, 0, test
))
4513 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4514 FOR_EACH_WIDER_MODE_FROM (cmp_mode
, mode
)
4516 enum insn_code icode
;
4517 icode
= optab_handler (cbranch_optab
, cmp_mode
);
4518 if (icode
!= CODE_FOR_nothing
4519 && insn_operand_matches (icode
, 0, test
))
4521 rtx_insn
*last
= get_last_insn ();
4522 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
4523 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
4525 && insn_operand_matches (icode
, 1, op0
)
4526 && insn_operand_matches (icode
, 2, op1
))
4528 XEXP (test
, 0) = op0
;
4529 XEXP (test
, 1) = op1
;
4534 delete_insns_since (last
);
4537 if (methods
== OPTAB_DIRECT
)
4541 if (methods
!= OPTAB_LIB_WIDEN
)
4544 if (SCALAR_FLOAT_MODE_P (mode
))
4546 /* Small trick if UNORDERED isn't implemented by the hardware. */
4547 if (comparison
== UNORDERED
&& rtx_equal_p (x
, y
))
4549 prepare_cmp_insn (x
, y
, UNLT
, NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4555 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
4560 machine_mode ret_mode
;
4562 /* Handle a libcall just for the mode we are using. */
4563 libfunc
= optab_libfunc (cmp_optab
, mode
);
4564 gcc_assert (libfunc
);
4566 /* If we want unsigned, and this mode has a distinct unsigned
4567 comparison routine, use that. */
4570 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
4575 ret_mode
= targetm
.libgcc_cmp_return_mode ();
4576 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4577 ret_mode
, x
, mode
, y
, mode
);
4579 /* There are two kinds of comparison routines. Biased routines
4580 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4581 of gcc expect that the comparison operation is equivalent
4582 to the modified comparison. For signed comparisons compare the
4583 result against 1 in the biased case, and zero in the unbiased
4584 case. For unsigned comparisons always compare against 1 after
4585 biasing the unbiased result by adding 1. This gives us a way to
4587 The comparisons in the fixed-point helper library are always
4592 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
4595 x
= plus_constant (ret_mode
, result
, 1);
4601 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
4611 /* Before emitting an insn with code ICODE, make sure that X, which is going
4612 to be used for operand OPNUM of the insn, is converted from mode MODE to
4613 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4614 that it is accepted by the operand predicate. Return the new value. */
4617 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
4618 machine_mode wider_mode
, int unsignedp
)
4620 if (mode
!= wider_mode
)
4621 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
4623 if (!insn_operand_matches (icode
, opnum
, x
))
4625 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
4626 if (reload_completed
)
4628 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
4630 x
= copy_to_mode_reg (op_mode
, x
);
4636 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4637 we can do the branch. */
4640 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
4641 direct_optab cmp_optab
, profile_probability prob
,
4644 machine_mode optab_mode
;
4645 enum mode_class mclass
;
4646 enum insn_code icode
;
4649 mclass
= GET_MODE_CLASS (mode
);
4650 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
4651 icode
= optab_handler (cmp_optab
, optab_mode
);
4653 gcc_assert (icode
!= CODE_FOR_nothing
);
4654 gcc_assert (test_branch
|| insn_operand_matches (icode
, 0, test
));
4656 insn
= emit_jump_insn (GEN_FCN (icode
) (XEXP (test
, 0),
4657 XEXP (test
, 1), label
));
4659 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
4660 XEXP (test
, 1), label
));
4662 if (prob
.initialized_p ()
4663 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4666 && any_condjump_p (insn
)
4667 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4668 add_reg_br_prob_note (insn
, prob
);
4671 /* PTEST points to a comparison that compares its first operand with zero.
4672 Check to see if it can be performed as a bit-test-and-branch instead.
4673 On success, return the instruction that performs the bit-test-and-branch
4674 and replace the second operand of *PTEST with the bit number to test.
4675 On failure, return CODE_FOR_nothing and leave *PTEST unchanged.
4677 Note that the comparison described by *PTEST should not be taken
4678 literally after a successful return. *PTEST is just a convenient
4679 place to store the two operands of the bit-and-test.
4681 VAL must contain the original tree expression for the first operand
4684 static enum insn_code
4685 validate_test_and_branch (tree val
, rtx
*ptest
, machine_mode
*pmode
, optab
*res
)
4687 if (!val
|| TREE_CODE (val
) != SSA_NAME
)
4688 return CODE_FOR_nothing
;
4690 machine_mode mode
= TYPE_MODE (TREE_TYPE (val
));
4694 if (GET_CODE (test
) == EQ
)
4695 optab
= tbranch_eq_optab
;
4696 else if (GET_CODE (test
) == NE
)
4697 optab
= tbranch_ne_optab
;
4699 return CODE_FOR_nothing
;
4703 /* If the target supports the testbit comparison directly, great. */
4704 auto icode
= direct_optab_handler (optab
, mode
);
4705 if (icode
== CODE_FOR_nothing
)
4708 if (tree_zero_one_valued_p (val
))
4710 auto pos
= BITS_BIG_ENDIAN
? GET_MODE_BITSIZE (mode
) - 1 : 0;
4711 XEXP (test
, 1) = gen_int_mode (pos
, mode
);
4717 wide_int wcst
= get_nonzero_bits (val
);
4719 return CODE_FOR_nothing
;
4723 if ((bitpos
= wi::exact_log2 (wcst
)) == -1)
4724 return CODE_FOR_nothing
;
4726 auto pos
= BITS_BIG_ENDIAN
? GET_MODE_BITSIZE (mode
) - 1 - bitpos
: bitpos
;
4727 XEXP (test
, 1) = gen_int_mode (pos
, mode
);
4733 /* Generate code to compare X with Y so that the condition codes are
4734 set and to jump to LABEL if the condition is true. If X is a
4735 constant and Y is not a constant, then the comparison is swapped to
4736 ensure that the comparison RTL has the canonical form.
4738 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4739 need to be widened. UNSIGNEDP is also used to select the proper
4740 branch condition code.
4742 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4744 MODE is the mode of the inputs (in case they are const_int).
4746 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4747 It will be potentially converted into an unsigned variant based on
4748 UNSIGNEDP to select a proper jump instruction.
4750 PROB is the probability of jumping to LABEL. If the comparison is against
4751 zero then VAL contains the expression from which the non-zero RTL is
4755 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4756 machine_mode mode
, int unsignedp
, tree val
, rtx label
,
4757 profile_probability prob
)
4759 rtx op0
= x
, op1
= y
;
4762 /* Swap operands and condition to ensure canonical RTL. */
4763 if (swap_commutative_operands_p (x
, y
)
4764 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4767 comparison
= swap_condition (comparison
);
4770 /* If OP0 is still a constant, then both X and Y must be constants
4771 or the opposite comparison is not supported. Force X into a register
4772 to create canonical RTL. */
4773 if (CONSTANT_P (op0
))
4774 op0
= force_reg (mode
, op0
);
4777 comparison
= unsigned_condition (comparison
);
4779 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4782 /* Check if we're comparing a truth type with 0, and if so check if
4783 the target supports tbranch. */
4784 machine_mode tmode
= mode
;
4786 if (op1
== CONST0_RTX (GET_MODE (op1
))
4787 && validate_test_and_branch (val
, &test
, &tmode
,
4788 &optab
) != CODE_FOR_nothing
)
4790 emit_cmp_and_jump_insn_1 (test
, tmode
, label
, optab
, prob
, true);
4794 emit_cmp_and_jump_insn_1 (test
, mode
, label
, cbranch_optab
, prob
, false);
4797 /* Overloaded version of emit_cmp_and_jump_insns in which VAL is unknown. */
4800 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4801 machine_mode mode
, int unsignedp
, rtx label
,
4802 profile_probability prob
)
4804 emit_cmp_and_jump_insns (x
, y
, comparison
, size
, mode
, unsignedp
, NULL
,
4809 /* Emit a library call comparison between floating point X and Y.
4810 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4813 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4814 rtx
*ptest
, machine_mode
*pmode
)
4816 enum rtx_code swapped
= swap_condition (comparison
);
4817 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4818 machine_mode orig_mode
= GET_MODE (x
);
4820 rtx true_rtx
, false_rtx
;
4821 rtx value
, target
, equiv
;
4824 bool reversed_p
= false;
4825 scalar_int_mode cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4827 FOR_EACH_WIDER_MODE_FROM (mode
, orig_mode
)
4829 if (code_to_optab (comparison
)
4830 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4833 if (code_to_optab (swapped
)
4834 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4837 comparison
= swapped
;
4841 if (code_to_optab (reversed
)
4842 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4844 comparison
= reversed
;
4850 gcc_assert (mode
!= VOIDmode
);
4852 if (mode
!= orig_mode
)
4854 x
= convert_to_mode (mode
, x
, 0);
4855 y
= convert_to_mode (mode
, y
, 0);
4858 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4859 the RTL. The allows the RTL optimizers to delete the libcall if the
4860 condition can be determined at compile-time. */
4861 if (comparison
== UNORDERED
4862 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4864 true_rtx
= const_true_rtx
;
4865 false_rtx
= const0_rtx
;
4872 true_rtx
= const0_rtx
;
4873 false_rtx
= const_true_rtx
;
4877 true_rtx
= const_true_rtx
;
4878 false_rtx
= const0_rtx
;
4882 true_rtx
= const1_rtx
;
4883 false_rtx
= const0_rtx
;
4887 true_rtx
= const0_rtx
;
4888 false_rtx
= constm1_rtx
;
4892 true_rtx
= constm1_rtx
;
4893 false_rtx
= const0_rtx
;
4897 true_rtx
= const0_rtx
;
4898 false_rtx
= const1_rtx
;
4906 if (comparison
== UNORDERED
)
4908 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4909 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4910 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4911 temp
, const_true_rtx
, equiv
);
4915 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4916 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4917 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4918 equiv
, true_rtx
, false_rtx
);
4922 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4923 cmp_mode
, x
, mode
, y
, mode
);
4924 insns
= get_insns ();
4927 target
= gen_reg_rtx (cmp_mode
);
4928 emit_libcall_block (insns
, target
, value
, equiv
);
4930 if (comparison
== UNORDERED
4931 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
4933 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
4935 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
4940 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4943 emit_indirect_jump (rtx loc
)
4945 if (!targetm
.have_indirect_jump ())
4946 sorry ("indirect jumps are not available on this target");
4949 class expand_operand ops
[1];
4950 create_address_operand (&ops
[0], loc
);
4951 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
4957 /* Emit a conditional move instruction if the machine supports one for that
4958 condition and machine mode.
4960 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4961 the mode to use should they be constants. If it is VOIDmode, they cannot
4964 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4965 should be stored there. MODE is the mode to use should they be constants.
4966 If it is VOIDmode, they cannot both be constants.
4968 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4969 is not supported. */
4972 emit_conditional_move (rtx target
, struct rtx_comparison comp
,
4974 machine_mode mode
, int unsignedp
)
4978 enum insn_code icode
;
4979 enum rtx_code reversed
;
4981 /* If the two source operands are identical, that's just a move. */
4983 if (rtx_equal_p (op2
, op3
))
4986 target
= gen_reg_rtx (mode
);
4988 emit_move_insn (target
, op3
);
4992 /* If one operand is constant, make it the second one. Only do this
4993 if the other operand is not constant as well. */
4995 if (swap_commutative_operands_p (comp
.op0
, comp
.op1
))
4997 std::swap (comp
.op0
, comp
.op1
);
4998 comp
.code
= swap_condition (comp
.code
);
5001 /* get_condition will prefer to generate LT and GT even if the old
5002 comparison was against zero, so undo that canonicalization here since
5003 comparisons against zero are cheaper. */
5005 if (comp
.code
== LT
&& comp
.op1
== const1_rtx
)
5006 comp
.code
= LE
, comp
.op1
= const0_rtx
;
5007 else if (comp
.code
== GT
&& comp
.op1
== constm1_rtx
)
5008 comp
.code
= GE
, comp
.op1
= const0_rtx
;
5010 if (comp
.mode
== VOIDmode
)
5011 comp
.mode
= GET_MODE (comp
.op0
);
5013 enum rtx_code orig_code
= comp
.code
;
5014 bool swapped
= false;
5015 if (swap_commutative_operands_p (op2
, op3
)
5017 reversed_comparison_code_parts (comp
.code
, comp
.op0
, comp
.op1
, NULL
))
5020 std::swap (op2
, op3
);
5021 comp
.code
= reversed
;
5025 if (mode
== VOIDmode
)
5026 mode
= GET_MODE (op2
);
5028 icode
= direct_optab_handler (movcc_optab
, mode
);
5030 if (icode
== CODE_FOR_nothing
)
5034 target
= gen_reg_rtx (mode
);
5036 for (int pass
= 0; ; pass
++)
5038 comp
.code
= unsignedp
? unsigned_condition (comp
.code
) : comp
.code
;
5040 simplify_gen_relational (comp
.code
, VOIDmode
,
5041 comp
.mode
, comp
.op0
, comp
.op1
);
5043 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5044 punt and let the caller figure out how best to deal with this
5046 if (COMPARISON_P (comparison
))
5048 saved_pending_stack_adjust save
;
5049 save_pending_stack_adjust (&save
);
5050 last
= get_last_insn ();
5051 do_pending_stack_adjust ();
5052 machine_mode cmpmode
= comp
.mode
;
5053 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
5054 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
5055 OPTAB_WIDEN
, &comparison
, &cmpmode
);
5058 rtx res
= emit_conditional_move_1 (target
, comparison
,
5060 if (res
!= NULL_RTX
)
5063 delete_insns_since (last
);
5064 restore_pending_stack_adjust (&save
);
5070 /* If the preferred op2/op3 order is not usable, retry with other
5071 operand order, perhaps it will expand successfully. */
5073 comp
.code
= orig_code
;
5074 else if ((reversed
=
5075 reversed_comparison_code_parts (orig_code
, comp
.op0
, comp
.op1
,
5078 comp
.code
= reversed
;
5081 std::swap (op2
, op3
);
5085 /* Helper function that, in addition to COMPARISON, also tries
5086 the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
5087 to when we pass the specific constituents of a comparison, no
5088 additional insns are emitted for it. It might still be necessary
5089 to emit more than one insn for the final conditional move, though. */
5092 emit_conditional_move (rtx target
, rtx comparison
, rtx rev_comparison
,
5093 rtx op2
, rtx op3
, machine_mode mode
)
5095 rtx res
= emit_conditional_move_1 (target
, comparison
, op2
, op3
, mode
);
5097 if (res
!= NULL_RTX
)
5100 return emit_conditional_move_1 (target
, rev_comparison
, op3
, op2
, mode
);
5103 /* Helper for emitting a conditional move. */
5106 emit_conditional_move_1 (rtx target
, rtx comparison
,
5107 rtx op2
, rtx op3
, machine_mode mode
)
5109 enum insn_code icode
;
5111 if (comparison
== NULL_RTX
|| !COMPARISON_P (comparison
))
5114 /* If the two source operands are identical, that's just a move.
5115 As the comparison comes in non-canonicalized, we must make
5116 sure not to discard any possible side effects. If there are
5117 side effects, just let the target handle it. */
5118 if (!side_effects_p (comparison
) && rtx_equal_p (op2
, op3
))
5121 target
= gen_reg_rtx (mode
);
5123 emit_move_insn (target
, op3
);
5127 if (mode
== VOIDmode
)
5128 mode
= GET_MODE (op2
);
5130 icode
= direct_optab_handler (movcc_optab
, mode
);
5132 if (icode
== CODE_FOR_nothing
)
5136 target
= gen_reg_rtx (mode
);
5138 class expand_operand ops
[4];
5140 create_output_operand (&ops
[0], target
, mode
);
5141 create_fixed_operand (&ops
[1], comparison
);
5142 create_input_operand (&ops
[2], op2
, mode
);
5143 create_input_operand (&ops
[3], op3
, mode
);
5145 if (maybe_expand_insn (icode
, 4, ops
))
5147 if (ops
[0].value
!= target
)
5148 convert_move (target
, ops
[0].value
, false);
5156 /* Emit a conditional negate or bitwise complement using the
5157 negcc or notcc optabs if available. Return NULL_RTX if such operations
5158 are not available. Otherwise return the RTX holding the result.
5159 TARGET is the desired destination of the result. COMP is the comparison
5160 on which to negate. If COND is true move into TARGET the negation
5161 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5162 CODE is either NEG or NOT. MODE is the machine mode in which the
5163 operation is performed. */
5166 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
5167 machine_mode mode
, rtx cond
, rtx op1
,
5170 optab op
= unknown_optab
;
5173 else if (code
== NOT
)
5178 insn_code icode
= direct_optab_handler (op
, mode
);
5180 if (icode
== CODE_FOR_nothing
)
5184 target
= gen_reg_rtx (mode
);
5186 rtx_insn
*last
= get_last_insn ();
5187 class expand_operand ops
[4];
5189 create_output_operand (&ops
[0], target
, mode
);
5190 create_fixed_operand (&ops
[1], cond
);
5191 create_input_operand (&ops
[2], op1
, mode
);
5192 create_input_operand (&ops
[3], op2
, mode
);
5194 if (maybe_expand_insn (icode
, 4, ops
))
5196 if (ops
[0].value
!= target
)
5197 convert_move (target
, ops
[0].value
, false);
5201 delete_insns_since (last
);
5205 /* Emit a conditional addition instruction if the machine supports one for that
5206 condition and machine mode.
5208 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5209 the mode to use should they be constants. If it is VOIDmode, they cannot
5212 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5213 should be stored there. MODE is the mode to use should they be constants.
5214 If it is VOIDmode, they cannot both be constants.
5216 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5217 is not supported. */
5220 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
5221 machine_mode cmode
, rtx op2
, rtx op3
,
5222 machine_mode mode
, int unsignedp
)
5226 enum insn_code icode
;
5228 /* If one operand is constant, make it the second one. Only do this
5229 if the other operand is not constant as well. */
5231 if (swap_commutative_operands_p (op0
, op1
))
5233 std::swap (op0
, op1
);
5234 code
= swap_condition (code
);
5237 /* get_condition will prefer to generate LT and GT even if the old
5238 comparison was against zero, so undo that canonicalization here since
5239 comparisons against zero are cheaper. */
5240 if (code
== LT
&& op1
== const1_rtx
)
5241 code
= LE
, op1
= const0_rtx
;
5242 else if (code
== GT
&& op1
== constm1_rtx
)
5243 code
= GE
, op1
= const0_rtx
;
5245 if (cmode
== VOIDmode
)
5246 cmode
= GET_MODE (op0
);
5248 if (mode
== VOIDmode
)
5249 mode
= GET_MODE (op2
);
5251 icode
= optab_handler (addcc_optab
, mode
);
5253 if (icode
== CODE_FOR_nothing
)
5257 target
= gen_reg_rtx (mode
);
5259 code
= unsignedp
? unsigned_condition (code
) : code
;
5260 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
5262 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5263 return NULL and let the caller figure out how best to deal with this
5265 if (!COMPARISON_P (comparison
))
5268 do_pending_stack_adjust ();
5269 last
= get_last_insn ();
5270 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
5271 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
5272 &comparison
, &cmode
);
5275 class expand_operand ops
[4];
5277 create_output_operand (&ops
[0], target
, mode
);
5278 create_fixed_operand (&ops
[1], comparison
);
5279 create_input_operand (&ops
[2], op2
, mode
);
5280 create_input_operand (&ops
[3], op3
, mode
);
5281 if (maybe_expand_insn (icode
, 4, ops
))
5283 if (ops
[0].value
!= target
)
5284 convert_move (target
, ops
[0].value
, false);
5288 delete_insns_since (last
);
5292 /* These functions attempt to generate an insn body, rather than
5293 emitting the insn, but if the gen function already emits them, we
5294 make no attempt to turn them back into naked patterns. */
5296 /* Generate and return an insn body to add Y to X. */
5299 gen_add2_insn (rtx x
, rtx y
)
5301 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
5303 gcc_assert (insn_operand_matches (icode
, 0, x
));
5304 gcc_assert (insn_operand_matches (icode
, 1, x
));
5305 gcc_assert (insn_operand_matches (icode
, 2, y
));
5307 return GEN_FCN (icode
) (x
, x
, y
);
5310 /* Generate and return an insn body to add r1 and c,
5311 storing the result in r0. */
5314 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
5316 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
5318 if (icode
== CODE_FOR_nothing
5319 || !insn_operand_matches (icode
, 0, r0
)
5320 || !insn_operand_matches (icode
, 1, r1
)
5321 || !insn_operand_matches (icode
, 2, c
))
5324 return GEN_FCN (icode
) (r0
, r1
, c
);
5328 have_add2_insn (rtx x
, rtx y
)
5330 enum insn_code icode
;
5332 gcc_assert (GET_MODE (x
) != VOIDmode
);
5334 icode
= optab_handler (add_optab
, GET_MODE (x
));
5336 if (icode
== CODE_FOR_nothing
)
5339 if (!insn_operand_matches (icode
, 0, x
)
5340 || !insn_operand_matches (icode
, 1, x
)
5341 || !insn_operand_matches (icode
, 2, y
))
5347 /* Generate and return an insn body to add Y to X. */
5350 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
5352 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
5354 gcc_assert (insn_operand_matches (icode
, 0, x
));
5355 gcc_assert (insn_operand_matches (icode
, 1, y
));
5356 gcc_assert (insn_operand_matches (icode
, 2, z
));
5358 return GEN_FCN (icode
) (x
, y
, z
);
5361 /* Return true if the target implements an addptr pattern and X, Y,
5362 and Z are valid for the pattern predicates. */
5365 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
5367 enum insn_code icode
;
5369 gcc_assert (GET_MODE (x
) != VOIDmode
);
5371 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
5373 if (icode
== CODE_FOR_nothing
)
5376 if (!insn_operand_matches (icode
, 0, x
)
5377 || !insn_operand_matches (icode
, 1, y
)
5378 || !insn_operand_matches (icode
, 2, z
))
5384 /* Generate and return an insn body to subtract Y from X. */
5387 gen_sub2_insn (rtx x
, rtx y
)
5389 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
5391 gcc_assert (insn_operand_matches (icode
, 0, x
));
5392 gcc_assert (insn_operand_matches (icode
, 1, x
));
5393 gcc_assert (insn_operand_matches (icode
, 2, y
));
5395 return GEN_FCN (icode
) (x
, x
, y
);
5398 /* Generate and return an insn body to subtract r1 and c,
5399 storing the result in r0. */
5402 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
5404 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
5406 if (icode
== CODE_FOR_nothing
5407 || !insn_operand_matches (icode
, 0, r0
)
5408 || !insn_operand_matches (icode
, 1, r1
)
5409 || !insn_operand_matches (icode
, 2, c
))
5412 return GEN_FCN (icode
) (r0
, r1
, c
);
5416 have_sub2_insn (rtx x
, rtx y
)
5418 enum insn_code icode
;
5420 gcc_assert (GET_MODE (x
) != VOIDmode
);
5422 icode
= optab_handler (sub_optab
, GET_MODE (x
));
5424 if (icode
== CODE_FOR_nothing
)
5427 if (!insn_operand_matches (icode
, 0, x
)
5428 || !insn_operand_matches (icode
, 1, x
)
5429 || !insn_operand_matches (icode
, 2, y
))
5435 /* Generate the body of an insn to extend Y (with mode MFROM)
5436 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5439 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
5440 machine_mode mfrom
, int unsignedp
)
5442 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
5443 return GEN_FCN (icode
) (x
, y
);
5446 /* Generate code to convert FROM to floating point
5447 and store in TO. FROM must be fixed point and not VOIDmode.
5448 UNSIGNEDP nonzero means regard FROM as unsigned.
5449 Normally this is done by correcting the final value
5450 if it is negative. */
5453 expand_float (rtx to
, rtx from
, int unsignedp
)
5455 enum insn_code icode
;
5457 scalar_mode from_mode
, to_mode
;
5458 machine_mode fmode
, imode
;
5459 bool can_do_signed
= false;
5461 /* Crash now, because we won't be able to decide which mode to use. */
5462 gcc_assert (GET_MODE (from
) != VOIDmode
);
5464 /* Look for an insn to do the conversion. Do it in the specified
5465 modes if possible; otherwise convert either input, output or both to
5466 wider mode. If the integer mode is wider than the mode of FROM,
5467 we can do the conversion signed even if the input is unsigned. */
5469 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
5470 FOR_EACH_MODE_FROM (imode
, GET_MODE (from
))
5472 int doing_unsigned
= unsignedp
;
5474 if (fmode
!= GET_MODE (to
)
5475 && (significand_size (fmode
)
5476 < GET_MODE_UNIT_PRECISION (GET_MODE (from
))))
5479 icode
= can_float_p (fmode
, imode
, unsignedp
);
5480 if (icode
== CODE_FOR_nothing
&& unsignedp
)
5482 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
5483 if (scode
!= CODE_FOR_nothing
)
5484 can_do_signed
= true;
5485 if (imode
!= GET_MODE (from
))
5486 icode
= scode
, doing_unsigned
= 0;
5489 if (icode
!= CODE_FOR_nothing
)
5491 if (imode
!= GET_MODE (from
))
5492 from
= convert_to_mode (imode
, from
, unsignedp
);
5494 if (fmode
!= GET_MODE (to
))
5495 target
= gen_reg_rtx (fmode
);
5497 emit_unop_insn (icode
, target
, from
,
5498 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
5501 convert_move (to
, target
, 0);
5506 /* Unsigned integer, and no way to convert directly. Convert as signed,
5507 then unconditionally adjust the result. */
5510 && is_a
<scalar_mode
> (GET_MODE (to
), &to_mode
)
5511 && is_a
<scalar_mode
> (GET_MODE (from
), &from_mode
))
5513 opt_scalar_mode fmode_iter
;
5514 rtx_code_label
*label
= gen_label_rtx ();
5516 REAL_VALUE_TYPE offset
;
5518 /* Look for a usable floating mode FMODE wider than the source and at
5519 least as wide as the target. Using FMODE will avoid rounding woes
5520 with unsigned values greater than the signed maximum value. */
5522 FOR_EACH_MODE_FROM (fmode_iter
, to_mode
)
5524 scalar_mode fmode
= fmode_iter
.require ();
5525 if (GET_MODE_PRECISION (from_mode
) < GET_MODE_BITSIZE (fmode
)
5526 && can_float_p (fmode
, from_mode
, 0) != CODE_FOR_nothing
)
5530 if (!fmode_iter
.exists (&fmode
))
5532 /* There is no such mode. Pretend the target is wide enough. */
5535 /* Avoid double-rounding when TO is narrower than FROM. */
5536 if ((significand_size (fmode
) + 1)
5537 < GET_MODE_PRECISION (from_mode
))
5540 rtx_code_label
*neglabel
= gen_label_rtx ();
5542 /* Don't use TARGET if it isn't a register, is a hard register,
5543 or is the wrong mode. */
5545 || REGNO (target
) < FIRST_PSEUDO_REGISTER
5546 || GET_MODE (target
) != fmode
)
5547 target
= gen_reg_rtx (fmode
);
5550 do_pending_stack_adjust ();
5552 /* Test whether the sign bit is set. */
5553 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
5556 /* The sign bit is not set. Convert as signed. */
5557 expand_float (target
, from
, 0);
5558 emit_jump_insn (targetm
.gen_jump (label
));
5561 /* The sign bit is set.
5562 Convert to a usable (positive signed) value by shifting right
5563 one bit, while remembering if a nonzero bit was shifted
5564 out; i.e., compute (from & 1) | (from >> 1). */
5566 emit_label (neglabel
);
5567 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
5568 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
5569 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
5570 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
5572 expand_float (target
, temp
, 0);
5574 /* Multiply by 2 to undo the shift above. */
5575 temp
= expand_binop (fmode
, add_optab
, target
, target
,
5576 target
, 0, OPTAB_LIB_WIDEN
);
5578 emit_move_insn (target
, temp
);
5580 do_pending_stack_adjust ();
5586 /* If we are about to do some arithmetic to correct for an
5587 unsigned operand, do it in a pseudo-register. */
5589 if (to_mode
!= fmode
5590 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
5591 target
= gen_reg_rtx (fmode
);
5593 /* Convert as signed integer to floating. */
5594 expand_float (target
, from
, 0);
5596 /* If FROM is negative (and therefore TO is negative),
5597 correct its value by 2**bitwidth. */
5599 do_pending_stack_adjust ();
5600 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, from_mode
,
5604 real_2expN (&offset
, GET_MODE_PRECISION (from_mode
), fmode
);
5605 temp
= expand_binop (fmode
, add_optab
, target
,
5606 const_double_from_real_value (offset
, fmode
),
5607 target
, 0, OPTAB_LIB_WIDEN
);
5609 emit_move_insn (target
, temp
);
5611 do_pending_stack_adjust ();
5616 /* No hardware instruction available; call a library routine. */
5621 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
5623 if (is_narrower_int_mode (GET_MODE (from
), SImode
))
5624 from
= convert_to_mode (SImode
, from
, unsignedp
);
5626 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5627 gcc_assert (libfunc
);
5631 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5632 GET_MODE (to
), from
, GET_MODE (from
));
5633 insns
= get_insns ();
5636 emit_libcall_block (insns
, target
, value
,
5637 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
5638 GET_MODE (to
), from
));
5643 /* Copy result to requested destination
5644 if we have been computing in a temp location. */
5648 if (GET_MODE (target
) == GET_MODE (to
))
5649 emit_move_insn (to
, target
);
5651 convert_move (to
, target
, 0);
5655 /* Generate code to convert FROM to fixed point and store in TO. FROM
5656 must be floating point. */
5659 expand_fix (rtx to
, rtx from
, int unsignedp
)
5661 enum insn_code icode
;
5663 machine_mode fmode
, imode
;
5664 opt_scalar_mode fmode_iter
;
5665 bool must_trunc
= false;
5667 /* We first try to find a pair of modes, one real and one integer, at
5668 least as wide as FROM and TO, respectively, in which we can open-code
5669 this conversion. If the integer mode is wider than the mode of TO,
5670 we can do the conversion either signed or unsigned. */
5672 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5673 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5675 int doing_unsigned
= unsignedp
;
5677 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
5678 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
5679 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
5681 if (icode
!= CODE_FOR_nothing
)
5683 rtx_insn
*last
= get_last_insn ();
5685 if (fmode
!= GET_MODE (from
))
5687 if (REAL_MODE_FORMAT (GET_MODE (from
))
5688 == &arm_bfloat_half_format
5689 && REAL_MODE_FORMAT (fmode
) == &ieee_single_format
)
5690 /* The BF -> SF conversions can be just a shift, doesn't
5691 need to handle sNANs. */
5693 int save_flag_finite_math_only
= flag_finite_math_only
;
5694 flag_finite_math_only
= true;
5695 from1
= convert_to_mode (fmode
, from
, 0);
5696 flag_finite_math_only
= save_flag_finite_math_only
;
5699 from1
= convert_to_mode (fmode
, from
, 0);
5704 rtx temp
= gen_reg_rtx (GET_MODE (from1
));
5705 from1
= expand_unop (GET_MODE (from1
), ftrunc_optab
, from1
,
5709 if (imode
!= GET_MODE (to
))
5710 target
= gen_reg_rtx (imode
);
5712 if (maybe_emit_unop_insn (icode
, target
, from1
,
5713 doing_unsigned
? UNSIGNED_FIX
: FIX
))
5716 convert_move (to
, target
, unsignedp
);
5719 delete_insns_since (last
);
5723 /* For an unsigned conversion, there is one more way to do it.
5724 If we have a signed conversion, we generate code that compares
5725 the real value to the largest representable positive number. If if
5726 is smaller, the conversion is done normally. Otherwise, subtract
5727 one plus the highest signed number, convert, and add it back.
5729 We only need to check all real modes, since we know we didn't find
5730 anything with a wider integer mode.
5732 This code used to extend FP value into mode wider than the destination.
5733 This is needed for decimal float modes which cannot accurately
5734 represent one plus the highest signed number of the same size, but
5735 not for binary modes. Consider, for instance conversion from SFmode
5738 The hot path through the code is dealing with inputs smaller than 2^63
5739 and doing just the conversion, so there is no bits to lose.
5741 In the other path we know the value is positive in the range 2^63..2^64-1
5742 inclusive. (as for other input overflow happens and result is undefined)
5743 So we know that the most important bit set in mantissa corresponds to
5744 2^63. The subtraction of 2^63 should not generate any rounding as it
5745 simply clears out that bit. The rest is trivial. */
5747 scalar_int_mode to_mode
;
5749 && is_a
<scalar_int_mode
> (GET_MODE (to
), &to_mode
)
5750 && HWI_COMPUTABLE_MODE_P (to_mode
))
5751 FOR_EACH_MODE_FROM (fmode_iter
, as_a
<scalar_mode
> (GET_MODE (from
)))
5753 scalar_mode fmode
= fmode_iter
.require ();
5754 if (CODE_FOR_nothing
!= can_fix_p (to_mode
, fmode
,
5756 && (!DECIMAL_FLOAT_MODE_P (fmode
)
5757 || (GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (to_mode
))))
5760 REAL_VALUE_TYPE offset
;
5762 rtx_code_label
*lab1
, *lab2
;
5765 bitsize
= GET_MODE_PRECISION (to_mode
);
5766 real_2expN (&offset
, bitsize
- 1, fmode
);
5767 limit
= const_double_from_real_value (offset
, fmode
);
5768 lab1
= gen_label_rtx ();
5769 lab2
= gen_label_rtx ();
5771 if (fmode
!= GET_MODE (from
))
5773 if (REAL_MODE_FORMAT (GET_MODE (from
))
5774 == &arm_bfloat_half_format
5775 && REAL_MODE_FORMAT (fmode
) == &ieee_single_format
)
5776 /* The BF -> SF conversions can be just a shift, doesn't
5777 need to handle sNANs. */
5779 int save_flag_finite_math_only
= flag_finite_math_only
;
5780 flag_finite_math_only
= true;
5781 from
= convert_to_mode (fmode
, from
, 0);
5782 flag_finite_math_only
= save_flag_finite_math_only
;
5785 from
= convert_to_mode (fmode
, from
, 0);
5788 /* See if we need to do the subtraction. */
5789 do_pending_stack_adjust ();
5790 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
,
5791 GET_MODE (from
), 0, lab1
);
5793 /* If not, do the signed "fix" and branch around fixup code. */
5794 expand_fix (to
, from
, 0);
5795 emit_jump_insn (targetm
.gen_jump (lab2
));
5798 /* Otherwise, subtract 2**(N-1), convert to signed number,
5799 then add 2**(N-1). Do the addition using XOR since this
5800 will often generate better code. */
5802 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
5803 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
5804 expand_fix (to
, target
, 0);
5805 target
= expand_binop (to_mode
, xor_optab
, to
,
5807 (HOST_WIDE_INT_1
<< (bitsize
- 1),
5809 to
, 1, OPTAB_LIB_WIDEN
);
5812 emit_move_insn (to
, target
);
5816 if (optab_handler (mov_optab
, to_mode
) != CODE_FOR_nothing
)
5818 /* Make a place for a REG_NOTE and add it. */
5819 insn
= emit_move_insn (to
, to
);
5820 set_dst_reg_note (insn
, REG_EQUAL
,
5821 gen_rtx_fmt_e (UNSIGNED_FIX
, to_mode
,
5831 if (REAL_MODE_FORMAT (GET_MODE (from
)) == &arm_bfloat_half_format
5832 && REAL_MODE_FORMAT (SFmode
) == &ieee_single_format
)
5833 /* We don't have BF -> TI library functions, use BF -> SF -> TI
5834 instead but the BF -> SF conversion can be just a shift, doesn't
5835 need to handle sNANs. */
5837 int save_flag_finite_math_only
= flag_finite_math_only
;
5838 flag_finite_math_only
= true;
5839 from
= convert_to_mode (SFmode
, from
, 0);
5840 flag_finite_math_only
= save_flag_finite_math_only
;
5841 expand_fix (to
, from
, unsignedp
);
5846 /* We can't do it with an insn, so use a library call. But first ensure
5847 that the mode of TO is at least as wide as SImode, since those are the
5848 only library calls we know about. */
5850 if (is_narrower_int_mode (GET_MODE (to
), SImode
))
5852 target
= gen_reg_rtx (SImode
);
5854 expand_fix (target
, from
, unsignedp
);
5862 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
5863 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5864 gcc_assert (libfunc
);
5868 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5869 GET_MODE (to
), from
, GET_MODE (from
));
5870 insns
= get_insns ();
5873 emit_libcall_block (insns
, target
, value
,
5874 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5875 GET_MODE (to
), from
));
5880 if (GET_MODE (to
) == GET_MODE (target
))
5881 emit_move_insn (to
, target
);
5883 convert_move (to
, target
, 0);
5888 /* Promote integer arguments for a libcall if necessary.
5889 emit_library_call_value cannot do the promotion because it does not
5890 know if it should do a signed or unsigned promotion. This is because
5891 there are no tree types defined for libcalls. */
5894 prepare_libcall_arg (rtx arg
, int uintp
)
5896 scalar_int_mode mode
;
5897 machine_mode arg_mode
;
5898 if (is_a
<scalar_int_mode
> (GET_MODE (arg
), &mode
))
5900 /* If we need to promote the integer function argument we need to do
5901 it here instead of inside emit_library_call_value because in
5902 emit_library_call_value we don't know if we should do a signed or
5903 unsigned promotion. */
5906 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
5907 &unsigned_p
, NULL_TREE
, 0);
5908 if (arg_mode
!= mode
)
5909 return convert_to_mode (arg_mode
, arg
, uintp
);
5914 /* Generate code to convert FROM or TO a fixed-point.
5915 If UINTP is true, either TO or FROM is an unsigned integer.
5916 If SATP is true, we need to saturate the result. */
5919 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
5921 machine_mode to_mode
= GET_MODE (to
);
5922 machine_mode from_mode
= GET_MODE (from
);
5924 enum rtx_code this_code
;
5925 enum insn_code code
;
5930 if (to_mode
== from_mode
)
5932 emit_move_insn (to
, from
);
5938 tab
= satp
? satfractuns_optab
: fractuns_optab
;
5939 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
5943 tab
= satp
? satfract_optab
: fract_optab
;
5944 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
5946 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
5947 if (code
!= CODE_FOR_nothing
)
5949 emit_unop_insn (code
, to
, from
, this_code
);
5953 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
5954 gcc_assert (libfunc
);
5956 from
= prepare_libcall_arg (from
, uintp
);
5957 from_mode
= GET_MODE (from
);
5960 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
5962 insns
= get_insns ();
5965 emit_libcall_block (insns
, to
, value
,
5966 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
5969 /* Generate code to convert FROM to fixed point and store in TO. FROM
5970 must be floating point, TO must be signed. Use the conversion optab
5971 TAB to do the conversion. */
5974 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
5976 enum insn_code icode
;
5978 machine_mode fmode
, imode
;
5980 /* We first try to find a pair of modes, one real and one integer, at
5981 least as wide as FROM and TO, respectively, in which we can open-code
5982 this conversion. If the integer mode is wider than the mode of TO,
5983 we can do the conversion either signed or unsigned. */
5985 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5986 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5988 icode
= convert_optab_handler (tab
, imode
, fmode
,
5989 insn_optimization_type ());
5990 if (icode
!= CODE_FOR_nothing
)
5992 rtx_insn
*last
= get_last_insn ();
5993 if (fmode
!= GET_MODE (from
))
5994 from
= convert_to_mode (fmode
, from
, 0);
5996 if (imode
!= GET_MODE (to
))
5997 target
= gen_reg_rtx (imode
);
5999 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
6001 delete_insns_since (last
);
6005 convert_move (to
, target
, 0);
6013 /* Report whether we have an instruction to perform the operation
6014 specified by CODE on operands of mode MODE. */
6016 have_insn_for (enum rtx_code code
, machine_mode mode
)
6018 return (code_to_optab (code
)
6019 && (optab_handler (code_to_optab (code
), mode
)
6020 != CODE_FOR_nothing
));
6023 /* Print information about the current contents of the optabs on
6027 debug_optab_libfuncs (void)
6031 /* Dump the arithmetic optabs. */
6032 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
6033 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
6035 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
6038 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
6039 fprintf (stderr
, "%s\t%s:\t%s\n",
6040 GET_RTX_NAME (optab_to_code ((optab
) i
)),
6046 /* Dump the conversion optabs. */
6047 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
6048 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
6049 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
6051 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
6055 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
6056 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
6057 GET_RTX_NAME (optab_to_code ((optab
) i
)),
6065 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6066 CODE. Return 0 on failure. */
6069 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
6071 machine_mode mode
= GET_MODE (op1
);
6072 enum insn_code icode
;
6076 if (mode
== VOIDmode
)
6079 icode
= optab_handler (ctrap_optab
, mode
);
6080 if (icode
== CODE_FOR_nothing
)
6083 /* Some targets only accept a zero trap code. */
6084 if (!insn_operand_matches (icode
, 3, tcode
))
6087 do_pending_stack_adjust ();
6089 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
6094 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
6097 /* If that failed, then give up. */
6105 insn
= get_insns ();
6110 /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
6111 or unsigned operation code. */
6114 get_rtx_code_1 (enum tree_code tcode
, bool unsignedp
)
6126 code
= unsignedp
? LTU
: LT
;
6129 code
= unsignedp
? LEU
: LE
;
6132 code
= unsignedp
? GTU
: GT
;
6135 code
= unsignedp
? GEU
: GE
;
6138 case UNORDERED_EXPR
:
6178 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6179 or unsigned operation code. */
6182 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
6184 enum rtx_code code
= get_rtx_code_1 (tcode
, unsignedp
);
6185 gcc_assert (code
!= UNKNOWN
);
6189 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6190 select signed or unsigned operators. OPNO holds the index of the
6191 first comparison operand for insn ICODE. Do not generate the
6192 compare instruction itself. */
6195 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
6196 tree t_op0
, tree t_op1
, bool unsignedp
,
6197 enum insn_code icode
, unsigned int opno
)
6199 class expand_operand ops
[2];
6200 rtx rtx_op0
, rtx_op1
;
6201 machine_mode m0
, m1
;
6202 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
6204 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
6206 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6207 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6208 cases, use the original mode. */
6209 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
6211 m0
= GET_MODE (rtx_op0
);
6213 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
6215 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
6217 m1
= GET_MODE (rtx_op1
);
6219 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
6221 create_input_operand (&ops
[0], rtx_op0
, m0
);
6222 create_input_operand (&ops
[1], rtx_op1
, m1
);
6223 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
6225 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
6228 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
6229 the first vec_perm operand, assuming the second operand (for left shift
6230 first operand) is a constant vector of zeros. Return the shift distance
6231 in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6232 mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6233 shift or vec_shl_optab for left shift. */
6235 shift_amt_for_vec_perm_mask (machine_mode mode
, const vec_perm_indices
&sel
,
6238 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (mode
);
6239 poly_int64 first
= sel
[0];
6240 if (maybe_ge (sel
[0], GET_MODE_NUNITS (mode
)))
6243 if (shift_optab
== vec_shl_optab
)
6246 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
6248 unsigned firstidx
= 0;
6249 for (unsigned int i
= 0; i
< nelt
; i
++)
6251 if (known_eq (sel
[i
], nelt
))
6253 if (i
== 0 || firstidx
)
6258 ? maybe_ne (sel
[i
], nelt
+ i
- firstidx
)
6259 : maybe_ge (sel
[i
], nelt
))
6267 else if (!sel
.series_p (0, 1, first
, 1))
6270 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
6272 for (unsigned int i
= 1; i
< nelt
; i
++)
6274 poly_int64 expected
= i
+ first
;
6275 /* Indices into the second vector are all equivalent. */
6276 if (maybe_lt (sel
[i
], nelt
)
6277 ? maybe_ne (sel
[i
], expected
)
6278 : maybe_lt (expected
, nelt
))
6283 return gen_int_shift_amount (mode
, first
* bitsize
);
6286 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6289 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
6290 rtx v0
, rtx v1
, rtx sel
)
6292 machine_mode tmode
= GET_MODE (target
);
6293 machine_mode smode
= GET_MODE (sel
);
6294 class expand_operand ops
[4];
6296 gcc_assert (GET_MODE_CLASS (smode
) == MODE_VECTOR_INT
6297 || related_int_vector_mode (tmode
).require () == smode
);
6298 create_output_operand (&ops
[0], target
, tmode
);
6299 create_input_operand (&ops
[3], sel
, smode
);
6301 /* Make an effort to preserve v0 == v1. The target expander is able to
6302 rely on this to determine if we're permuting a single input operand. */
6303 if (rtx_equal_p (v0
, v1
))
6305 if (!insn_operand_matches (icode
, 1, v0
))
6306 v0
= force_reg (tmode
, v0
);
6307 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
6308 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
6310 create_fixed_operand (&ops
[1], v0
);
6311 create_fixed_operand (&ops
[2], v0
);
6315 create_input_operand (&ops
[1], v0
, tmode
);
6316 create_input_operand (&ops
[2], v1
, tmode
);
6319 if (maybe_expand_insn (icode
, 4, ops
))
6320 return ops
[0].value
;
6324 /* Implement a permutation of vectors v0 and v1 using the permutation
6325 vector in SEL and return the result. Use TARGET to hold the result
6326 if nonnull and convenient.
6328 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
6329 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
6330 to have a particular mode. */
6333 expand_vec_perm_const (machine_mode mode
, rtx v0
, rtx v1
,
6334 const vec_perm_builder
&sel
, machine_mode sel_mode
,
6337 if (!target
|| !register_operand (target
, mode
))
6338 target
= gen_reg_rtx (mode
);
6340 /* Set QIMODE to a different vector mode with byte elements.
6341 If no such mode, or if MODE already has byte elements, use VOIDmode. */
6342 machine_mode qimode
;
6343 if (!qimode_for_vec_perm (mode
).exists (&qimode
))
6346 rtx_insn
*last
= get_last_insn ();
6348 bool single_arg_p
= rtx_equal_p (v0
, v1
);
6349 /* Always specify two input vectors here and leave the target to handle
6350 cases in which the inputs are equal. Not all backends can cope with
6351 the single-input representation when testing for a double-input
6352 target instruction. */
6353 vec_perm_indices
indices (sel
, 2, GET_MODE_NUNITS (mode
));
6355 /* See if this can be handled with a vec_shr or vec_shl. We only do this
6356 if the second (for vec_shr) or first (for vec_shl) vector is all
6358 insn_code shift_code
= CODE_FOR_nothing
;
6359 insn_code shift_code_qi
= CODE_FOR_nothing
;
6360 optab shift_optab
= unknown_optab
;
6362 if (v1
== CONST0_RTX (GET_MODE (v1
)))
6363 shift_optab
= vec_shr_optab
;
6364 else if (v0
== CONST0_RTX (GET_MODE (v0
)))
6366 shift_optab
= vec_shl_optab
;
6369 if (shift_optab
!= unknown_optab
)
6371 shift_code
= optab_handler (shift_optab
, mode
);
6372 shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
6373 ? optab_handler (shift_optab
, qimode
)
6374 : CODE_FOR_nothing
);
6376 if (shift_code
!= CODE_FOR_nothing
|| shift_code_qi
!= CODE_FOR_nothing
)
6378 rtx shift_amt
= shift_amt_for_vec_perm_mask (mode
, indices
, shift_optab
);
6381 class expand_operand ops
[3];
6382 if (shift_amt
== const0_rtx
)
6384 if (shift_code
!= CODE_FOR_nothing
)
6386 create_output_operand (&ops
[0], target
, mode
);
6387 create_input_operand (&ops
[1], v2
, mode
);
6388 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
6389 if (maybe_expand_insn (shift_code
, 3, ops
))
6390 return ops
[0].value
;
6392 if (shift_code_qi
!= CODE_FOR_nothing
)
6394 rtx tmp
= gen_reg_rtx (qimode
);
6395 create_output_operand (&ops
[0], tmp
, qimode
);
6396 create_input_operand (&ops
[1], gen_lowpart (qimode
, v2
), qimode
);
6397 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
6398 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
6399 return gen_lowpart (mode
, ops
[0].value
);
6404 if (targetm
.vectorize
.vec_perm_const
!= NULL
)
6409 gcc_checking_assert (GET_MODE (v0
) == GET_MODE (v1
));
6410 machine_mode op_mode
= GET_MODE (v0
);
6411 if (targetm
.vectorize
.vec_perm_const (mode
, op_mode
, target
, v0
, v1
,
6416 /* Fall back to a constant byte-based permutation. */
6417 vec_perm_indices qimode_indices
;
6418 rtx target_qi
= NULL_RTX
, v0_qi
= NULL_RTX
, v1_qi
= NULL_RTX
;
6419 if (qimode
!= VOIDmode
)
6421 qimode_indices
.new_expanded_vector (indices
, GET_MODE_UNIT_SIZE (mode
));
6422 target_qi
= gen_reg_rtx (qimode
);
6423 v0_qi
= gen_lowpart (qimode
, v0
);
6424 v1_qi
= gen_lowpart (qimode
, v1
);
6425 if (targetm
.vectorize
.vec_perm_const
!= NULL
6426 && targetm
.vectorize
.vec_perm_const (qimode
, qimode
, target_qi
, v0_qi
,
6427 v1_qi
, qimode_indices
))
6428 return gen_lowpart (mode
, target_qi
);
6431 v0
= force_reg (mode
, v0
);
6434 v1
= force_reg (mode
, v1
);
6436 /* Otherwise expand as a fully variable permuation. */
6438 /* The optabs are only defined for selectors with the same width
6439 as the values being permuted. */
6440 machine_mode required_sel_mode
;
6441 if (!related_int_vector_mode (mode
).exists (&required_sel_mode
))
6443 delete_insns_since (last
);
6447 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
6448 If that isn't the mode we want then we need to prove that using
6449 REQUIRED_SEL_MODE is OK. */
6450 if (sel_mode
!= required_sel_mode
)
6452 if (!selector_fits_mode_p (required_sel_mode
, indices
))
6454 delete_insns_since (last
);
6457 sel_mode
= required_sel_mode
;
6460 insn_code icode
= direct_optab_handler (vec_perm_optab
, mode
);
6461 if (icode
!= CODE_FOR_nothing
)
6463 rtx sel_rtx
= vec_perm_indices_to_rtx (sel_mode
, indices
);
6464 rtx tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel_rtx
);
6469 if (qimode
!= VOIDmode
6470 && selector_fits_mode_p (qimode
, qimode_indices
))
6472 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6473 if (icode
!= CODE_FOR_nothing
)
6475 rtx sel_qi
= vec_perm_indices_to_rtx (qimode
, qimode_indices
);
6476 rtx tmp
= expand_vec_perm_1 (icode
, target_qi
, v0_qi
, v1_qi
, sel_qi
);
6478 return gen_lowpart (mode
, tmp
);
6482 delete_insns_since (last
);
6486 /* Implement a permutation of vectors v0 and v1 using the permutation
6487 vector in SEL and return the result. Use TARGET to hold the result
6488 if nonnull and convenient.
6490 MODE is the mode of the vectors being permuted (V0 and V1).
6491 SEL must have the integer equivalent of MODE and is known to be
6492 unsuitable for permutes with a constant permutation vector. */
6495 expand_vec_perm_var (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
6497 enum insn_code icode
;
6501 u
= GET_MODE_UNIT_SIZE (mode
);
6503 if (!target
|| GET_MODE (target
) != mode
)
6504 target
= gen_reg_rtx (mode
);
6506 icode
= direct_optab_handler (vec_perm_optab
, mode
);
6507 if (icode
!= CODE_FOR_nothing
)
6509 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
6514 /* As a special case to aid several targets, lower the element-based
6515 permutation to a byte-based permutation and try again. */
6516 machine_mode qimode
;
6517 if (!qimode_for_vec_perm (mode
).exists (&qimode
)
6518 || maybe_gt (GET_MODE_NUNITS (qimode
), GET_MODE_MASK (QImode
) + 1))
6520 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6521 if (icode
== CODE_FOR_nothing
)
6524 /* Multiply each element by its byte size. */
6525 machine_mode selmode
= GET_MODE (sel
);
6527 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
6528 NULL
, 0, OPTAB_DIRECT
);
6530 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
6531 gen_int_shift_amount (selmode
, exact_log2 (u
)),
6532 NULL
, 0, OPTAB_DIRECT
);
6533 gcc_assert (sel
!= NULL
);
6535 /* Broadcast the low byte each element into each of its bytes.
6536 The encoding has U interleaved stepped patterns, one for each
6537 byte of an element. */
6538 vec_perm_builder
const_sel (GET_MODE_SIZE (mode
), u
, 3);
6539 unsigned int low_byte_in_u
= BYTES_BIG_ENDIAN
? u
- 1 : 0;
6540 for (i
= 0; i
< 3; ++i
)
6541 for (unsigned int j
= 0; j
< u
; ++j
)
6542 const_sel
.quick_push (i
* u
+ low_byte_in_u
);
6543 sel
= gen_lowpart (qimode
, sel
);
6544 sel
= expand_vec_perm_const (qimode
, sel
, sel
, const_sel
, qimode
, NULL
);
6545 gcc_assert (sel
!= NULL
);
6547 /* Add the byte offset to each byte element. */
6548 /* Note that the definition of the indicies here is memory ordering,
6549 so there should be no difference between big and little endian. */
6550 rtx_vector_builder
byte_indices (qimode
, u
, 1);
6551 for (i
= 0; i
< u
; ++i
)
6552 byte_indices
.quick_push (GEN_INT (i
));
6553 tmp
= byte_indices
.build ();
6554 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
6555 sel
, 0, OPTAB_DIRECT
);
6556 gcc_assert (sel_qi
!= NULL
);
6558 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
6559 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
6560 gen_lowpart (qimode
, v1
), sel_qi
);
6562 tmp
= gen_lowpart (mode
, tmp
);
6566 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
6567 Use TARGET for the result if nonnull and convenient. */
6570 expand_vec_series_expr (machine_mode vmode
, rtx op0
, rtx op1
, rtx target
)
6572 class expand_operand ops
[3];
6573 enum insn_code icode
;
6574 machine_mode emode
= GET_MODE_INNER (vmode
);
6576 icode
= direct_optab_handler (vec_series_optab
, vmode
);
6577 gcc_assert (icode
!= CODE_FOR_nothing
);
6579 create_output_operand (&ops
[0], target
, vmode
);
6580 create_input_operand (&ops
[1], op0
, emode
);
6581 create_input_operand (&ops
[2], op1
, emode
);
6583 expand_insn (icode
, 3, ops
);
6584 return ops
[0].value
;
6587 /* Generate insns for a vector comparison into a mask. */
6590 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
6592 class expand_operand ops
[4];
6593 enum insn_code icode
;
6595 machine_mode mask_mode
= TYPE_MODE (type
);
6599 enum tree_code tcode
;
6601 op0a
= TREE_OPERAND (exp
, 0);
6602 op0b
= TREE_OPERAND (exp
, 1);
6603 tcode
= TREE_CODE (exp
);
6605 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
6606 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
6608 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
6609 if (icode
== CODE_FOR_nothing
)
6611 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
6612 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
6613 if (icode
== CODE_FOR_nothing
)
6617 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
6618 unsignedp
, icode
, 2);
6619 create_output_operand (&ops
[0], target
, mask_mode
);
6620 create_fixed_operand (&ops
[1], comparison
);
6621 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
6622 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
6623 expand_insn (icode
, 4, ops
);
6624 return ops
[0].value
;
6627 /* Expand a highpart multiply. */
6630 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
6631 rtx target
, bool uns_p
)
6633 class expand_operand eops
[3];
6634 enum insn_code icode
;
6640 method
= can_mult_highpart_p (mode
, uns_p
);
6646 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
6647 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
6650 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
6651 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
6654 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
6655 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
6656 if (BYTES_BIG_ENDIAN
)
6657 std::swap (tab1
, tab2
);
6663 icode
= optab_handler (tab1
, mode
);
6664 wmode
= insn_data
[icode
].operand
[0].mode
;
6665 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode
),
6666 GET_MODE_NUNITS (mode
)));
6667 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode
), GET_MODE_SIZE (mode
)));
6669 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
6670 create_input_operand (&eops
[1], op0
, mode
);
6671 create_input_operand (&eops
[2], op1
, mode
);
6672 expand_insn (icode
, 3, eops
);
6673 m1
= gen_lowpart (mode
, eops
[0].value
);
6675 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
6676 create_input_operand (&eops
[1], op0
, mode
);
6677 create_input_operand (&eops
[2], op1
, mode
);
6678 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
6679 m2
= gen_lowpart (mode
, eops
[0].value
);
6681 vec_perm_builder sel
;
6684 /* The encoding has 2 interleaved stepped patterns. */
6685 sel
.new_vector (GET_MODE_NUNITS (mode
), 2, 3);
6686 for (i
= 0; i
< 6; ++i
)
6687 sel
.quick_push (!BYTES_BIG_ENDIAN
+ (i
& ~1)
6688 + ((i
& 1) ? GET_MODE_NUNITS (mode
) : 0));
6692 /* The encoding has a single interleaved stepped pattern. */
6693 sel
.new_vector (GET_MODE_NUNITS (mode
), 1, 3);
6694 for (i
= 0; i
< 3; ++i
)
6695 sel
.quick_push (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
6698 return expand_vec_perm_const (mode
, m1
, m2
, sel
, BLKmode
, target
);
6701 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6705 find_cc_set (rtx x
, const_rtx pat
, void *data
)
6707 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
6708 && GET_CODE (pat
) == SET
)
6710 rtx
*p_cc_reg
= (rtx
*) data
;
6711 gcc_assert (!*p_cc_reg
);
6716 /* This is a helper function for the other atomic operations. This function
6717 emits a loop that contains SEQ that iterates until a compare-and-swap
6718 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6719 a set of instructions that takes a value from OLD_REG as an input and
6720 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6721 set to the current contents of MEM. After SEQ, a compare-and-swap will
6722 attempt to update MEM with NEW_REG. The function returns true when the
6723 loop was generated successfully. */
6726 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
6728 machine_mode mode
= GET_MODE (mem
);
6729 rtx_code_label
*label
;
6730 rtx cmp_reg
, success
, oldval
;
6732 /* The loop we want to generate looks like
6738 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6742 Note that we only do the plain load from memory once. Subsequent
6743 iterations use the value loaded by the compare-and-swap pattern. */
6745 label
= gen_label_rtx ();
6746 cmp_reg
= gen_reg_rtx (mode
);
6748 emit_move_insn (cmp_reg
, mem
);
6750 emit_move_insn (old_reg
, cmp_reg
);
6756 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
6757 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
6761 if (oldval
!= cmp_reg
)
6762 emit_move_insn (cmp_reg
, oldval
);
6764 /* Mark this jump predicted not taken. */
6765 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
6766 GET_MODE (success
), 1, label
,
6767 profile_probability::guessed_never ());
6772 /* This function tries to emit an atomic_exchange intruction. VAL is written
6773 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6774 using TARGET if possible. */
6777 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6779 machine_mode mode
= GET_MODE (mem
);
6780 enum insn_code icode
;
6782 /* If the target supports the exchange directly, great. */
6783 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
6784 if (icode
!= CODE_FOR_nothing
)
6786 class expand_operand ops
[4];
6788 create_output_operand (&ops
[0], target
, mode
);
6789 create_fixed_operand (&ops
[1], mem
);
6790 create_input_operand (&ops
[2], val
, mode
);
6791 create_integer_operand (&ops
[3], model
);
6792 if (maybe_expand_insn (icode
, 4, ops
))
6793 return ops
[0].value
;
6799 /* This function tries to implement an atomic exchange operation using
6800 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6801 The previous contents of *MEM are returned, using TARGET if possible.
6802 Since this instructionn is an acquire barrier only, stronger memory
6803 models may require additional barriers to be emitted. */
6806 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
6807 enum memmodel model
)
6809 machine_mode mode
= GET_MODE (mem
);
6810 enum insn_code icode
;
6811 rtx_insn
*last_insn
= get_last_insn ();
6813 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
6815 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6816 exists, and the memory model is stronger than acquire, add a release
6817 barrier before the instruction. */
6819 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
6820 expand_mem_thread_fence (model
);
6822 if (icode
!= CODE_FOR_nothing
)
6824 class expand_operand ops
[3];
6825 create_output_operand (&ops
[0], target
, mode
);
6826 create_fixed_operand (&ops
[1], mem
);
6827 create_input_operand (&ops
[2], val
, mode
);
6828 if (maybe_expand_insn (icode
, 3, ops
))
6829 return ops
[0].value
;
6832 /* If an external test-and-set libcall is provided, use that instead of
6833 any external compare-and-swap that we might get from the compare-and-
6834 swap-loop expansion later. */
6835 if (!can_compare_and_swap_p (mode
, false))
6837 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
6838 if (libfunc
!= NULL
)
6842 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6843 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6844 mode
, addr
, ptr_mode
,
6849 /* If the test_and_set can't be emitted, eliminate any barrier that might
6850 have been emitted. */
6851 delete_insns_since (last_insn
);
6855 /* This function tries to implement an atomic exchange operation using a
6856 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6857 *MEM are returned, using TARGET if possible. No memory model is required
6858 since a compare_and_swap loop is seq-cst. */
6861 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
6863 machine_mode mode
= GET_MODE (mem
);
6865 if (can_compare_and_swap_p (mode
, true))
6867 if (!target
|| !register_operand (target
, mode
))
6868 target
= gen_reg_rtx (mode
);
6869 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
6876 /* This function tries to implement an atomic test-and-set operation
6877 using the atomic_test_and_set instruction pattern. A boolean value
6878 is returned from the operation, using TARGET if possible. */
6881 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6883 machine_mode pat_bool_mode
;
6884 class expand_operand ops
[3];
6886 if (!targetm
.have_atomic_test_and_set ())
6889 /* While we always get QImode from __atomic_test_and_set, we get
6890 other memory modes from __sync_lock_test_and_set. Note that we
6891 use no endian adjustment here. This matches the 4.6 behavior
6892 in the Sparc backend. */
6893 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
6894 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
6895 if (GET_MODE (mem
) != QImode
)
6896 mem
= adjust_address_nv (mem
, QImode
, 0);
6898 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
6899 create_output_operand (&ops
[0], target
, pat_bool_mode
);
6900 create_fixed_operand (&ops
[1], mem
);
6901 create_integer_operand (&ops
[2], model
);
6903 if (maybe_expand_insn (icode
, 3, ops
))
6904 return ops
[0].value
;
6908 /* This function expands the legacy _sync_lock test_and_set operation which is
6909 generally an atomic exchange. Some limited targets only allow the
6910 constant 1 to be stored. This is an ACQUIRE operation.
6912 TARGET is an optional place to stick the return value.
6913 MEM is where VAL is stored. */
6916 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
6920 /* Try an atomic_exchange first. */
6921 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
6925 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
6926 MEMMODEL_SYNC_ACQUIRE
);
6930 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6934 /* If there are no other options, try atomic_test_and_set if the value
6935 being stored is 1. */
6936 if (val
== const1_rtx
)
6937 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
6942 /* This function expands the atomic test_and_set operation:
6943 atomically store a boolean TRUE into MEM and return the previous value.
6945 MEMMODEL is the memory model variant to use.
6946 TARGET is an optional place to stick the return value. */
6949 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6951 machine_mode mode
= GET_MODE (mem
);
6952 rtx ret
, trueval
, subtarget
;
6954 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
6958 /* Be binary compatible with non-default settings of trueval, and different
6959 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6960 another only has atomic-exchange. */
6961 if (targetm
.atomic_test_and_set_trueval
== 1)
6963 trueval
= const1_rtx
;
6964 subtarget
= target
? target
: gen_reg_rtx (mode
);
6968 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
6969 subtarget
= gen_reg_rtx (mode
);
6972 /* Try the atomic-exchange optab... */
6973 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
6975 /* ... then an atomic-compare-and-swap loop ... */
6977 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
6979 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6981 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
6983 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6984 things with the value 1. Thus we try again without trueval. */
6985 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
6986 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
6988 /* Failing all else, assume a single threaded environment and simply
6989 perform the operation. */
6992 /* If the result is ignored skip the move to target. */
6993 if (subtarget
!= const0_rtx
)
6994 emit_move_insn (subtarget
, mem
);
6996 emit_move_insn (mem
, trueval
);
7000 /* Recall that have to return a boolean value; rectify if trueval
7001 is not exactly one. */
7002 if (targetm
.atomic_test_and_set_trueval
!= 1)
7003 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
7008 /* This function expands the atomic exchange operation:
7009 atomically store VAL in MEM and return the previous value in MEM.
7011 MEMMODEL is the memory model variant to use.
7012 TARGET is an optional place to stick the return value. */
7015 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
7017 machine_mode mode
= GET_MODE (mem
);
7020 /* If loads are not atomic for the required size and we are not called to
7021 provide a __sync builtin, do not do anything so that we stay consistent
7022 with atomic loads of the same size. */
7023 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
7026 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7028 /* Next try a compare-and-swap loop for the exchange. */
7030 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
7035 /* This function expands the atomic compare exchange operation:
7037 *PTARGET_BOOL is an optional place to store the boolean success/failure.
7038 *PTARGET_OVAL is an optional place to store the old value from memory.
7039 Both target parameters may be NULL or const0_rtx to indicate that we do
7040 not care about that return value. Both target parameters are updated on
7041 success to the actual location of the corresponding result.
7043 MEMMODEL is the memory model variant to use.
7045 The return value of the function is true for success. */
7048 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
7049 rtx mem
, rtx expected
, rtx desired
,
7050 bool is_weak
, enum memmodel succ_model
,
7051 enum memmodel fail_model
)
7053 machine_mode mode
= GET_MODE (mem
);
7054 class expand_operand ops
[8];
7055 enum insn_code icode
;
7056 rtx target_oval
, target_bool
= NULL_RTX
;
7059 /* If loads are not atomic for the required size and we are not called to
7060 provide a __sync builtin, do not do anything so that we stay consistent
7061 with atomic loads of the same size. */
7062 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
7065 /* Load expected into a register for the compare and swap. */
7066 if (MEM_P (expected
))
7067 expected
= copy_to_reg (expected
);
7069 /* Make sure we always have some place to put the return oldval.
7070 Further, make sure that place is distinct from the input expected,
7071 just in case we need that path down below. */
7072 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
7073 ptarget_oval
= NULL
;
7075 if (ptarget_oval
== NULL
7076 || (target_oval
= *ptarget_oval
) == NULL
7077 || reg_overlap_mentioned_p (expected
, target_oval
))
7078 target_oval
= gen_reg_rtx (mode
);
7080 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
7081 if (icode
!= CODE_FOR_nothing
)
7083 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
7085 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
7086 ptarget_bool
= NULL
;
7088 /* Make sure we always have a place for the bool operand. */
7089 if (ptarget_bool
== NULL
7090 || (target_bool
= *ptarget_bool
) == NULL
7091 || GET_MODE (target_bool
) != bool_mode
)
7092 target_bool
= gen_reg_rtx (bool_mode
);
7094 /* Emit the compare_and_swap. */
7095 create_output_operand (&ops
[0], target_bool
, bool_mode
);
7096 create_output_operand (&ops
[1], target_oval
, mode
);
7097 create_fixed_operand (&ops
[2], mem
);
7098 create_input_operand (&ops
[3], expected
, mode
);
7099 create_input_operand (&ops
[4], desired
, mode
);
7100 create_integer_operand (&ops
[5], is_weak
);
7101 create_integer_operand (&ops
[6], succ_model
);
7102 create_integer_operand (&ops
[7], fail_model
);
7103 if (maybe_expand_insn (icode
, 8, ops
))
7105 /* Return success/failure. */
7106 target_bool
= ops
[0].value
;
7107 target_oval
= ops
[1].value
;
7112 /* Otherwise fall back to the original __sync_val_compare_and_swap
7113 which is always seq-cst. */
7114 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
7115 if (icode
!= CODE_FOR_nothing
)
7119 create_output_operand (&ops
[0], target_oval
, mode
);
7120 create_fixed_operand (&ops
[1], mem
);
7121 create_input_operand (&ops
[2], expected
, mode
);
7122 create_input_operand (&ops
[3], desired
, mode
);
7123 if (!maybe_expand_insn (icode
, 4, ops
))
7126 target_oval
= ops
[0].value
;
7128 /* If the caller isn't interested in the boolean return value,
7129 skip the computation of it. */
7130 if (ptarget_bool
== NULL
)
7133 /* Otherwise, work out if the compare-and-swap succeeded. */
7135 if (have_insn_for (COMPARE
, CCmode
))
7136 note_stores (get_last_insn (), find_cc_set
, &cc_reg
);
7139 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
7140 const0_rtx
, VOIDmode
, 0, 1);
7143 goto success_bool_from_val
;
7146 /* Also check for library support for __sync_val_compare_and_swap. */
7147 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
7148 if (libfunc
!= NULL
)
7150 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
7151 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
7152 mode
, addr
, ptr_mode
,
7153 expected
, mode
, desired
, mode
);
7154 emit_move_insn (target_oval
, target
);
7156 /* Compute the boolean return value only if requested. */
7158 goto success_bool_from_val
;
7166 success_bool_from_val
:
7167 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
7168 expected
, VOIDmode
, 1, 1);
7170 /* Make sure that the oval output winds up where the caller asked. */
7172 *ptarget_oval
= target_oval
;
7174 *ptarget_bool
= target_bool
;
7178 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
7181 expand_asm_memory_blockage (void)
7185 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
7186 rtvec_alloc (0), rtvec_alloc (0),
7187 rtvec_alloc (0), UNKNOWN_LOCATION
);
7188 MEM_VOLATILE_P (asm_op
) = 1;
7190 clob
= gen_rtx_SCRATCH (VOIDmode
);
7191 clob
= gen_rtx_MEM (BLKmode
, clob
);
7192 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
7194 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
7197 /* Do not propagate memory accesses across this point. */
7200 expand_memory_blockage (void)
7202 if (targetm
.have_memory_blockage ())
7203 emit_insn (targetm
.gen_memory_blockage ());
7205 expand_asm_memory_blockage ();
7208 /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7209 same time clobbering the register set specified by REGS. */
7212 expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs
)
7214 rtx asm_op
, clob_mem
;
7216 unsigned int num_of_regs
= 0;
7217 for (unsigned int i
= 0; i
< FIRST_PSEUDO_REGISTER
; i
++)
7218 if (TEST_HARD_REG_BIT (regs
, i
))
7221 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
7222 rtvec_alloc (0), rtvec_alloc (0),
7223 rtvec_alloc (0), UNKNOWN_LOCATION
);
7224 MEM_VOLATILE_P (asm_op
) = 1;
7226 rtvec v
= rtvec_alloc (num_of_regs
+ 2);
7228 clob_mem
= gen_rtx_SCRATCH (VOIDmode
);
7229 clob_mem
= gen_rtx_MEM (BLKmode
, clob_mem
);
7230 clob_mem
= gen_rtx_CLOBBER (VOIDmode
, clob_mem
);
7232 RTVEC_ELT (v
, 0) = asm_op
;
7233 RTVEC_ELT (v
, 1) = clob_mem
;
7235 if (num_of_regs
> 0)
7238 for (unsigned int i
= 0; i
< FIRST_PSEUDO_REGISTER
; i
++)
7239 if (TEST_HARD_REG_BIT (regs
, i
))
7241 RTVEC_ELT (v
, j
) = gen_rtx_CLOBBER (VOIDmode
, regno_reg_rtx
[i
]);
7244 gcc_assert (j
== (num_of_regs
+ 2));
7247 emit_insn (gen_rtx_PARALLEL (VOIDmode
, v
));
7250 /* This routine will either emit the mem_thread_fence pattern or issue a
7251 sync_synchronize to generate a fence for memory model MEMMODEL. */
7254 expand_mem_thread_fence (enum memmodel model
)
7256 if (is_mm_relaxed (model
))
7258 if (targetm
.have_mem_thread_fence ())
7260 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
7261 expand_memory_blockage ();
7263 else if (targetm
.have_memory_barrier ())
7264 emit_insn (targetm
.gen_memory_barrier ());
7265 else if (synchronize_libfunc
!= NULL_RTX
)
7266 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
);
7268 expand_memory_blockage ();
7271 /* Emit a signal fence with given memory model. */
7274 expand_mem_signal_fence (enum memmodel model
)
7276 /* No machine barrier is required to implement a signal fence, but
7277 a compiler memory barrier must be issued, except for relaxed MM. */
7278 if (!is_mm_relaxed (model
))
7279 expand_memory_blockage ();
7282 /* This function expands the atomic load operation:
7283 return the atomically loaded value in MEM.
7285 MEMMODEL is the memory model variant to use.
7286 TARGET is an option place to stick the return value. */
7289 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
7291 machine_mode mode
= GET_MODE (mem
);
7292 enum insn_code icode
;
7294 /* If the target supports the load directly, great. */
7295 icode
= direct_optab_handler (atomic_load_optab
, mode
);
7296 if (icode
!= CODE_FOR_nothing
)
7298 class expand_operand ops
[3];
7299 rtx_insn
*last
= get_last_insn ();
7300 if (is_mm_seq_cst (model
))
7301 expand_memory_blockage ();
7303 create_output_operand (&ops
[0], target
, mode
);
7304 create_fixed_operand (&ops
[1], mem
);
7305 create_integer_operand (&ops
[2], model
);
7306 if (maybe_expand_insn (icode
, 3, ops
))
7308 if (!is_mm_relaxed (model
))
7309 expand_memory_blockage ();
7310 return ops
[0].value
;
7312 delete_insns_since (last
);
7315 /* If the size of the object is greater than word size on this target,
7316 then we assume that a load will not be atomic. We could try to
7317 emulate a load with a compare-and-swap operation, but the store that
7318 doing this could result in would be incorrect if this is a volatile
7319 atomic load or targetting read-only-mapped memory. */
7320 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
7321 /* If there is no atomic load, leave the library call. */
7324 /* Otherwise assume loads are atomic, and emit the proper barriers. */
7325 if (!target
|| target
== const0_rtx
)
7326 target
= gen_reg_rtx (mode
);
7328 /* For SEQ_CST, emit a barrier before the load. */
7329 if (is_mm_seq_cst (model
))
7330 expand_mem_thread_fence (model
);
7332 emit_move_insn (target
, mem
);
7334 /* Emit the appropriate barrier after the load. */
7335 expand_mem_thread_fence (model
);
7340 /* This function expands the atomic store operation:
7341 Atomically store VAL in MEM.
7342 MEMMODEL is the memory model variant to use.
7343 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7344 function returns const0_rtx if a pattern was emitted. */
7347 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
7349 machine_mode mode
= GET_MODE (mem
);
7350 enum insn_code icode
;
7351 class expand_operand ops
[3];
7353 /* If the target supports the store directly, great. */
7354 icode
= direct_optab_handler (atomic_store_optab
, mode
);
7355 if (icode
!= CODE_FOR_nothing
)
7357 rtx_insn
*last
= get_last_insn ();
7358 if (!is_mm_relaxed (model
))
7359 expand_memory_blockage ();
7360 create_fixed_operand (&ops
[0], mem
);
7361 create_input_operand (&ops
[1], val
, mode
);
7362 create_integer_operand (&ops
[2], model
);
7363 if (maybe_expand_insn (icode
, 3, ops
))
7365 if (is_mm_seq_cst (model
))
7366 expand_memory_blockage ();
7369 delete_insns_since (last
);
7372 /* If using __sync_lock_release is a viable alternative, try it.
7373 Note that this will not be set to true if we are expanding a generic
7374 __atomic_store_n. */
7377 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
7378 if (icode
!= CODE_FOR_nothing
)
7380 create_fixed_operand (&ops
[0], mem
);
7381 create_input_operand (&ops
[1], const0_rtx
, mode
);
7382 if (maybe_expand_insn (icode
, 2, ops
))
7384 /* lock_release is only a release barrier. */
7385 if (is_mm_seq_cst (model
))
7386 expand_mem_thread_fence (model
);
7392 /* If the size of the object is greater than word size on this target,
7393 a default store will not be atomic. */
7394 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
7396 /* If loads are atomic or we are called to provide a __sync builtin,
7397 we can try a atomic_exchange and throw away the result. Otherwise,
7398 don't do anything so that we do not create an inconsistency between
7399 loads and stores. */
7400 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
7402 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
7404 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
7412 /* Otherwise assume stores are atomic, and emit the proper barriers. */
7413 expand_mem_thread_fence (model
);
7415 emit_move_insn (mem
, val
);
7417 /* For SEQ_CST, also emit a barrier after the store. */
7418 if (is_mm_seq_cst (model
))
7419 expand_mem_thread_fence (model
);
7425 /* Structure containing the pointers and values required to process the
7426 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7428 struct atomic_op_functions
7430 direct_optab mem_fetch_before
;
7431 direct_optab mem_fetch_after
;
7432 direct_optab mem_no_result
;
7435 direct_optab no_result
;
7436 enum rtx_code reverse_code
;
7440 /* Fill in structure pointed to by OP with the various optab entries for an
7441 operation of type CODE. */
7444 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
7446 gcc_assert (op
!= NULL
);
7448 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7449 in the source code during compilation, and the optab entries are not
7450 computable until runtime. Fill in the values at runtime. */
7454 op
->mem_fetch_before
= atomic_fetch_add_optab
;
7455 op
->mem_fetch_after
= atomic_add_fetch_optab
;
7456 op
->mem_no_result
= atomic_add_optab
;
7457 op
->fetch_before
= sync_old_add_optab
;
7458 op
->fetch_after
= sync_new_add_optab
;
7459 op
->no_result
= sync_add_optab
;
7460 op
->reverse_code
= MINUS
;
7463 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
7464 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
7465 op
->mem_no_result
= atomic_sub_optab
;
7466 op
->fetch_before
= sync_old_sub_optab
;
7467 op
->fetch_after
= sync_new_sub_optab
;
7468 op
->no_result
= sync_sub_optab
;
7469 op
->reverse_code
= PLUS
;
7472 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
7473 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
7474 op
->mem_no_result
= atomic_xor_optab
;
7475 op
->fetch_before
= sync_old_xor_optab
;
7476 op
->fetch_after
= sync_new_xor_optab
;
7477 op
->no_result
= sync_xor_optab
;
7478 op
->reverse_code
= XOR
;
7481 op
->mem_fetch_before
= atomic_fetch_and_optab
;
7482 op
->mem_fetch_after
= atomic_and_fetch_optab
;
7483 op
->mem_no_result
= atomic_and_optab
;
7484 op
->fetch_before
= sync_old_and_optab
;
7485 op
->fetch_after
= sync_new_and_optab
;
7486 op
->no_result
= sync_and_optab
;
7487 op
->reverse_code
= UNKNOWN
;
7490 op
->mem_fetch_before
= atomic_fetch_or_optab
;
7491 op
->mem_fetch_after
= atomic_or_fetch_optab
;
7492 op
->mem_no_result
= atomic_or_optab
;
7493 op
->fetch_before
= sync_old_ior_optab
;
7494 op
->fetch_after
= sync_new_ior_optab
;
7495 op
->no_result
= sync_ior_optab
;
7496 op
->reverse_code
= UNKNOWN
;
7499 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
7500 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
7501 op
->mem_no_result
= atomic_nand_optab
;
7502 op
->fetch_before
= sync_old_nand_optab
;
7503 op
->fetch_after
= sync_new_nand_optab
;
7504 op
->no_result
= sync_nand_optab
;
7505 op
->reverse_code
= UNKNOWN
;
7512 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7513 using memory order MODEL. If AFTER is true the operation needs to return
7514 the value of *MEM after the operation, otherwise the previous value.
7515 TARGET is an optional place to place the result. The result is unused if
7517 Return the result if there is a better sequence, otherwise NULL_RTX. */
7520 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7521 enum memmodel model
, bool after
)
7523 /* If the value is prefetched, or not used, it may be possible to replace
7524 the sequence with a native exchange operation. */
7525 if (!after
|| target
== const0_rtx
)
7527 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7528 if (code
== AND
&& val
== const0_rtx
)
7530 if (target
== const0_rtx
)
7531 target
= gen_reg_rtx (GET_MODE (mem
));
7532 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7535 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7536 if (code
== IOR
&& val
== constm1_rtx
)
7538 if (target
== const0_rtx
)
7539 target
= gen_reg_rtx (GET_MODE (mem
));
7540 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7547 /* Try to emit an instruction for a specific operation varaition.
7548 OPTAB contains the OP functions.
7549 TARGET is an optional place to return the result. const0_rtx means unused.
7550 MEM is the memory location to operate on.
7551 VAL is the value to use in the operation.
7552 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7553 MODEL is the memory model, if used.
7554 AFTER is true if the returned result is the value after the operation. */
7557 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
7558 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
7560 machine_mode mode
= GET_MODE (mem
);
7561 class expand_operand ops
[4];
7562 enum insn_code icode
;
7566 /* Check to see if there is a result returned. */
7567 if (target
== const0_rtx
)
7571 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
7572 create_integer_operand (&ops
[2], model
);
7577 icode
= direct_optab_handler (optab
->no_result
, mode
);
7581 /* Otherwise, we need to generate a result. */
7586 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
7587 : optab
->mem_fetch_before
, mode
);
7588 create_integer_operand (&ops
[3], model
);
7593 icode
= optab_handler (after
? optab
->fetch_after
7594 : optab
->fetch_before
, mode
);
7597 create_output_operand (&ops
[op_counter
++], target
, mode
);
7599 if (icode
== CODE_FOR_nothing
)
7602 create_fixed_operand (&ops
[op_counter
++], mem
);
7603 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7604 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
7606 if (maybe_expand_insn (icode
, num_ops
, ops
))
7607 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
7613 /* This function expands an atomic fetch_OP or OP_fetch operation:
7614 TARGET is an option place to stick the return value. const0_rtx indicates
7615 the result is unused.
7616 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7617 CODE is the operation being performed (OP)
7618 MEMMODEL is the memory model variant to use.
7619 AFTER is true to return the result of the operation (OP_fetch).
7620 AFTER is false to return the value before the operation (fetch_OP).
7622 This function will *only* generate instructions if there is a direct
7623 optab. No compare and swap loops or libcalls will be generated. */
7626 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
7627 enum rtx_code code
, enum memmodel model
,
7630 machine_mode mode
= GET_MODE (mem
);
7631 struct atomic_op_functions optab
;
7633 bool unused_result
= (target
== const0_rtx
);
7635 get_atomic_op_for_code (&optab
, code
);
7637 /* Check to see if there are any better instructions. */
7638 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
7642 /* Check for the case where the result isn't used and try those patterns. */
7645 /* Try the memory model variant first. */
7646 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
7650 /* Next try the old style withuot a memory model. */
7651 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
7655 /* There is no no-result pattern, so try patterns with a result. */
7659 /* Try the __atomic version. */
7660 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
7664 /* Try the older __sync version. */
7665 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
7669 /* If the fetch value can be calculated from the other variation of fetch,
7670 try that operation. */
7671 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
7673 /* Try the __atomic version, then the older __sync version. */
7674 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
7676 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
7680 /* If the result isn't used, no need to do compensation code. */
7684 /* Issue compensation code. Fetch_after == fetch_before OP val.
7685 Fetch_before == after REVERSE_OP val. */
7687 code
= optab
.reverse_code
;
7690 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
7691 true, OPTAB_LIB_WIDEN
);
7692 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
7695 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
7696 true, OPTAB_LIB_WIDEN
);
7701 /* No direct opcode can be generated. */
7707 /* This function expands an atomic fetch_OP or OP_fetch operation:
7708 TARGET is an option place to stick the return value. const0_rtx indicates
7709 the result is unused.
7710 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7711 CODE is the operation being performed (OP)
7712 MEMMODEL is the memory model variant to use.
7713 AFTER is true to return the result of the operation (OP_fetch).
7714 AFTER is false to return the value before the operation (fetch_OP). */
7716 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7717 enum memmodel model
, bool after
)
7719 machine_mode mode
= GET_MODE (mem
);
7721 bool unused_result
= (target
== const0_rtx
);
7723 /* If loads are not atomic for the required size and we are not called to
7724 provide a __sync builtin, do not do anything so that we stay consistent
7725 with atomic loads of the same size. */
7726 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
7729 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
7735 /* Add/sub can be implemented by doing the reverse operation with -(val). */
7736 if (code
== PLUS
|| code
== MINUS
)
7739 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
7742 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
7743 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
7747 /* PLUS worked so emit the insns and return. */
7754 /* PLUS did not work, so throw away the negation code and continue. */
7758 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7759 if (!can_compare_and_swap_p (mode
, false))
7763 enum rtx_code orig_code
= code
;
7764 struct atomic_op_functions optab
;
7766 get_atomic_op_for_code (&optab
, code
);
7767 libfunc
= optab_libfunc (after
? optab
.fetch_after
7768 : optab
.fetch_before
, mode
);
7770 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
7774 code
= optab
.reverse_code
;
7775 libfunc
= optab_libfunc (after
? optab
.fetch_before
7776 : optab
.fetch_after
, mode
);
7778 if (libfunc
!= NULL
)
7780 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
7781 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
7782 addr
, ptr_mode
, val
, mode
);
7784 if (!unused_result
&& fixup
)
7785 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
7786 true, OPTAB_LIB_WIDEN
);
7790 /* We need the original code for any further attempts. */
7794 /* If nothing else has succeeded, default to a compare and swap loop. */
7795 if (can_compare_and_swap_p (mode
, true))
7798 rtx t0
= gen_reg_rtx (mode
), t1
;
7802 /* If the result is used, get a register for it. */
7805 if (!target
|| !register_operand (target
, mode
))
7806 target
= gen_reg_rtx (mode
);
7807 /* If fetch_before, copy the value now. */
7809 emit_move_insn (target
, t0
);
7812 target
= const0_rtx
;
7817 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
7818 true, OPTAB_LIB_WIDEN
);
7819 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
7822 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
7825 /* For after, copy the value now. */
7826 if (!unused_result
&& after
)
7827 emit_move_insn (target
, t1
);
7828 insn
= get_insns ();
7831 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
7838 /* Return true if OPERAND is suitable for operand number OPNO of
7839 instruction ICODE. */
7842 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
7844 return (!insn_data
[(int) icode
].operand
[opno
].predicate
7845 || (insn_data
[(int) icode
].operand
[opno
].predicate
7846 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
7849 /* TARGET is a target of a multiword operation that we are going to
7850 implement as a series of word-mode operations. Return true if
7851 TARGET is suitable for this purpose. */
7854 valid_multiword_target_p (rtx target
)
7859 mode
= GET_MODE (target
);
7860 if (!GET_MODE_SIZE (mode
).is_constant (&size
))
7862 for (i
= 0; i
< size
; i
+= UNITS_PER_WORD
)
7863 if (!validate_subreg (word_mode
, mode
, target
, i
))
7868 /* Make OP describe an input operand that has value INTVAL and that has
7869 no inherent mode. This function should only be used for operands that
7870 are always expand-time constants. The backend may request that INTVAL
7871 be copied into a different kind of rtx, but it must specify the mode
7872 of that rtx if so. */
7875 create_integer_operand (class expand_operand
*op
, poly_int64 intval
)
7877 create_expand_operand (op
, EXPAND_INTEGER
,
7878 gen_int_mode (intval
, MAX_MODE_INT
),
7879 VOIDmode
, false, intval
);
7882 /* Like maybe_legitimize_operand, but do not change the code of the
7883 current rtx value. */
7886 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
7887 class expand_operand
*op
)
7889 /* See if the operand matches in its current form. */
7890 if (insn_operand_matches (icode
, opno
, op
->value
))
7893 /* If the operand is a memory whose address has no side effects,
7894 try forcing the address into a non-virtual pseudo register.
7895 The check for side effects is important because copy_to_mode_reg
7896 cannot handle things like auto-modified addresses. */
7897 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
7902 addr
= XEXP (mem
, 0);
7903 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
7904 && !side_effects_p (addr
))
7909 last
= get_last_insn ();
7910 mode
= get_address_mode (mem
);
7911 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
7912 if (insn_operand_matches (icode
, opno
, mem
))
7917 delete_insns_since (last
);
7924 /* Try to make OP match operand OPNO of instruction ICODE. Return true
7925 on success, storing the new operand value back in OP. */
7928 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
7929 class expand_operand
*op
)
7931 machine_mode mode
, imode
, tmode
;
7938 temporary_volatile_ok
v (true);
7939 return maybe_legitimize_operand_same_code (icode
, opno
, op
);
7943 gcc_assert (mode
!= VOIDmode
);
7945 && op
->value
!= const0_rtx
7946 && GET_MODE (op
->value
) == mode
7947 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
7950 op
->value
= gen_reg_rtx (mode
);
7956 gcc_assert (mode
!= VOIDmode
);
7957 gcc_assert (GET_MODE (op
->value
) == VOIDmode
7958 || GET_MODE (op
->value
) == mode
);
7959 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
7962 op
->value
= copy_to_mode_reg (mode
, op
->value
);
7965 case EXPAND_CONVERT_TO
:
7966 gcc_assert (mode
!= VOIDmode
);
7967 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
7970 case EXPAND_CONVERT_FROM
:
7971 if (GET_MODE (op
->value
) != VOIDmode
)
7972 mode
= GET_MODE (op
->value
);
7974 /* The caller must tell us what mode this value has. */
7975 gcc_assert (mode
!= VOIDmode
);
7977 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7978 tmode
= (VECTOR_MODE_P (imode
) && !VECTOR_MODE_P (mode
)
7979 ? GET_MODE_INNER (imode
) : imode
);
7980 if (tmode
!= VOIDmode
&& tmode
!= mode
)
7982 op
->value
= convert_modes (tmode
, mode
, op
->value
, op
->unsigned_p
);
7985 if (imode
!= VOIDmode
&& imode
!= mode
)
7987 gcc_assert (VECTOR_MODE_P (imode
) && !VECTOR_MODE_P (mode
));
7988 op
->value
= expand_vector_broadcast (imode
, op
->value
);
7993 case EXPAND_ADDRESS
:
7994 op
->value
= convert_memory_address (as_a
<scalar_int_mode
> (mode
),
7998 case EXPAND_INTEGER
:
7999 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
8000 if (mode
!= VOIDmode
8001 && known_eq (trunc_int_for_mode (op
->int_value
, mode
),
8004 op
->value
= gen_int_mode (op
->int_value
, mode
);
8009 return insn_operand_matches (icode
, opno
, op
->value
);
8012 /* Make OP describe an input operand that should have the same value
8013 as VALUE, after any mode conversion that the target might request.
8014 TYPE is the type of VALUE. */
8017 create_convert_operand_from_type (class expand_operand
*op
,
8018 rtx value
, tree type
)
8020 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
8021 TYPE_UNSIGNED (type
));
8024 /* Return true if the requirements on operands OP1 and OP2 of instruction
8025 ICODE are similar enough for the result of legitimizing OP1 to be
8026 reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
8027 with OP1 and OP2 respectively. */
8030 can_reuse_operands_p (enum insn_code icode
,
8031 unsigned int opno1
, unsigned int opno2
,
8032 const class expand_operand
*op1
,
8033 const class expand_operand
*op2
)
8035 /* Check requirements that are common to all types. */
8036 if (op1
->type
!= op2
->type
8037 || op1
->mode
!= op2
->mode
8038 || (insn_data
[(int) icode
].operand
[opno1
].mode
8039 != insn_data
[(int) icode
].operand
[opno2
].mode
))
8042 /* Check the requirements for specific types. */
8046 /* Outputs must remain distinct. */
8051 case EXPAND_ADDRESS
:
8052 case EXPAND_INTEGER
:
8055 case EXPAND_CONVERT_TO
:
8056 case EXPAND_CONVERT_FROM
:
8057 return op1
->unsigned_p
== op2
->unsigned_p
;
8062 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8063 of instruction ICODE. Return true on success, leaving the new operand
8064 values in the OPS themselves. Emit no code on failure. */
8067 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
8068 unsigned int nops
, class expand_operand
*ops
)
8070 rtx_insn
*last
= get_last_insn ();
8071 rtx
*orig_values
= XALLOCAVEC (rtx
, nops
);
8072 for (unsigned int i
= 0; i
< nops
; i
++)
8074 orig_values
[i
] = ops
[i
].value
;
8076 /* First try reusing the result of an earlier legitimization.
8077 This avoids duplicate rtl and ensures that tied operands
8080 This search is linear, but NOPS is bounded at compile time
8081 to a small number (current a single digit). */
8084 if (can_reuse_operands_p (icode
, opno
+ j
, opno
+ i
, &ops
[j
], &ops
[i
])
8085 && rtx_equal_p (orig_values
[j
], orig_values
[i
])
8087 && insn_operand_matches (icode
, opno
+ i
, ops
[j
].value
))
8089 ops
[i
].value
= copy_rtx (ops
[j
].value
);
8093 /* Otherwise try legitimizing the operand on its own. */
8094 if (j
== i
&& !maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
8096 delete_insns_since (last
);
8103 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8104 as its operands. Return the instruction pattern on success,
8105 and emit any necessary set-up code. Return null and emit no
8109 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
8110 class expand_operand
*ops
)
8112 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
8113 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
8119 return GEN_FCN (icode
) ();
8121 return GEN_FCN (icode
) (ops
[0].value
);
8123 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
8125 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
8127 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8130 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8131 ops
[3].value
, ops
[4].value
);
8133 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8134 ops
[3].value
, ops
[4].value
, ops
[5].value
);
8136 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8137 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8140 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8141 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8142 ops
[6].value
, ops
[7].value
);
8144 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8145 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8146 ops
[6].value
, ops
[7].value
, ops
[8].value
);
8148 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8149 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8150 ops
[6].value
, ops
[7].value
, ops
[8].value
,
8153 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8154 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8155 ops
[6].value
, ops
[7].value
, ops
[8].value
,
8156 ops
[9].value
, ops
[10].value
);
8161 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8162 as its operands. Return true on success and emit no code on failure. */
8165 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
8166 class expand_operand
*ops
)
8168 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
8177 /* Like maybe_expand_insn, but for jumps. */
8180 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
8181 class expand_operand
*ops
)
8183 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
8186 emit_jump_insn (pat
);
8192 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8196 expand_insn (enum insn_code icode
, unsigned int nops
,
8197 class expand_operand
*ops
)
8199 if (!maybe_expand_insn (icode
, nops
, ops
))
8203 /* Like expand_insn, but for jumps. */
8206 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
8207 class expand_operand
*ops
)
8209 if (!maybe_expand_jump_insn (icode
, nops
, ops
))