1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2022 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
45 #include "optabs-tree.h"
47 #include "internal-fn.h"
48 #include "langhooks.h"
50 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
52 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
53 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
55 static rtx
emit_conditional_move_1 (rtx
, rtx
, rtx
, rtx
, machine_mode
);
57 /* Debug facility for use in GDB. */
58 void debug_optab_libfuncs (void);
60 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
61 the result of operation CODE applied to OP0 (and OP1 if it is a binary
62 operation). OP0_MODE is OP0's mode.
64 If the last insn does not set TARGET, don't do anything, but return 1.
66 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
67 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
68 try again, ensuring that TARGET is not one of the operands. */
71 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
,
72 rtx op1
, machine_mode op0_mode
)
78 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
80 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
81 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
82 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
83 && GET_RTX_CLASS (code
) != RTX_COMPARE
84 && GET_RTX_CLASS (code
) != RTX_UNARY
)
87 if (GET_CODE (target
) == ZERO_EXTRACT
)
90 for (last_insn
= insns
;
91 NEXT_INSN (last_insn
) != NULL_RTX
;
92 last_insn
= NEXT_INSN (last_insn
))
95 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
96 a value changing in the insn, so the note would be invalid for CSE. */
97 if (reg_overlap_mentioned_p (target
, op0
)
98 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
101 && (rtx_equal_p (target
, op0
)
102 || (op1
&& rtx_equal_p (target
, op1
))))
104 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
105 over expanding it as temp = MEM op X, MEM = temp. If the target
106 supports MEM = MEM op X instructions, it is sometimes too hard
107 to reconstruct that form later, especially if X is also a memory,
108 and due to multiple occurrences of addresses the address might
109 be forced into register unnecessarily.
110 Note that not emitting the REG_EQUIV note might inhibit
111 CSE in some cases. */
112 set
= single_set (last_insn
);
114 && GET_CODE (SET_SRC (set
)) == code
115 && MEM_P (SET_DEST (set
))
116 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
117 || (op1
&& rtx_equal_p (SET_DEST (set
),
118 XEXP (SET_SRC (set
), 1)))))
124 set
= set_for_reg_notes (last_insn
);
128 if (! rtx_equal_p (SET_DEST (set
), target
)
129 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
130 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
131 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
134 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
144 if (op0_mode
!= VOIDmode
&& GET_MODE (target
) != op0_mode
)
146 note
= gen_rtx_fmt_e (code
, op0_mode
, copy_rtx (op0
));
147 if (GET_MODE_UNIT_SIZE (op0_mode
)
148 > GET_MODE_UNIT_SIZE (GET_MODE (target
)))
149 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
152 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
158 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
162 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
164 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
169 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
170 for a widening operation would be. In most cases this would be OP0, but if
171 that's a constant it'll be VOIDmode, which isn't useful. */
174 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
176 machine_mode m0
= GET_MODE (op0
);
177 machine_mode m1
= GET_MODE (op1
);
180 if (m0
== VOIDmode
&& m1
== VOIDmode
)
182 else if (m0
== VOIDmode
|| GET_MODE_UNIT_SIZE (m0
) < GET_MODE_UNIT_SIZE (m1
))
187 if (GET_MODE_UNIT_SIZE (result
) > GET_MODE_UNIT_SIZE (to_mode
))
193 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
194 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
195 not actually do a sign-extend or zero-extend, but can leave the
196 higher-order bits of the result rtx undefined, for example, in the case
197 of logical operations, but not right shifts. */
200 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
201 int unsignedp
, int no_extend
)
204 scalar_int_mode int_mode
;
206 /* If we don't have to extend and this is a constant, return it. */
207 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
210 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
211 extend since it will be more efficient to do so unless the signedness of
212 a promoted object differs from our extension. */
214 || !is_a
<scalar_int_mode
> (mode
, &int_mode
)
215 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
216 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
217 return convert_modes (mode
, oldmode
, op
, unsignedp
);
219 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
221 if (GET_MODE_SIZE (int_mode
) <= UNITS_PER_WORD
)
222 return gen_lowpart (int_mode
, force_reg (GET_MODE (op
), op
));
224 /* Otherwise, get an object of MODE, clobber it, and set the low-order
227 result
= gen_reg_rtx (int_mode
);
228 emit_clobber (result
);
229 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
233 /* Expand vector widening operations.
235 There are two different classes of operations handled here:
236 1) Operations whose result is wider than all the arguments to the operation.
237 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
238 In this case OP0 and optionally OP1 would be initialized,
239 but WIDE_OP wouldn't (not relevant for this case).
240 2) Operations whose result is of the same size as the last argument to the
241 operation, but wider than all the other arguments to the operation.
242 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
243 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
245 E.g, when called to expand the following operations, this is how
246 the arguments will be initialized:
248 widening-sum 2 oprnd0 - oprnd1
249 widening-dot-product 3 oprnd0 oprnd1 oprnd2
250 widening-mult 2 oprnd0 oprnd1 -
251 type-promotion (vec-unpack) 1 oprnd0 - - */
254 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
255 rtx target
, int unsignedp
)
257 class expand_operand eops
[4];
258 tree oprnd0
, oprnd1
, oprnd2
;
259 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
260 optab widen_pattern_optab
;
261 enum insn_code icode
;
262 int nops
= TREE_CODE_LENGTH (ops
->code
);
272 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
273 if (ops
->code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
274 || ops
->code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
)
275 /* The sign is from the result type rather than operand's type
278 = optab_for_tree_code (ops
->code
, ops
->type
, optab_default
);
279 else if ((ops
->code
== VEC_UNPACK_HI_EXPR
280 || ops
->code
== VEC_UNPACK_LO_EXPR
)
281 && VECTOR_BOOLEAN_TYPE_P (ops
->type
)
282 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0
))
283 && TYPE_MODE (ops
->type
) == TYPE_MODE (TREE_TYPE (oprnd0
))
284 && SCALAR_INT_MODE_P (TYPE_MODE (ops
->type
)))
286 /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
287 the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
288 vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
289 the pattern number of elements in the wider vector. */
291 = (ops
->code
== VEC_UNPACK_HI_EXPR
292 ? vec_unpacks_sbool_hi_optab
: vec_unpacks_sbool_lo_optab
);
295 else if (ops
->code
== DOT_PROD_EXPR
)
297 enum optab_subtype subtype
= optab_default
;
298 signop sign1
= TYPE_SIGN (TREE_TYPE (oprnd0
));
299 signop sign2
= TYPE_SIGN (TREE_TYPE (oprnd1
));
302 else if (sign1
== SIGNED
&& sign2
== UNSIGNED
)
304 subtype
= optab_vector_mixed_sign
;
305 /* Same as optab_vector_mixed_sign but flip the operands. */
306 std::swap (op0
, op1
);
308 else if (sign1
== UNSIGNED
&& sign2
== SIGNED
)
309 subtype
= optab_vector_mixed_sign
;
314 = optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), subtype
);
318 = optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
319 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
320 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
321 icode
= find_widening_optab_handler (widen_pattern_optab
,
322 TYPE_MODE (TREE_TYPE (ops
->op2
)),
325 icode
= optab_handler (widen_pattern_optab
, tmode0
);
326 gcc_assert (icode
!= CODE_FOR_nothing
);
329 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
333 op1
= GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0
)).to_constant ());
337 /* The last operand is of a wider mode than the rest of the operands. */
342 gcc_assert (tmode1
== tmode0
);
344 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
348 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
349 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
351 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
353 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
354 expand_insn (icode
, op
, eops
);
355 return eops
[0].value
;
358 /* Generate code to perform an operation specified by TERNARY_OPTAB
359 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
361 UNSIGNEDP is for the case where we have to widen the operands
362 to perform the operation. It says to use zero-extension.
364 If TARGET is nonzero, the value
365 is generated there, if it is convenient to do so.
366 In all cases an rtx is returned for the locus of the value;
367 this may or may not be TARGET. */
370 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
371 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
373 class expand_operand ops
[4];
374 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
376 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
378 create_output_operand (&ops
[0], target
, mode
);
379 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
380 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
381 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
382 expand_insn (icode
, 4, ops
);
387 /* Like expand_binop, but return a constant rtx if the result can be
388 calculated at compile time. The arguments and return value are
389 otherwise the same as for expand_binop. */
392 simplify_expand_binop (machine_mode mode
, optab binoptab
,
393 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
394 enum optab_methods methods
)
396 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
398 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
404 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
407 /* Like simplify_expand_binop, but always put the result in TARGET.
408 Return true if the expansion succeeded. */
411 force_expand_binop (machine_mode mode
, optab binoptab
,
412 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
413 enum optab_methods methods
)
415 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
416 target
, unsignedp
, methods
);
420 emit_move_insn (target
, x
);
424 /* Create a new vector value in VMODE with all elements set to OP. The
425 mode of OP must be the element mode of VMODE. If OP is a constant,
426 then the return value will be a constant. */
429 expand_vector_broadcast (machine_mode vmode
, rtx op
)
434 gcc_checking_assert (VECTOR_MODE_P (vmode
));
436 if (valid_for_const_vector_p (vmode
, op
))
437 return gen_const_vec_duplicate (vmode
, op
);
439 insn_code icode
= optab_handler (vec_duplicate_optab
, vmode
);
440 if (icode
!= CODE_FOR_nothing
)
442 class expand_operand ops
[2];
443 create_output_operand (&ops
[0], NULL_RTX
, vmode
);
444 create_input_operand (&ops
[1], op
, GET_MODE (op
));
445 expand_insn (icode
, 2, ops
);
449 if (!GET_MODE_NUNITS (vmode
).is_constant (&n
))
452 /* ??? If the target doesn't have a vec_init, then we have no easy way
453 of performing this operation. Most of this sort of generic support
454 is hidden away in the vector lowering support in gimple. */
455 icode
= convert_optab_handler (vec_init_optab
, vmode
,
456 GET_MODE_INNER (vmode
));
457 if (icode
== CODE_FOR_nothing
)
460 vec
= rtvec_alloc (n
);
461 for (int i
= 0; i
< n
; ++i
)
462 RTVEC_ELT (vec
, i
) = op
;
463 rtx ret
= gen_reg_rtx (vmode
);
464 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
469 /* This subroutine of expand_doubleword_shift handles the cases in which
470 the effective shift value is >= BITS_PER_WORD. The arguments and return
471 value are the same as for the parent routine, except that SUPERWORD_OP1
472 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
473 INTO_TARGET may be null if the caller has decided to calculate it. */
476 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
477 rtx outof_target
, rtx into_target
,
478 int unsignedp
, enum optab_methods methods
)
480 if (into_target
!= 0)
481 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
482 into_target
, unsignedp
, methods
))
485 if (outof_target
!= 0)
487 /* For a signed right shift, we must fill OUTOF_TARGET with copies
488 of the sign bit, otherwise we must fill it with zeros. */
489 if (binoptab
!= ashr_optab
)
490 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
492 if (!force_expand_binop (word_mode
, binoptab
, outof_input
,
493 gen_int_shift_amount (word_mode
,
495 outof_target
, unsignedp
, methods
))
501 /* This subroutine of expand_doubleword_shift handles the cases in which
502 the effective shift value is < BITS_PER_WORD. The arguments and return
503 value are the same as for the parent routine. */
506 expand_subword_shift (scalar_int_mode op1_mode
, optab binoptab
,
507 rtx outof_input
, rtx into_input
, rtx op1
,
508 rtx outof_target
, rtx into_target
,
509 int unsignedp
, enum optab_methods methods
,
510 unsigned HOST_WIDE_INT shift_mask
)
512 optab reverse_unsigned_shift
, unsigned_shift
;
515 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
516 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
518 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
519 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
520 the opposite direction to BINOPTAB. */
521 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
523 carries
= outof_input
;
524 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
525 op1_mode
), op1_mode
);
526 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
531 /* We must avoid shifting by BITS_PER_WORD bits since that is either
532 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
533 has unknown behavior. Do a single shift first, then shift by the
534 remainder. It's OK to use ~OP1 as the remainder if shift counts
535 are truncated to the mode size. */
536 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
537 outof_input
, const1_rtx
, 0, unsignedp
, methods
);
538 if (shift_mask
== BITS_PER_WORD
- 1)
540 tmp
= immed_wide_int_const
541 (wi::minus_one (GET_MODE_PRECISION (op1_mode
)), op1_mode
);
542 tmp
= simplify_expand_binop (op1_mode
, xor_optab
, op1
, tmp
,
547 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
548 op1_mode
), op1_mode
);
549 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
553 if (tmp
== 0 || carries
== 0)
555 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
556 carries
, tmp
, 0, unsignedp
, methods
);
560 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
561 so the result can go directly into INTO_TARGET if convenient. */
562 tmp
= expand_binop (word_mode
, unsigned_shift
, into_input
, op1
,
563 into_target
, unsignedp
, methods
);
567 /* Now OR in the bits carried over from OUTOF_INPUT. */
568 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
569 into_target
, unsignedp
, methods
))
572 /* Use a standard word_mode shift for the out-of half. */
573 if (outof_target
!= 0)
574 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
575 outof_target
, unsignedp
, methods
))
582 /* Try implementing expand_doubleword_shift using conditional moves.
583 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
584 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
585 are the shift counts to use in the former and latter case. All other
586 arguments are the same as the parent routine. */
589 expand_doubleword_shift_condmove (scalar_int_mode op1_mode
, optab binoptab
,
590 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
591 rtx outof_input
, rtx into_input
,
592 rtx subword_op1
, rtx superword_op1
,
593 rtx outof_target
, rtx into_target
,
594 int unsignedp
, enum optab_methods methods
,
595 unsigned HOST_WIDE_INT shift_mask
)
597 rtx outof_superword
, into_superword
;
599 /* Put the superword version of the output into OUTOF_SUPERWORD and
601 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
602 if (outof_target
!= 0 && subword_op1
== superword_op1
)
604 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
605 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
606 into_superword
= outof_target
;
607 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
608 outof_superword
, 0, unsignedp
, methods
))
613 into_superword
= gen_reg_rtx (word_mode
);
614 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
615 outof_superword
, into_superword
,
620 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
621 if (!expand_subword_shift (op1_mode
, binoptab
,
622 outof_input
, into_input
, subword_op1
,
623 outof_target
, into_target
,
624 unsignedp
, methods
, shift_mask
))
627 /* Select between them. Do the INTO half first because INTO_SUPERWORD
628 might be the current value of OUTOF_TARGET. */
629 if (!emit_conditional_move (into_target
, { cmp_code
, cmp1
, cmp2
, op1_mode
},
630 into_target
, into_superword
, word_mode
, false))
633 if (outof_target
!= 0)
634 if (!emit_conditional_move (outof_target
,
635 { cmp_code
, cmp1
, cmp2
, op1_mode
},
636 outof_target
, outof_superword
,
643 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
644 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
645 input operand; the shift moves bits in the direction OUTOF_INPUT->
646 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
647 of the target. OP1 is the shift count and OP1_MODE is its mode.
648 If OP1 is constant, it will have been truncated as appropriate
649 and is known to be nonzero.
651 If SHIFT_MASK is zero, the result of word shifts is undefined when the
652 shift count is outside the range [0, BITS_PER_WORD). This routine must
653 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
655 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
656 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
657 fill with zeros or sign bits as appropriate.
659 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
660 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
661 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
662 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
665 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
666 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
667 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
668 function wants to calculate it itself.
670 Return true if the shift could be successfully synthesized. */
673 expand_doubleword_shift (scalar_int_mode op1_mode
, optab binoptab
,
674 rtx outof_input
, rtx into_input
, rtx op1
,
675 rtx outof_target
, rtx into_target
,
676 int unsignedp
, enum optab_methods methods
,
677 unsigned HOST_WIDE_INT shift_mask
)
679 rtx superword_op1
, tmp
, cmp1
, cmp2
;
680 enum rtx_code cmp_code
;
682 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
683 fill the result with sign or zero bits as appropriate. If so, the value
684 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
685 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
686 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
688 This isn't worthwhile for constant shifts since the optimizers will
689 cope better with in-range shift counts. */
690 if (shift_mask
>= BITS_PER_WORD
692 && !CONSTANT_P (op1
))
694 if (!expand_doubleword_shift (op1_mode
, binoptab
,
695 outof_input
, into_input
, op1
,
697 unsignedp
, methods
, shift_mask
))
699 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
700 outof_target
, unsignedp
, methods
))
705 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
706 is true when the effective shift value is less than BITS_PER_WORD.
707 Set SUPERWORD_OP1 to the shift count that should be used to shift
708 OUTOF_INPUT into INTO_TARGET when the condition is false. */
709 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
710 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
712 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
713 is a subword shift count. */
714 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
716 cmp2
= CONST0_RTX (op1_mode
);
722 /* Set CMP1 to OP1 - BITS_PER_WORD. */
723 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
725 cmp2
= CONST0_RTX (op1_mode
);
727 superword_op1
= cmp1
;
732 /* If we can compute the condition at compile time, pick the
733 appropriate subroutine. */
734 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
735 if (tmp
!= 0 && CONST_INT_P (tmp
))
737 if (tmp
== const0_rtx
)
738 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
739 outof_target
, into_target
,
742 return expand_subword_shift (op1_mode
, binoptab
,
743 outof_input
, into_input
, op1
,
744 outof_target
, into_target
,
745 unsignedp
, methods
, shift_mask
);
748 /* Try using conditional moves to generate straight-line code. */
749 if (HAVE_conditional_move
)
751 rtx_insn
*start
= get_last_insn ();
752 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
753 cmp_code
, cmp1
, cmp2
,
754 outof_input
, into_input
,
756 outof_target
, into_target
,
757 unsignedp
, methods
, shift_mask
))
759 delete_insns_since (start
);
762 /* As a last resort, use branches to select the correct alternative. */
763 rtx_code_label
*subword_label
= gen_label_rtx ();
764 rtx_code_label
*done_label
= gen_label_rtx ();
767 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
769 profile_probability::uninitialized ());
772 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
773 outof_target
, into_target
,
777 emit_jump_insn (targetm
.gen_jump (done_label
));
779 emit_label (subword_label
);
781 if (!expand_subword_shift (op1_mode
, binoptab
,
782 outof_input
, into_input
, op1
,
783 outof_target
, into_target
,
784 unsignedp
, methods
, shift_mask
))
787 emit_label (done_label
);
791 /* Subroutine of expand_binop. Perform a double word multiplication of
792 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
793 as the target's word_mode. This function return NULL_RTX if anything
794 goes wrong, in which case it may have already emitted instructions
795 which need to be deleted.
797 If we want to multiply two two-word values and have normal and widening
798 multiplies of single-word values, we can do this with three smaller
801 The multiplication proceeds as follows:
802 _______________________
803 [__op0_high_|__op0_low__]
804 _______________________
805 * [__op1_high_|__op1_low__]
806 _______________________________________________
807 _______________________
808 (1) [__op0_low__*__op1_low__]
809 _______________________
810 (2a) [__op0_low__*__op1_high_]
811 _______________________
812 (2b) [__op0_high_*__op1_low__]
813 _______________________
814 (3) [__op0_high_*__op1_high_]
817 This gives a 4-word result. Since we are only interested in the
818 lower 2 words, partial result (3) and the upper words of (2a) and
819 (2b) don't need to be calculated. Hence (2a) and (2b) can be
820 calculated using non-widening multiplication.
822 (1), however, needs to be calculated with an unsigned widening
823 multiplication. If this operation is not directly supported we
824 try using a signed widening multiplication and adjust the result.
825 This adjustment works as follows:
827 If both operands are positive then no adjustment is needed.
829 If the operands have different signs, for example op0_low < 0 and
830 op1_low >= 0, the instruction treats the most significant bit of
831 op0_low as a sign bit instead of a bit with significance
832 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
833 with 2**BITS_PER_WORD - op0_low, and two's complements the
834 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
837 Similarly, if both operands are negative, we need to add
838 (op0_low + op1_low) * 2**BITS_PER_WORD.
840 We use a trick to adjust quickly. We logically shift op0_low right
841 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
842 op0_high (op1_high) before it is used to calculate 2b (2a). If no
843 logical shift exists, we do an arithmetic right shift and subtract
847 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
848 bool umulp
, enum optab_methods methods
)
850 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
851 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
852 rtx wordm1
= (umulp
? NULL_RTX
853 : gen_int_shift_amount (word_mode
, BITS_PER_WORD
- 1));
854 rtx product
, adjust
, product_high
, temp
;
856 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
857 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
858 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
859 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
861 /* If we're using an unsigned multiply to directly compute the product
862 of the low-order words of the operands and perform any required
863 adjustments of the operands, we begin by trying two more multiplications
864 and then computing the appropriate sum.
866 We have checked above that the required addition is provided.
867 Full-word addition will normally always succeed, especially if
868 it is provided at all, so we don't worry about its failure. The
869 multiplication may well fail, however, so we do handle that. */
873 /* ??? This could be done with emit_store_flag where available. */
874 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
875 NULL_RTX
, 1, methods
);
877 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
878 NULL_RTX
, 0, OPTAB_DIRECT
);
881 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
882 NULL_RTX
, 0, methods
);
885 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
886 NULL_RTX
, 0, OPTAB_DIRECT
);
893 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
894 NULL_RTX
, 0, OPTAB_DIRECT
);
898 /* OP0_HIGH should now be dead. */
902 /* ??? This could be done with emit_store_flag where available. */
903 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
904 NULL_RTX
, 1, methods
);
906 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
907 NULL_RTX
, 0, OPTAB_DIRECT
);
910 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
911 NULL_RTX
, 0, methods
);
914 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
915 NULL_RTX
, 0, OPTAB_DIRECT
);
922 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
923 NULL_RTX
, 0, OPTAB_DIRECT
);
927 /* OP1_HIGH should now be dead. */
929 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
930 NULL_RTX
, 0, OPTAB_DIRECT
);
932 if (target
&& !REG_P (target
))
935 /* *_widen_optab needs to determine operand mode, make sure at least
936 one operand has non-VOID mode. */
937 if (GET_MODE (op0_low
) == VOIDmode
&& GET_MODE (op1_low
) == VOIDmode
)
938 op0_low
= force_reg (word_mode
, op0_low
);
941 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
942 target
, 1, OPTAB_DIRECT
);
944 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
945 target
, 1, OPTAB_DIRECT
);
950 product_high
= operand_subword (product
, high
, 1, mode
);
951 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
952 NULL_RTX
, 0, OPTAB_DIRECT
);
953 emit_move_insn (product_high
, adjust
);
957 /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
958 constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
959 (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
960 computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
961 + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
962 depends on the bit value, if 2, then carry from the addition needs to be
963 added too, i.e. like:
964 sum += __builtin_add_overflow (low, high, &sum)
966 Optimize signed double-word OP0 % OP1 similarly, just apply some correction
967 factor to the sum before doing unsigned remainder, in the form of
968 sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
969 then perform unsigned
970 remainder = sum % OP1;
972 remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
975 expand_doubleword_mod (machine_mode mode
, rtx op0
, rtx op1
, bool unsignedp
)
977 if (INTVAL (op1
) <= 1 || (INTVAL (op1
) & 1) == 0)
980 rtx_insn
*last
= get_last_insn ();
981 for (int bit
= BITS_PER_WORD
; bit
>= BITS_PER_WORD
/ 2; bit
--)
983 wide_int w
= wi::shifted_mask (bit
, 1, false, 2 * BITS_PER_WORD
);
984 if (wi::ne_p (wi::umod_trunc (w
, INTVAL (op1
)), 1))
986 rtx sum
= NULL_RTX
, mask
= NULL_RTX
;
987 if (bit
== BITS_PER_WORD
)
989 /* For signed modulo we need to add correction to the sum
990 and that might again overflow. */
993 if (optab_handler (uaddv4_optab
, word_mode
) == CODE_FOR_nothing
)
995 tree wtype
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
996 if (wtype
== NULL_TREE
)
998 tree ctype
= build_complex_type (wtype
);
999 if (TYPE_MODE (ctype
) != GET_MODE_COMPLEX_MODE (word_mode
))
1001 machine_mode cmode
= TYPE_MODE (ctype
);
1002 rtx op00
= operand_subword_force (op0
, 0, mode
);
1003 rtx op01
= operand_subword_force (op0
, 1, mode
);
1004 rtx cres
= gen_rtx_CONCAT (cmode
, gen_reg_rtx (word_mode
),
1005 gen_reg_rtx (word_mode
));
1006 tree lhs
= make_tree (ctype
, cres
);
1007 tree arg0
= make_tree (wtype
, op00
);
1008 tree arg1
= make_tree (wtype
, op01
);
1009 expand_addsub_overflow (UNKNOWN_LOCATION
, PLUS_EXPR
, lhs
, arg0
,
1010 arg1
, true, true, true, false, NULL
);
1011 sum
= expand_simple_binop (word_mode
, PLUS
, XEXP (cres
, 0),
1012 XEXP (cres
, 1), NULL_RTX
, 1,
1014 if (sum
== NULL_RTX
)
1019 /* Code below uses GEN_INT, so we need the masks to be representable
1020 in HOST_WIDE_INTs. */
1021 if (bit
>= HOST_BITS_PER_WIDE_INT
)
1023 /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1024 overflow. Consider 64-bit -1ULL for word size 32, if we add
1025 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1026 if (bit
== BITS_PER_WORD
- 1)
1029 int count
= (2 * BITS_PER_WORD
+ bit
- 1) / bit
;
1030 rtx sum_corr
= NULL_RTX
;
1034 /* For signed modulo, compute it as unsigned modulo of
1035 sum with a correction added to it if OP0 is negative,
1036 such that the result can be computed as unsigned
1037 remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1038 w
= wi::min_value (2 * BITS_PER_WORD
, SIGNED
);
1039 wide_int wmod1
= wi::umod_trunc (w
, INTVAL (op1
));
1040 wide_int wmod2
= wi::smod_trunc (w
, INTVAL (op1
));
1041 /* wmod2 == -wmod1. */
1042 wmod2
= wmod2
+ (INTVAL (op1
) - 1);
1043 if (wi::ne_p (wmod1
, wmod2
))
1045 wide_int wcorr
= wmod2
- wmod1
;
1047 wcorr
= wcorr
+ INTVAL (op1
);
1048 /* Now verify if the count sums can't overflow, and punt
1050 w
= wi::mask (bit
, false, 2 * BITS_PER_WORD
);
1051 w
= w
* (count
- 1);
1052 w
= w
+ wi::mask (2 * BITS_PER_WORD
- (count
- 1) * bit
,
1053 false, 2 * BITS_PER_WORD
);
1055 w
= wi::lrshift (w
, BITS_PER_WORD
);
1056 if (wi::ne_p (w
, 0))
1059 mask
= operand_subword_force (op0
, WORDS_BIG_ENDIAN
? 0 : 1,
1061 mask
= expand_simple_binop (word_mode
, ASHIFTRT
, mask
,
1062 GEN_INT (BITS_PER_WORD
- 1),
1063 NULL_RTX
, 0, OPTAB_DIRECT
);
1064 if (mask
== NULL_RTX
)
1066 sum_corr
= immed_wide_int_const (wcorr
, word_mode
);
1067 sum_corr
= expand_simple_binop (word_mode
, AND
, mask
,
1068 sum_corr
, NULL_RTX
, 1,
1070 if (sum_corr
== NULL_RTX
)
1075 for (int i
= 0; i
< count
; i
++)
1079 v
= expand_simple_binop (mode
, LSHIFTRT
, v
, GEN_INT (i
* bit
),
1080 NULL_RTX
, 1, OPTAB_DIRECT
);
1083 v
= lowpart_subreg (word_mode
, v
, mode
);
1087 v
= expand_simple_binop (word_mode
, AND
, v
,
1088 GEN_INT ((HOST_WIDE_INT_1U
<< bit
)
1093 if (sum
== NULL_RTX
)
1096 sum
= expand_simple_binop (word_mode
, PLUS
, sum
, v
, NULL_RTX
,
1098 if (sum
== NULL_RTX
)
1103 sum
= expand_simple_binop (word_mode
, PLUS
, sum
, sum_corr
,
1104 NULL_RTX
, 1, OPTAB_DIRECT
);
1105 if (sum
== NULL_RTX
)
1109 rtx remainder
= expand_divmod (1, TRUNC_MOD_EXPR
, word_mode
, sum
,
1110 gen_int_mode (INTVAL (op1
), word_mode
),
1111 NULL_RTX
, 1, OPTAB_DIRECT
);
1112 if (remainder
== NULL_RTX
)
1117 if (mask
== NULL_RTX
)
1119 mask
= operand_subword_force (op0
, WORDS_BIG_ENDIAN
? 0 : 1,
1121 mask
= expand_simple_binop (word_mode
, ASHIFTRT
, mask
,
1122 GEN_INT (BITS_PER_WORD
- 1),
1123 NULL_RTX
, 0, OPTAB_DIRECT
);
1124 if (mask
== NULL_RTX
)
1127 mask
= expand_simple_binop (word_mode
, AND
, mask
,
1128 gen_int_mode (1 - INTVAL (op1
),
1130 NULL_RTX
, 1, OPTAB_DIRECT
);
1131 if (mask
== NULL_RTX
)
1133 remainder
= expand_simple_binop (word_mode
, PLUS
, remainder
,
1134 mask
, NULL_RTX
, 1, OPTAB_DIRECT
);
1135 if (remainder
== NULL_RTX
)
1139 remainder
= convert_modes (mode
, word_mode
, remainder
, unsignedp
);
1140 /* Punt if we need any library calls. */
1142 last
= NEXT_INSN (last
);
1144 last
= get_insns ();
1145 for (; last
; last
= NEXT_INSN (last
))
1153 /* Similarly to the above function, but compute both quotient and remainder.
1154 Quotient can be computed from the remainder as:
1155 rem = op0 % op1; // Handled using expand_doubleword_mod
1156 quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1157 // 2 * BITS_PER_WORD
1159 We can also handle cases where op1 is a multiple of power of two constant
1160 and constant handled by expand_doubleword_mod.
1161 op11 = 1 << __builtin_ctz (op1);
1163 rem1 = op0 % op12; // Handled using expand_doubleword_mod
1164 quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1165 // 2 * BITS_PER_WORD
1166 rem = (quot1 % op11) * op12 + rem1;
1167 quot = quot1 / op11; */
1170 expand_doubleword_divmod (machine_mode mode
, rtx op0
, rtx op1
, rtx
*rem
,
1175 /* Negative dividend should have been optimized into positive,
1176 similarly modulo by 1 and modulo by power of two is optimized
1178 if (INTVAL (op1
) <= 1 || pow2p_hwi (INTVAL (op1
)))
1181 rtx op11
= const1_rtx
;
1183 if ((INTVAL (op1
) & 1) == 0)
1185 int bit
= ctz_hwi (INTVAL (op1
));
1186 op11
= GEN_INT (HOST_WIDE_INT_1
<< bit
);
1187 op12
= GEN_INT (INTVAL (op1
) >> bit
);
1190 rtx rem1
= expand_doubleword_mod (mode
, op0
, op12
, unsignedp
);
1191 if (rem1
== NULL_RTX
)
1194 int prec
= 2 * BITS_PER_WORD
;
1195 wide_int a
= wide_int::from (INTVAL (op12
), prec
+ 1, UNSIGNED
);
1196 wide_int b
= wi::shifted_mask (prec
, 1, false, prec
+ 1);
1197 wide_int m
= wide_int::from (wi::mod_inv (a
, b
), prec
, UNSIGNED
);
1198 rtx inv
= immed_wide_int_const (m
, mode
);
1200 rtx_insn
*last
= get_last_insn ();
1201 rtx quot1
= expand_simple_binop (mode
, MINUS
, op0
, rem1
,
1202 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1203 if (quot1
== NULL_RTX
)
1206 quot1
= expand_simple_binop (mode
, MULT
, quot1
, inv
,
1207 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1208 if (quot1
== NULL_RTX
)
1211 if (op11
!= const1_rtx
)
1213 rtx rem2
= expand_divmod (1, TRUNC_MOD_EXPR
, mode
, quot1
, op11
,
1214 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1215 if (rem2
== NULL_RTX
)
1218 rem2
= expand_simple_binop (mode
, MULT
, rem2
, op12
, NULL_RTX
,
1219 unsignedp
, OPTAB_DIRECT
);
1220 if (rem2
== NULL_RTX
)
1223 rem2
= expand_simple_binop (mode
, PLUS
, rem2
, rem1
, NULL_RTX
,
1224 unsignedp
, OPTAB_DIRECT
);
1225 if (rem2
== NULL_RTX
)
1228 rtx quot2
= expand_divmod (0, TRUNC_DIV_EXPR
, mode
, quot1
, op11
,
1229 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1230 if (quot2
== NULL_RTX
)
1237 /* Punt if we need any library calls. */
1239 last
= NEXT_INSN (last
);
1241 last
= get_insns ();
1242 for (; last
; last
= NEXT_INSN (last
))
1250 /* Wrapper around expand_binop which takes an rtx code to specify
1251 the operation to perform, not an optab pointer. All other
1252 arguments are the same. */
1254 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
1255 rtx op1
, rtx target
, int unsignedp
,
1256 enum optab_methods methods
)
1258 optab binop
= code_to_optab (code
);
1261 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
1264 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1265 binop. Order them according to commutative_operand_precedence and, if
1266 possible, try to put TARGET or a pseudo first. */
1268 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
1270 int op0_prec
= commutative_operand_precedence (op0
);
1271 int op1_prec
= commutative_operand_precedence (op1
);
1273 if (op0_prec
< op1_prec
)
1276 if (op0_prec
> op1_prec
)
1279 /* With equal precedence, both orders are ok, but it is better if the
1280 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1281 if (target
== 0 || REG_P (target
))
1282 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
1284 return rtx_equal_p (op1
, target
);
1287 /* Return true if BINOPTAB implements a shift operation. */
1290 shift_optab_p (optab binoptab
)
1292 switch (optab_to_code (binoptab
))
1308 /* Return true if BINOPTAB implements a commutative binary operation. */
1311 commutative_optab_p (optab binoptab
)
1313 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
1314 || binoptab
== smul_widen_optab
1315 || binoptab
== umul_widen_optab
1316 || binoptab
== smul_highpart_optab
1317 || binoptab
== umul_highpart_optab
);
1320 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1321 optimizing, and if the operand is a constant that costs more than
1322 1 instruction, force the constant into a register and return that
1323 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1326 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
1327 int opn
, rtx x
, bool unsignedp
)
1329 bool speed
= optimize_insn_for_speed_p ();
1331 if (mode
!= VOIDmode
1334 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
1335 > set_src_cost (x
, mode
, speed
)))
1337 if (CONST_INT_P (x
))
1339 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
1340 if (intval
!= INTVAL (x
))
1341 x
= GEN_INT (intval
);
1344 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
1345 x
= force_reg (mode
, x
);
1350 /* Helper function for expand_binop: handle the case where there
1351 is an insn ICODE that directly implements the indicated operation.
1352 Returns null if this is not possible. */
1354 expand_binop_directly (enum insn_code icode
, machine_mode mode
, optab binoptab
,
1356 rtx target
, int unsignedp
, enum optab_methods methods
,
1359 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
1360 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
1361 machine_mode mode0
, mode1
, tmp_mode
;
1362 class expand_operand ops
[3];
1365 rtx xop0
= op0
, xop1
= op1
;
1366 bool canonicalize_op1
= false;
1368 /* If it is a commutative operator and the modes would match
1369 if we would swap the operands, we can save the conversions. */
1370 commutative_p
= commutative_optab_p (binoptab
);
1372 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1373 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode0
)
1374 std::swap (xop0
, xop1
);
1376 /* If we are optimizing, force expensive constants into a register. */
1377 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1378 if (!shift_optab_p (binoptab
))
1379 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1381 /* Shifts and rotates often use a different mode for op1 from op0;
1382 for VOIDmode constants we don't know the mode, so force it
1383 to be canonicalized using convert_modes. */
1384 canonicalize_op1
= true;
1386 /* In case the insn wants input operands in modes different from
1387 those of the actual operands, convert the operands. It would
1388 seem that we don't need to convert CONST_INTs, but we do, so
1389 that they're properly zero-extended, sign-extended or truncated
1392 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1393 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1395 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1399 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1400 ? GET_MODE (xop1
) : mode
);
1401 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1403 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1407 /* If operation is commutative,
1408 try to make the first operand a register.
1409 Even better, try to make it the same as the target.
1410 Also try to make the last operand a constant. */
1412 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1413 std::swap (xop0
, xop1
);
1415 /* Now, if insn's predicates don't allow our operands, put them into
1418 if (binoptab
== vec_pack_trunc_optab
1419 || binoptab
== vec_pack_usat_optab
1420 || binoptab
== vec_pack_ssat_optab
1421 || binoptab
== vec_pack_ufix_trunc_optab
1422 || binoptab
== vec_pack_sfix_trunc_optab
1423 || binoptab
== vec_packu_float_optab
1424 || binoptab
== vec_packs_float_optab
)
1426 /* The mode of the result is different then the mode of the
1428 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1429 if (VECTOR_MODE_P (mode
)
1430 && maybe_ne (GET_MODE_NUNITS (tmp_mode
), 2 * GET_MODE_NUNITS (mode
)))
1432 delete_insns_since (last
);
1439 create_output_operand (&ops
[0], target
, tmp_mode
);
1440 create_input_operand (&ops
[1], xop0
, mode0
);
1441 create_input_operand (&ops
[2], xop1
, mode1
);
1442 pat
= maybe_gen_insn (icode
, 3, ops
);
1445 /* If PAT is composed of more than one insn, try to add an appropriate
1446 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1447 operand, call expand_binop again, this time without a target. */
1448 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1449 && ! add_equal_note (pat
, ops
[0].value
,
1450 optab_to_code (binoptab
),
1451 ops
[1].value
, ops
[2].value
, mode0
))
1453 delete_insns_since (last
);
1454 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1455 unsignedp
, methods
);
1459 return ops
[0].value
;
1461 delete_insns_since (last
);
1465 /* Generate code to perform an operation specified by BINOPTAB
1466 on operands OP0 and OP1, with result having machine-mode MODE.
1468 UNSIGNEDP is for the case where we have to widen the operands
1469 to perform the operation. It says to use zero-extension.
1471 If TARGET is nonzero, the value
1472 is generated there, if it is convenient to do so.
1473 In all cases an rtx is returned for the locus of the value;
1474 this may or may not be TARGET. */
1477 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1478 rtx target
, int unsignedp
, enum optab_methods methods
)
1480 enum optab_methods next_methods
1481 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1482 ? OPTAB_WIDEN
: methods
);
1483 enum mode_class mclass
;
1484 enum insn_code icode
;
1485 machine_mode wider_mode
;
1486 scalar_int_mode int_mode
;
1489 rtx_insn
*entry_last
= get_last_insn ();
1492 mclass
= GET_MODE_CLASS (mode
);
1494 /* If subtracting an integer constant, convert this into an addition of
1495 the negated constant. */
1497 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1499 op1
= negate_rtx (mode
, op1
);
1500 binoptab
= add_optab
;
1502 /* For shifts, constant invalid op1 might be expanded from different
1503 mode than MODE. As those are invalid, force them to a register
1504 to avoid further problems during expansion. */
1505 else if (CONST_INT_P (op1
)
1506 && shift_optab_p (binoptab
)
1507 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1509 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1510 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1513 /* Record where to delete back to if we backtrack. */
1514 last
= get_last_insn ();
1516 /* If we can do it with a three-operand insn, do so. */
1518 if (methods
!= OPTAB_MUST_WIDEN
)
1520 if (convert_optab_p (binoptab
))
1522 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
1523 icode
= find_widening_optab_handler (binoptab
, mode
, from_mode
);
1526 icode
= optab_handler (binoptab
, mode
);
1527 if (icode
!= CODE_FOR_nothing
)
1529 temp
= expand_binop_directly (icode
, mode
, binoptab
, op0
, op1
,
1530 target
, unsignedp
, methods
, last
);
1536 /* If we were trying to rotate, and that didn't work, try rotating
1537 the other direction before falling back to shifts and bitwise-or. */
1538 if (((binoptab
== rotl_optab
1539 && (icode
= optab_handler (rotr_optab
, mode
)) != CODE_FOR_nothing
)
1540 || (binoptab
== rotr_optab
1541 && (icode
= optab_handler (rotl_optab
, mode
)) != CODE_FOR_nothing
))
1542 && is_int_mode (mode
, &int_mode
))
1544 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1546 unsigned int bits
= GET_MODE_PRECISION (int_mode
);
1548 if (CONST_INT_P (op1
))
1549 newop1
= gen_int_shift_amount (int_mode
, bits
- INTVAL (op1
));
1550 else if (targetm
.shift_truncation_mask (int_mode
) == bits
- 1)
1551 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1553 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1554 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1555 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1557 temp
= expand_binop_directly (icode
, int_mode
, otheroptab
, op0
, newop1
,
1558 target
, unsignedp
, methods
, last
);
1563 /* If this is a multiply, see if we can do a widening operation that
1564 takes operands of this mode and makes a wider mode. */
1566 if (binoptab
== smul_optab
1567 && GET_MODE_2XWIDER_MODE (mode
).exists (&wider_mode
)
1568 && (convert_optab_handler ((unsignedp
1570 : smul_widen_optab
),
1571 wider_mode
, mode
) != CODE_FOR_nothing
))
1573 /* *_widen_optab needs to determine operand mode, make sure at least
1574 one operand has non-VOID mode. */
1575 if (GET_MODE (op0
) == VOIDmode
&& GET_MODE (op1
) == VOIDmode
)
1576 op0
= force_reg (mode
, op0
);
1577 temp
= expand_binop (wider_mode
,
1578 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1579 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1583 if (GET_MODE_CLASS (mode
) == MODE_INT
1584 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1585 return gen_lowpart (mode
, temp
);
1587 return convert_to_mode (mode
, temp
, unsignedp
);
1591 /* If this is a vector shift by a scalar, see if we can do a vector
1592 shift by a vector. If so, broadcast the scalar into a vector. */
1593 if (mclass
== MODE_VECTOR_INT
)
1595 optab otheroptab
= unknown_optab
;
1597 if (binoptab
== ashl_optab
)
1598 otheroptab
= vashl_optab
;
1599 else if (binoptab
== ashr_optab
)
1600 otheroptab
= vashr_optab
;
1601 else if (binoptab
== lshr_optab
)
1602 otheroptab
= vlshr_optab
;
1603 else if (binoptab
== rotl_optab
)
1604 otheroptab
= vrotl_optab
;
1605 else if (binoptab
== rotr_optab
)
1606 otheroptab
= vrotr_optab
;
1609 && (icode
= optab_handler (otheroptab
, mode
)) != CODE_FOR_nothing
)
1611 /* The scalar may have been extended to be too wide. Truncate
1612 it back to the proper size to fit in the broadcast vector. */
1613 scalar_mode inner_mode
= GET_MODE_INNER (mode
);
1614 if (!CONST_INT_P (op1
)
1615 && (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (op1
)))
1616 > GET_MODE_BITSIZE (inner_mode
)))
1617 op1
= force_reg (inner_mode
,
1618 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1620 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1623 temp
= expand_binop_directly (icode
, mode
, otheroptab
, op0
, vop1
,
1624 target
, unsignedp
, methods
, last
);
1631 /* Look for a wider mode of the same class for which we think we
1632 can open-code the operation. Check for a widening multiply at the
1633 wider mode as well. */
1635 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1636 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1637 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1639 machine_mode next_mode
;
1640 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1641 || (binoptab
== smul_optab
1642 && GET_MODE_WIDER_MODE (wider_mode
).exists (&next_mode
)
1643 && (find_widening_optab_handler ((unsignedp
1645 : smul_widen_optab
),
1647 != CODE_FOR_nothing
)))
1649 rtx xop0
= op0
, xop1
= op1
;
1652 /* For certain integer operations, we need not actually extend
1653 the narrow operands, as long as we will truncate
1654 the results to the same narrowness. */
1656 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1657 || binoptab
== xor_optab
1658 || binoptab
== add_optab
|| binoptab
== sub_optab
1659 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1660 && mclass
== MODE_INT
)
1663 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1665 if (binoptab
!= ashl_optab
)
1666 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1670 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1672 /* The second operand of a shift must always be extended. */
1673 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1674 no_extend
&& binoptab
!= ashl_optab
);
1676 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1677 unsignedp
, OPTAB_DIRECT
);
1680 if (mclass
!= MODE_INT
1681 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1684 target
= gen_reg_rtx (mode
);
1685 convert_move (target
, temp
, 0);
1689 return gen_lowpart (mode
, temp
);
1692 delete_insns_since (last
);
1696 /* If operation is commutative,
1697 try to make the first operand a register.
1698 Even better, try to make it the same as the target.
1699 Also try to make the last operand a constant. */
1700 if (commutative_optab_p (binoptab
)
1701 && swap_commutative_operands_with_target (target
, op0
, op1
))
1702 std::swap (op0
, op1
);
1704 /* These can be done a word at a time. */
1705 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1706 && is_int_mode (mode
, &int_mode
)
1707 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
1708 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1713 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1714 won't be accurate, so use a new target. */
1718 || reg_overlap_mentioned_p (target
, op0
)
1719 || reg_overlap_mentioned_p (target
, op1
)
1720 || !valid_multiword_target_p (target
))
1721 target
= gen_reg_rtx (int_mode
);
1725 /* Do the actual arithmetic. */
1726 machine_mode op0_mode
= GET_MODE (op0
);
1727 machine_mode op1_mode
= GET_MODE (op1
);
1728 if (op0_mode
== VOIDmode
)
1729 op0_mode
= int_mode
;
1730 if (op1_mode
== VOIDmode
)
1731 op1_mode
= int_mode
;
1732 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
1734 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
1735 rtx x
= expand_binop (word_mode
, binoptab
,
1736 operand_subword_force (op0
, i
, op0_mode
),
1737 operand_subword_force (op1
, i
, op1_mode
),
1738 target_piece
, unsignedp
, next_methods
);
1743 if (target_piece
!= x
)
1744 emit_move_insn (target_piece
, x
);
1747 insns
= get_insns ();
1750 if (i
== GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
)
1757 /* Synthesize double word shifts from single word shifts. */
1758 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1759 || binoptab
== ashr_optab
)
1760 && is_int_mode (mode
, &int_mode
)
1761 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1762 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1763 && GET_MODE_PRECISION (int_mode
) == GET_MODE_BITSIZE (int_mode
)
1764 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1765 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1766 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1768 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1769 scalar_int_mode op1_mode
;
1771 double_shift_mask
= targetm
.shift_truncation_mask (int_mode
);
1772 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1773 op1_mode
= (GET_MODE (op1
) != VOIDmode
1774 ? as_a
<scalar_int_mode
> (GET_MODE (op1
))
1777 /* Apply the truncation to constant shifts. */
1778 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1779 op1
= gen_int_mode (INTVAL (op1
) & double_shift_mask
, op1_mode
);
1781 if (op1
== CONST0_RTX (op1_mode
))
1784 /* Make sure that this is a combination that expand_doubleword_shift
1785 can handle. See the comments there for details. */
1786 if (double_shift_mask
== 0
1787 || (shift_mask
== BITS_PER_WORD
- 1
1788 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1791 rtx into_target
, outof_target
;
1792 rtx into_input
, outof_input
;
1793 int left_shift
, outof_word
;
1795 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1796 won't be accurate, so use a new target. */
1800 || reg_overlap_mentioned_p (target
, op0
)
1801 || reg_overlap_mentioned_p (target
, op1
)
1802 || !valid_multiword_target_p (target
))
1803 target
= gen_reg_rtx (int_mode
);
1807 /* OUTOF_* is the word we are shifting bits away from, and
1808 INTO_* is the word that we are shifting bits towards, thus
1809 they differ depending on the direction of the shift and
1810 WORDS_BIG_ENDIAN. */
1812 left_shift
= binoptab
== ashl_optab
;
1813 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1815 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1816 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1818 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1819 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1821 if (expand_doubleword_shift (op1_mode
, binoptab
,
1822 outof_input
, into_input
, op1
,
1823 outof_target
, into_target
,
1824 unsignedp
, next_methods
, shift_mask
))
1826 insns
= get_insns ();
1836 /* Synthesize double word rotates from single word shifts. */
1837 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1838 && is_int_mode (mode
, &int_mode
)
1839 && CONST_INT_P (op1
)
1840 && GET_MODE_PRECISION (int_mode
) == 2 * BITS_PER_WORD
1841 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1842 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1845 rtx into_target
, outof_target
;
1846 rtx into_input
, outof_input
;
1848 int shift_count
, left_shift
, outof_word
;
1850 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1851 won't be accurate, so use a new target. Do this also if target is not
1852 a REG, first because having a register instead may open optimization
1853 opportunities, and second because if target and op0 happen to be MEMs
1854 designating the same location, we would risk clobbering it too early
1855 in the code sequence we generate below. */
1860 || reg_overlap_mentioned_p (target
, op0
)
1861 || reg_overlap_mentioned_p (target
, op1
)
1862 || !valid_multiword_target_p (target
))
1863 target
= gen_reg_rtx (int_mode
);
1867 shift_count
= INTVAL (op1
);
1869 /* OUTOF_* is the word we are shifting bits away from, and
1870 INTO_* is the word that we are shifting bits towards, thus
1871 they differ depending on the direction of the shift and
1872 WORDS_BIG_ENDIAN. */
1874 left_shift
= (binoptab
== rotl_optab
);
1875 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1877 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1878 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1880 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1881 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1883 if (shift_count
== BITS_PER_WORD
)
1885 /* This is just a word swap. */
1886 emit_move_insn (outof_target
, into_input
);
1887 emit_move_insn (into_target
, outof_input
);
1892 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1893 HOST_WIDE_INT first_shift_count
, second_shift_count
;
1894 optab reverse_unsigned_shift
, unsigned_shift
;
1896 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1897 ? lshr_optab
: ashl_optab
);
1899 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1900 ? ashl_optab
: lshr_optab
);
1902 if (shift_count
> BITS_PER_WORD
)
1904 first_shift_count
= shift_count
- BITS_PER_WORD
;
1905 second_shift_count
= 2 * BITS_PER_WORD
- shift_count
;
1909 first_shift_count
= BITS_PER_WORD
- shift_count
;
1910 second_shift_count
= shift_count
;
1912 rtx first_shift_count_rtx
1913 = gen_int_shift_amount (word_mode
, first_shift_count
);
1914 rtx second_shift_count_rtx
1915 = gen_int_shift_amount (word_mode
, second_shift_count
);
1917 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1918 outof_input
, first_shift_count_rtx
,
1919 NULL_RTX
, unsignedp
, next_methods
);
1920 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1921 into_input
, second_shift_count_rtx
,
1922 NULL_RTX
, unsignedp
, next_methods
);
1924 if (into_temp1
!= 0 && into_temp2
!= 0)
1925 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1926 into_target
, unsignedp
, next_methods
);
1930 if (inter
!= 0 && inter
!= into_target
)
1931 emit_move_insn (into_target
, inter
);
1933 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1934 into_input
, first_shift_count_rtx
,
1935 NULL_RTX
, unsignedp
, next_methods
);
1936 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1937 outof_input
, second_shift_count_rtx
,
1938 NULL_RTX
, unsignedp
, next_methods
);
1940 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1941 inter
= expand_binop (word_mode
, ior_optab
,
1942 outof_temp1
, outof_temp2
,
1943 outof_target
, unsignedp
, next_methods
);
1945 if (inter
!= 0 && inter
!= outof_target
)
1946 emit_move_insn (outof_target
, inter
);
1949 insns
= get_insns ();
1959 /* These can be done a word at a time by propagating carries. */
1960 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1961 && is_int_mode (mode
, &int_mode
)
1962 && GET_MODE_SIZE (int_mode
) >= 2 * UNITS_PER_WORD
1963 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1966 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1967 const unsigned int nwords
= GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
;
1968 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1969 rtx xop0
, xop1
, xtarget
;
1971 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1972 value is one of those, use it. Otherwise, use 1 since it is the
1973 one easiest to get. */
1974 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1975 int normalizep
= STORE_FLAG_VALUE
;
1980 /* Prepare the operands. */
1981 xop0
= force_reg (int_mode
, op0
);
1982 xop1
= force_reg (int_mode
, op1
);
1984 xtarget
= gen_reg_rtx (int_mode
);
1986 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1989 /* Indicate for flow that the entire target reg is being set. */
1991 emit_clobber (xtarget
);
1993 /* Do the actual arithmetic. */
1994 for (i
= 0; i
< nwords
; i
++)
1996 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
1997 rtx target_piece
= operand_subword (xtarget
, index
, 1, int_mode
);
1998 rtx op0_piece
= operand_subword_force (xop0
, index
, int_mode
);
1999 rtx op1_piece
= operand_subword_force (xop1
, index
, int_mode
);
2002 /* Main add/subtract of the input operands. */
2003 x
= expand_binop (word_mode
, binoptab
,
2004 op0_piece
, op1_piece
,
2005 target_piece
, unsignedp
, next_methods
);
2011 /* Store carry from main add/subtract. */
2012 carry_out
= gen_reg_rtx (word_mode
);
2013 carry_out
= emit_store_flag_force (carry_out
,
2014 (binoptab
== add_optab
2017 word_mode
, 1, normalizep
);
2024 /* Add/subtract previous carry to main result. */
2025 newx
= expand_binop (word_mode
,
2026 normalizep
== 1 ? binoptab
: otheroptab
,
2028 NULL_RTX
, 1, next_methods
);
2032 /* Get out carry from adding/subtracting carry in. */
2033 rtx carry_tmp
= gen_reg_rtx (word_mode
);
2034 carry_tmp
= emit_store_flag_force (carry_tmp
,
2035 (binoptab
== add_optab
2038 word_mode
, 1, normalizep
);
2040 /* Logical-ior the two poss. carry together. */
2041 carry_out
= expand_binop (word_mode
, ior_optab
,
2042 carry_out
, carry_tmp
,
2043 carry_out
, 0, next_methods
);
2047 emit_move_insn (target_piece
, newx
);
2051 if (x
!= target_piece
)
2052 emit_move_insn (target_piece
, x
);
2055 carry_in
= carry_out
;
2058 if (i
== GET_MODE_BITSIZE (int_mode
) / (unsigned) BITS_PER_WORD
)
2060 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
2061 || ! rtx_equal_p (target
, xtarget
))
2063 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
2065 set_dst_reg_note (temp
, REG_EQUAL
,
2066 gen_rtx_fmt_ee (optab_to_code (binoptab
),
2067 int_mode
, copy_rtx (xop0
),
2078 delete_insns_since (last
);
2081 /* Attempt to synthesize double word multiplies using a sequence of word
2082 mode multiplications. We first attempt to generate a sequence using a
2083 more efficient unsigned widening multiply, and if that fails we then
2084 try using a signed widening multiply. */
2086 if (binoptab
== smul_optab
2087 && is_int_mode (mode
, &int_mode
)
2088 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2089 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
2090 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
2092 rtx product
= NULL_RTX
;
2093 if (convert_optab_handler (umul_widen_optab
, int_mode
, word_mode
)
2094 != CODE_FOR_nothing
)
2096 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
2099 delete_insns_since (last
);
2102 if (product
== NULL_RTX
2103 && (convert_optab_handler (smul_widen_optab
, int_mode
, word_mode
)
2104 != CODE_FOR_nothing
))
2106 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
2109 delete_insns_since (last
);
2112 if (product
!= NULL_RTX
)
2114 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
2116 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
2118 set_dst_reg_note (move
,
2120 gen_rtx_fmt_ee (MULT
, int_mode
,
2123 target
? target
: product
);
2129 /* Attempt to synthetize double word modulo by constant divisor. */
2130 if ((binoptab
== umod_optab
2131 || binoptab
== smod_optab
2132 || binoptab
== udiv_optab
2133 || binoptab
== sdiv_optab
)
2135 && CONST_INT_P (op1
)
2136 && is_int_mode (mode
, &int_mode
)
2137 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2138 && optab_handler ((binoptab
== umod_optab
|| binoptab
== udiv_optab
)
2139 ? udivmod_optab
: sdivmod_optab
,
2140 int_mode
) == CODE_FOR_nothing
2141 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
2142 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
2143 && optimize_insn_for_speed_p ())
2146 if ((binoptab
== umod_optab
|| binoptab
== smod_optab
)
2147 && (INTVAL (op1
) & 1) == 0)
2148 res
= expand_doubleword_mod (int_mode
, op0
, op1
,
2149 binoptab
== umod_optab
);
2152 rtx quot
= expand_doubleword_divmod (int_mode
, op0
, op1
, &res
,
2153 binoptab
== umod_optab
2154 || binoptab
== udiv_optab
);
2155 if (quot
== NULL_RTX
)
2157 else if (binoptab
== udiv_optab
|| binoptab
== sdiv_optab
)
2160 if (res
!= NULL_RTX
)
2162 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
2164 rtx_insn
*move
= emit_move_insn (target
? target
: res
,
2166 set_dst_reg_note (move
, REG_EQUAL
,
2167 gen_rtx_fmt_ee (optab_to_code (binoptab
),
2168 int_mode
, copy_rtx (op0
), op1
),
2169 target
? target
: res
);
2174 delete_insns_since (last
);
2177 /* It can't be open-coded in this mode.
2178 Use a library call if one is available and caller says that's ok. */
2180 libfunc
= optab_libfunc (binoptab
, mode
);
2182 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
2186 machine_mode op1_mode
= mode
;
2191 if (shift_optab_p (binoptab
))
2193 op1_mode
= targetm
.libgcc_shift_count_mode ();
2194 /* Specify unsigned here,
2195 since negative shift counts are meaningless. */
2196 op1x
= convert_to_mode (op1_mode
, op1
, 1);
2199 if (GET_MODE (op0
) != VOIDmode
2200 && GET_MODE (op0
) != mode
)
2201 op0
= convert_to_mode (mode
, op0
, unsignedp
);
2203 /* Pass 1 for NO_QUEUE so we don't lose any increments
2204 if the libcall is cse'd or moved. */
2205 value
= emit_library_call_value (libfunc
,
2206 NULL_RTX
, LCT_CONST
, mode
,
2207 op0
, mode
, op1x
, op1_mode
);
2209 insns
= get_insns ();
2212 bool trapv
= trapv_binoptab_p (binoptab
);
2213 target
= gen_reg_rtx (mode
);
2214 emit_libcall_block_1 (insns
, target
, value
,
2216 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
2217 mode
, op0
, op1
), trapv
);
2222 delete_insns_since (last
);
2224 /* It can't be done in this mode. Can we do it in a wider mode? */
2226 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
2227 || methods
== OPTAB_MUST_WIDEN
))
2229 /* Caller says, don't even try. */
2230 delete_insns_since (entry_last
);
2234 /* Compute the value of METHODS to pass to recursive calls.
2235 Don't allow widening to be tried recursively. */
2237 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
2239 /* Look for a wider mode of the same class for which it appears we can do
2242 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2244 /* This code doesn't make sense for conversion optabs, since we
2245 wouldn't then want to extend the operands to be the same size
2247 gcc_assert (!convert_optab_p (binoptab
));
2248 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2250 if (optab_handler (binoptab
, wider_mode
)
2251 || (methods
== OPTAB_LIB
2252 && optab_libfunc (binoptab
, wider_mode
)))
2254 rtx xop0
= op0
, xop1
= op1
;
2257 /* For certain integer operations, we need not actually extend
2258 the narrow operands, as long as we will truncate
2259 the results to the same narrowness. */
2261 if ((binoptab
== ior_optab
|| binoptab
== and_optab
2262 || binoptab
== xor_optab
2263 || binoptab
== add_optab
|| binoptab
== sub_optab
2264 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
2265 && mclass
== MODE_INT
)
2268 xop0
= widen_operand (xop0
, wider_mode
, mode
,
2269 unsignedp
, no_extend
);
2271 /* The second operand of a shift must always be extended. */
2272 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
2273 no_extend
&& binoptab
!= ashl_optab
);
2275 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
2276 unsignedp
, methods
);
2279 if (mclass
!= MODE_INT
2280 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2283 target
= gen_reg_rtx (mode
);
2284 convert_move (target
, temp
, 0);
2288 return gen_lowpart (mode
, temp
);
2291 delete_insns_since (last
);
2296 delete_insns_since (entry_last
);
2300 /* Expand a binary operator which has both signed and unsigned forms.
2301 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2304 If we widen unsigned operands, we may use a signed wider operation instead
2305 of an unsigned wider operation, since the result would be the same. */
2308 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
2309 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
2310 enum optab_methods methods
)
2313 optab direct_optab
= unsignedp
? uoptab
: soptab
;
2316 /* Do it without widening, if possible. */
2317 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2318 unsignedp
, OPTAB_DIRECT
);
2319 if (temp
|| methods
== OPTAB_DIRECT
)
2322 /* Try widening to a signed int. Disable any direct use of any
2323 signed insn in the current mode. */
2324 save_enable
= swap_optab_enable (soptab
, mode
, false);
2326 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2327 unsignedp
, OPTAB_WIDEN
);
2329 /* For unsigned operands, try widening to an unsigned int. */
2330 if (!temp
&& unsignedp
)
2331 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2332 unsignedp
, OPTAB_WIDEN
);
2333 if (temp
|| methods
== OPTAB_WIDEN
)
2336 /* Use the right width libcall if that exists. */
2337 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2338 unsignedp
, OPTAB_LIB
);
2339 if (temp
|| methods
== OPTAB_LIB
)
2342 /* Must widen and use a libcall, use either signed or unsigned. */
2343 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2344 unsignedp
, methods
);
2345 if (!temp
&& unsignedp
)
2346 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2347 unsignedp
, methods
);
2350 /* Undo the fiddling above. */
2352 swap_optab_enable (soptab
, mode
, true);
2356 /* Generate code to perform an operation specified by UNOPPTAB
2357 on operand OP0, with two results to TARG0 and TARG1.
2358 We assume that the order of the operands for the instruction
2359 is TARG0, TARG1, OP0.
2361 Either TARG0 or TARG1 may be zero, but what that means is that
2362 the result is not actually wanted. We will generate it into
2363 a dummy pseudo-reg and discard it. They may not both be zero.
2365 Returns 1 if this operation can be performed; 0 if not. */
2368 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
2371 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2372 enum mode_class mclass
;
2373 machine_mode wider_mode
;
2374 rtx_insn
*entry_last
= get_last_insn ();
2377 mclass
= GET_MODE_CLASS (mode
);
2380 targ0
= gen_reg_rtx (mode
);
2382 targ1
= gen_reg_rtx (mode
);
2384 /* Record where to go back to if we fail. */
2385 last
= get_last_insn ();
2387 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2389 class expand_operand ops
[3];
2390 enum insn_code icode
= optab_handler (unoptab
, mode
);
2392 create_fixed_operand (&ops
[0], targ0
);
2393 create_fixed_operand (&ops
[1], targ1
);
2394 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
2395 if (maybe_expand_insn (icode
, 3, ops
))
2399 /* It can't be done in this mode. Can we do it in a wider mode? */
2401 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2403 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2405 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2407 rtx t0
= gen_reg_rtx (wider_mode
);
2408 rtx t1
= gen_reg_rtx (wider_mode
);
2409 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2411 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
2413 convert_move (targ0
, t0
, unsignedp
);
2414 convert_move (targ1
, t1
, unsignedp
);
2418 delete_insns_since (last
);
2423 delete_insns_since (entry_last
);
2427 /* Generate code to perform an operation specified by BINOPTAB
2428 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2429 We assume that the order of the operands for the instruction
2430 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2431 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2433 Either TARG0 or TARG1 may be zero, but what that means is that
2434 the result is not actually wanted. We will generate it into
2435 a dummy pseudo-reg and discard it. They may not both be zero.
2437 Returns 1 if this operation can be performed; 0 if not. */
2440 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
2443 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2444 enum mode_class mclass
;
2445 machine_mode wider_mode
;
2446 rtx_insn
*entry_last
= get_last_insn ();
2449 mclass
= GET_MODE_CLASS (mode
);
2452 targ0
= gen_reg_rtx (mode
);
2454 targ1
= gen_reg_rtx (mode
);
2456 /* Record where to go back to if we fail. */
2457 last
= get_last_insn ();
2459 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2461 class expand_operand ops
[4];
2462 enum insn_code icode
= optab_handler (binoptab
, mode
);
2463 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2464 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2465 rtx xop0
= op0
, xop1
= op1
;
2467 /* If we are optimizing, force expensive constants into a register. */
2468 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2469 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2471 create_fixed_operand (&ops
[0], targ0
);
2472 create_convert_operand_from (&ops
[1], xop0
, mode
, unsignedp
);
2473 create_convert_operand_from (&ops
[2], xop1
, mode
, unsignedp
);
2474 create_fixed_operand (&ops
[3], targ1
);
2475 if (maybe_expand_insn (icode
, 4, ops
))
2477 delete_insns_since (last
);
2480 /* It can't be done in this mode. Can we do it in a wider mode? */
2482 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2484 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2486 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2488 rtx t0
= gen_reg_rtx (wider_mode
);
2489 rtx t1
= gen_reg_rtx (wider_mode
);
2490 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2491 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2493 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2496 convert_move (targ0
, t0
, unsignedp
);
2497 convert_move (targ1
, t1
, unsignedp
);
2501 delete_insns_since (last
);
2506 delete_insns_since (entry_last
);
2510 /* Expand the two-valued library call indicated by BINOPTAB, but
2511 preserve only one of the values. If TARG0 is non-NULL, the first
2512 value is placed into TARG0; otherwise the second value is placed
2513 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2514 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2515 This routine assumes that the value returned by the library call is
2516 as if the return value was of an integral mode twice as wide as the
2517 mode of OP0. Returns 1 if the call was successful. */
2520 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2521 rtx targ0
, rtx targ1
, enum rtx_code code
)
2524 machine_mode libval_mode
;
2529 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2530 gcc_assert (!targ0
!= !targ1
);
2532 mode
= GET_MODE (op0
);
2533 libfunc
= optab_libfunc (binoptab
, mode
);
2537 /* The value returned by the library function will have twice as
2538 many bits as the nominal MODE. */
2539 libval_mode
= smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode
));
2541 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2545 /* Get the part of VAL containing the value that we want. */
2546 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2547 targ0
? 0 : GET_MODE_SIZE (mode
));
2548 insns
= get_insns ();
2550 /* Move the into the desired location. */
2551 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2552 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2558 /* Wrapper around expand_unop which takes an rtx code to specify
2559 the operation to perform, not an optab pointer. All other
2560 arguments are the same. */
2562 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2563 rtx target
, int unsignedp
)
2565 optab unop
= code_to_optab (code
);
2568 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2574 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2576 A similar operation can be used for clrsb. UNOPTAB says which operation
2577 we are trying to expand. */
2579 widen_leading (scalar_int_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2581 opt_scalar_int_mode wider_mode_iter
;
2582 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2584 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2585 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2590 last
= get_last_insn ();
2593 target
= gen_reg_rtx (mode
);
2594 xop0
= widen_operand (op0
, wider_mode
, mode
,
2595 unoptab
!= clrsb_optab
, false);
2596 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2597 unoptab
!= clrsb_optab
);
2600 (wider_mode
, sub_optab
, temp
,
2601 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2602 - GET_MODE_PRECISION (mode
),
2604 target
, true, OPTAB_DIRECT
);
2606 delete_insns_since (last
);
2614 /* Attempt to emit (clrsb:mode op0) as
2615 (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2617 if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2619 (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2620 (ashr:mode op0 (const_int prec-1)))
2625 expand_clrsb_using_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2627 if (optimize_insn_for_size_p ()
2628 || optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2632 HOST_WIDE_INT val
= 0;
2633 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) != 2
2634 || val
!= GET_MODE_PRECISION (mode
))
2642 temp2
= expand_binop (mode
, ashl_optab
, op0
, const1_rtx
,
2643 NULL_RTX
, 0, OPTAB_DIRECT
);
2652 rtx temp
= expand_binop (mode
, ashr_optab
, op0
,
2653 GEN_INT (GET_MODE_PRECISION (mode
) - 1),
2654 NULL_RTX
, 0, OPTAB_DIRECT
);
2658 temp
= expand_binop (mode
, xor_optab
, temp2
, temp
, NULL_RTX
, 0,
2665 temp
= expand_binop (mode
, ior_optab
, temp
, const1_rtx
,
2666 NULL_RTX
, 0, OPTAB_DIRECT
);
2670 temp
= expand_unop_direct (mode
, clz_optab
, temp
, val
? NULL_RTX
: target
,
2676 temp
= expand_binop (mode
, add_optab
, temp
, constm1_rtx
,
2677 target
, 0, OPTAB_DIRECT
);
2682 rtx_insn
*seq
= get_insns ();
2685 add_equal_note (seq
, temp
, CLRSB
, op0
, NULL_RTX
, mode
);
2690 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2691 quantities, choosing which based on whether the high word is nonzero. */
2693 expand_doubleword_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2695 rtx xop0
= force_reg (mode
, op0
);
2696 rtx subhi
= gen_highpart (word_mode
, xop0
);
2697 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2698 rtx_code_label
*hi0_label
= gen_label_rtx ();
2699 rtx_code_label
*after_label
= gen_label_rtx ();
2703 /* If we were not given a target, use a word_mode register, not a
2704 'mode' register. The result will fit, and nobody is expecting
2705 anything bigger (the return type of __builtin_clz* is int). */
2707 target
= gen_reg_rtx (word_mode
);
2709 /* In any case, write to a word_mode scratch in both branches of the
2710 conditional, so we can ensure there is a single move insn setting
2711 'target' to tag a REG_EQUAL note on. */
2712 result
= gen_reg_rtx (word_mode
);
2716 /* If the high word is not equal to zero,
2717 then clz of the full value is clz of the high word. */
2718 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2719 word_mode
, true, hi0_label
);
2721 temp
= expand_unop_direct (word_mode
, clz_optab
, subhi
, result
, true);
2726 convert_move (result
, temp
, true);
2728 emit_jump_insn (targetm
.gen_jump (after_label
));
2731 /* Else clz of the full value is clz of the low word plus the number
2732 of bits in the high word. */
2733 emit_label (hi0_label
);
2735 temp
= expand_unop_direct (word_mode
, clz_optab
, sublo
, 0, true);
2738 temp
= expand_binop (word_mode
, add_optab
, temp
,
2739 gen_int_mode (GET_MODE_BITSIZE (word_mode
), word_mode
),
2740 result
, true, OPTAB_DIRECT
);
2744 convert_move (result
, temp
, true);
2746 emit_label (after_label
);
2747 convert_move (target
, result
, true);
2752 add_equal_note (seq
, target
, CLZ
, xop0
, NULL_RTX
, mode
);
2761 /* Try calculating popcount of a double-word quantity as two popcount's of
2762 word-sized quantities and summing up the results. */
2764 expand_doubleword_popcount (scalar_int_mode mode
, rtx op0
, rtx target
)
2771 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2772 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2774 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2775 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2783 /* If we were not given a target, use a word_mode register, not a
2784 'mode' register. The result will fit, and nobody is expecting
2785 anything bigger (the return type of __builtin_popcount* is int). */
2787 target
= gen_reg_rtx (word_mode
);
2789 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2794 add_equal_note (seq
, t
, POPCOUNT
, op0
, NULL_RTX
, mode
);
2802 (parity:narrow (low (x) ^ high (x))) */
2804 expand_doubleword_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2806 rtx t
= expand_binop (word_mode
, xor_optab
,
2807 operand_subword_force (op0
, 0, mode
),
2808 operand_subword_force (op0
, 1, mode
),
2809 NULL_RTX
, 0, OPTAB_DIRECT
);
2810 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2816 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2818 widen_bswap (scalar_int_mode mode
, rtx op0
, rtx target
)
2822 opt_scalar_int_mode wider_mode_iter
;
2824 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2825 if (optab_handler (bswap_optab
, wider_mode_iter
.require ())
2826 != CODE_FOR_nothing
)
2829 if (!wider_mode_iter
.exists ())
2832 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2833 last
= get_last_insn ();
2835 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2836 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2838 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2839 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2841 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2842 GET_MODE_BITSIZE (wider_mode
)
2843 - GET_MODE_BITSIZE (mode
),
2849 target
= gen_reg_rtx (mode
);
2850 emit_move_insn (target
, gen_lowpart (mode
, x
));
2853 delete_insns_since (last
);
2858 /* Try calculating bswap as two bswaps of two word-sized operands. */
2861 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2865 t1
= expand_unop (word_mode
, bswap_optab
,
2866 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2867 t0
= expand_unop (word_mode
, bswap_optab
,
2868 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2870 if (target
== 0 || !valid_multiword_target_p (target
))
2871 target
= gen_reg_rtx (mode
);
2873 emit_clobber (target
);
2874 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2875 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2880 /* Try calculating (parity x) as (and (popcount x) 1), where
2881 popcount can also be done in a wider mode. */
2883 expand_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2885 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2886 opt_scalar_int_mode wider_mode_iter
;
2887 FOR_EACH_MODE_FROM (wider_mode_iter
, mode
)
2889 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2890 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2895 last
= get_last_insn ();
2897 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2898 target
= gen_reg_rtx (wider_mode
);
2900 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2901 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2904 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2905 target
, true, OPTAB_DIRECT
);
2909 if (mclass
!= MODE_INT
2910 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2911 return convert_to_mode (mode
, temp
, 0);
2913 return gen_lowpart (mode
, temp
);
2916 delete_insns_since (last
);
2922 /* Try calculating ctz(x) as K - clz(x & -x) ,
2923 where K is GET_MODE_PRECISION(mode) - 1.
2925 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2926 don't have to worry about what the hardware does in that case. (If
2927 the clz instruction produces the usual value at 0, which is K, the
2928 result of this code sequence will be -1; expand_ffs, below, relies
2929 on this. It might be nice to have it be K instead, for consistency
2930 with the (very few) processors that provide a ctz with a defined
2931 value, but that would take one more instruction, and it would be
2932 less convenient for expand_ffs anyway. */
2935 expand_ctz (scalar_int_mode mode
, rtx op0
, rtx target
)
2940 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2945 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2947 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2948 true, OPTAB_DIRECT
);
2950 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2952 temp
= expand_binop (mode
, sub_optab
,
2953 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
2955 true, OPTAB_DIRECT
);
2965 add_equal_note (seq
, temp
, CTZ
, op0
, NULL_RTX
, mode
);
2971 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2972 else with the sequence used by expand_clz.
2974 The ffs builtin promises to return zero for a zero value and ctz/clz
2975 may have an undefined value in that case. If they do not give us a
2976 convenient value, we have to generate a test and branch. */
2978 expand_ffs (scalar_int_mode mode
, rtx op0
, rtx target
)
2980 HOST_WIDE_INT val
= 0;
2981 bool defined_at_zero
= false;
2985 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
2989 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
2993 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
2995 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
2998 temp
= expand_ctz (mode
, op0
, 0);
3002 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
3004 defined_at_zero
= true;
3005 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
3011 if (defined_at_zero
&& val
== -1)
3012 /* No correction needed at zero. */;
3015 /* We don't try to do anything clever with the situation found
3016 on some processors (eg Alpha) where ctz(0:mode) ==
3017 bitsize(mode). If someone can think of a way to send N to -1
3018 and leave alone all values in the range 0..N-1 (where N is a
3019 power of two), cheaper than this test-and-branch, please add it.
3021 The test-and-branch is done after the operation itself, in case
3022 the operation sets condition codes that can be recycled for this.
3023 (This is true on i386, for instance.) */
3025 rtx_code_label
*nonzero_label
= gen_label_rtx ();
3026 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
3027 mode
, true, nonzero_label
);
3029 convert_move (temp
, GEN_INT (-1), false);
3030 emit_label (nonzero_label
);
3033 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
3034 to produce a value in the range 0..bitsize. */
3035 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
3036 target
, false, OPTAB_DIRECT
);
3043 add_equal_note (seq
, temp
, FFS
, op0
, NULL_RTX
, mode
);
3052 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
3053 conditions, VAL may already be a SUBREG against which we cannot generate
3054 a further SUBREG. In this case, we expect forcing the value into a
3055 register will work around the situation. */
3058 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
3062 ret
= lowpart_subreg (omode
, val
, imode
);
3065 val
= force_reg (imode
, val
);
3066 ret
= lowpart_subreg (omode
, val
, imode
);
3067 gcc_assert (ret
!= NULL
);
3072 /* Expand a floating point absolute value or negation operation via a
3073 logical operation on the sign bit. */
3076 expand_absneg_bit (enum rtx_code code
, scalar_float_mode mode
,
3077 rtx op0
, rtx target
)
3079 const struct real_format
*fmt
;
3080 int bitpos
, word
, nwords
, i
;
3081 scalar_int_mode imode
;
3085 /* The format has to have a simple sign bit. */
3086 fmt
= REAL_MODE_FORMAT (mode
);
3090 bitpos
= fmt
->signbit_rw
;
3094 /* Don't create negative zeros if the format doesn't support them. */
3095 if (code
== NEG
&& !fmt
->has_signed_zero
)
3098 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3100 if (!int_mode_for_mode (mode
).exists (&imode
))
3109 if (FLOAT_WORDS_BIG_ENDIAN
)
3110 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3112 word
= bitpos
/ BITS_PER_WORD
;
3113 bitpos
= bitpos
% BITS_PER_WORD
;
3114 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3117 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3123 || reg_overlap_mentioned_p (target
, op0
)
3124 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3125 target
= gen_reg_rtx (mode
);
3131 for (i
= 0; i
< nwords
; ++i
)
3133 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3134 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3138 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
3140 immed_wide_int_const (mask
, imode
),
3141 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3142 if (temp
!= targ_piece
)
3143 emit_move_insn (targ_piece
, temp
);
3146 emit_move_insn (targ_piece
, op0_piece
);
3149 insns
= get_insns ();
3156 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
3157 gen_lowpart (imode
, op0
),
3158 immed_wide_int_const (mask
, imode
),
3159 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3160 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3162 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
3163 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
3170 /* As expand_unop, but will fail rather than attempt the operation in a
3171 different mode or with a libcall. */
3173 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
3176 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
3178 class expand_operand ops
[2];
3179 enum insn_code icode
= optab_handler (unoptab
, mode
);
3180 rtx_insn
*last
= get_last_insn ();
3183 create_output_operand (&ops
[0], target
, mode
);
3184 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
3185 pat
= maybe_gen_insn (icode
, 2, ops
);
3188 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3189 && ! add_equal_note (pat
, ops
[0].value
,
3190 optab_to_code (unoptab
),
3191 ops
[1].value
, NULL_RTX
, mode
))
3193 delete_insns_since (last
);
3194 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
3199 return ops
[0].value
;
3205 /* Generate code to perform an operation specified by UNOPTAB
3206 on operand OP0, with result having machine-mode MODE.
3208 UNSIGNEDP is for the case where we have to widen the operands
3209 to perform the operation. It says to use zero-extension.
3211 If TARGET is nonzero, the value
3212 is generated there, if it is convenient to do so.
3213 In all cases an rtx is returned for the locus of the value;
3214 this may or may not be TARGET. */
3217 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
3220 enum mode_class mclass
= GET_MODE_CLASS (mode
);
3221 machine_mode wider_mode
;
3222 scalar_int_mode int_mode
;
3223 scalar_float_mode float_mode
;
3227 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
3231 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3233 /* Widening (or narrowing) clz needs special treatment. */
3234 if (unoptab
== clz_optab
)
3236 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3238 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
3242 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3243 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3245 temp
= expand_doubleword_clz (int_mode
, op0
, target
);
3254 if (unoptab
== clrsb_optab
)
3256 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3258 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
3261 temp
= expand_clrsb_using_clz (int_mode
, op0
, target
);
3268 if (unoptab
== popcount_optab
3269 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3270 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3271 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3272 && optimize_insn_for_speed_p ())
3274 temp
= expand_doubleword_popcount (int_mode
, op0
, target
);
3279 if (unoptab
== parity_optab
3280 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3281 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3282 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3283 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
3284 && optimize_insn_for_speed_p ())
3286 temp
= expand_doubleword_parity (int_mode
, op0
, target
);
3291 /* Widening (or narrowing) bswap needs special treatment. */
3292 if (unoptab
== bswap_optab
)
3294 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3295 or ROTATERT. First try these directly; if this fails, then try the
3296 obvious pair of shifts with allowed widening, as this will probably
3297 be always more efficient than the other fallback methods. */
3303 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
3305 temp
= expand_binop (mode
, rotl_optab
, op0
,
3306 gen_int_shift_amount (mode
, 8),
3307 target
, unsignedp
, OPTAB_DIRECT
);
3312 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
3314 temp
= expand_binop (mode
, rotr_optab
, op0
,
3315 gen_int_shift_amount (mode
, 8),
3316 target
, unsignedp
, OPTAB_DIRECT
);
3321 last
= get_last_insn ();
3323 temp1
= expand_binop (mode
, ashl_optab
, op0
,
3324 gen_int_shift_amount (mode
, 8), NULL_RTX
,
3325 unsignedp
, OPTAB_WIDEN
);
3326 temp2
= expand_binop (mode
, lshr_optab
, op0
,
3327 gen_int_shift_amount (mode
, 8), NULL_RTX
,
3328 unsignedp
, OPTAB_WIDEN
);
3331 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
3332 unsignedp
, OPTAB_WIDEN
);
3337 delete_insns_since (last
);
3340 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3342 temp
= widen_bswap (int_mode
, op0
, target
);
3346 /* We do not provide a 128-bit bswap in libgcc so force the use of
3347 a double bswap for 64-bit targets. */
3348 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3349 && (UNITS_PER_WORD
== 8
3350 || optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
))
3352 temp
= expand_doubleword_bswap (mode
, op0
, target
);
3361 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3362 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3364 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
3367 rtx_insn
*last
= get_last_insn ();
3369 /* For certain operations, we need not actually extend
3370 the narrow operand, as long as we will truncate the
3371 results to the same narrowness. */
3373 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3374 (unoptab
== neg_optab
3375 || unoptab
== one_cmpl_optab
)
3376 && mclass
== MODE_INT
);
3378 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3383 if (mclass
!= MODE_INT
3384 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
3387 target
= gen_reg_rtx (mode
);
3388 convert_move (target
, temp
, 0);
3392 return gen_lowpart (mode
, temp
);
3395 delete_insns_since (last
);
3399 /* These can be done a word at a time. */
3400 if (unoptab
== one_cmpl_optab
3401 && is_int_mode (mode
, &int_mode
)
3402 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
3403 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3410 || reg_overlap_mentioned_p (target
, op0
)
3411 || !valid_multiword_target_p (target
))
3412 target
= gen_reg_rtx (int_mode
);
3416 /* Do the actual arithmetic. */
3417 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
3419 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
3420 rtx x
= expand_unop (word_mode
, unoptab
,
3421 operand_subword_force (op0
, i
, int_mode
),
3422 target_piece
, unsignedp
);
3424 if (target_piece
!= x
)
3425 emit_move_insn (target_piece
, x
);
3428 insns
= get_insns ();
3435 /* Emit ~op0 as op0 ^ -1. */
3436 if (unoptab
== one_cmpl_optab
3437 && (SCALAR_INT_MODE_P (mode
) || GET_MODE_CLASS (mode
) == MODE_VECTOR_INT
)
3438 && optab_handler (xor_optab
, mode
) != CODE_FOR_nothing
)
3440 temp
= expand_binop (mode
, xor_optab
, op0
, CONSTM1_RTX (mode
),
3441 target
, unsignedp
, OPTAB_DIRECT
);
3446 if (optab_to_code (unoptab
) == NEG
)
3448 /* Try negating floating point values by flipping the sign bit. */
3449 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3451 temp
= expand_absneg_bit (NEG
, float_mode
, op0
, target
);
3456 /* If there is no negation pattern, and we have no negative zero,
3457 try subtracting from zero. */
3458 if (!HONOR_SIGNED_ZEROS (mode
))
3460 temp
= expand_binop (mode
, (unoptab
== negv_optab
3461 ? subv_optab
: sub_optab
),
3462 CONST0_RTX (mode
), op0
, target
,
3463 unsignedp
, OPTAB_DIRECT
);
3469 /* Try calculating parity (x) as popcount (x) % 2. */
3470 if (unoptab
== parity_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3472 temp
= expand_parity (int_mode
, op0
, target
);
3477 /* Try implementing ffs (x) in terms of clz (x). */
3478 if (unoptab
== ffs_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3480 temp
= expand_ffs (int_mode
, op0
, target
);
3485 /* Try implementing ctz (x) in terms of clz (x). */
3486 if (unoptab
== ctz_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3488 temp
= expand_ctz (int_mode
, op0
, target
);
3494 /* Now try a library call in this mode. */
3495 libfunc
= optab_libfunc (unoptab
, mode
);
3501 machine_mode outmode
= mode
;
3503 /* All of these functions return small values. Thus we choose to
3504 have them return something that isn't a double-word. */
3505 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
3506 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
3507 || unoptab
== parity_optab
)
3509 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
3510 optab_libfunc (unoptab
, mode
)));
3514 /* Pass 1 for NO_QUEUE so we don't lose any increments
3515 if the libcall is cse'd or moved. */
3516 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
3518 insns
= get_insns ();
3521 target
= gen_reg_rtx (outmode
);
3522 bool trapv
= trapv_unoptab_p (unoptab
);
3524 eq_value
= NULL_RTX
;
3527 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
3528 if (GET_MODE_UNIT_SIZE (outmode
) < GET_MODE_UNIT_SIZE (mode
))
3529 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
3530 else if (GET_MODE_UNIT_SIZE (outmode
) > GET_MODE_UNIT_SIZE (mode
))
3531 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
3532 outmode
, eq_value
, mode
);
3534 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
3539 /* It can't be done in this mode. Can we do it in a wider mode? */
3541 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3543 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3545 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
3546 || optab_libfunc (unoptab
, wider_mode
))
3549 rtx_insn
*last
= get_last_insn ();
3551 /* For certain operations, we need not actually extend
3552 the narrow operand, as long as we will truncate the
3553 results to the same narrowness. */
3554 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3555 (unoptab
== neg_optab
3556 || unoptab
== one_cmpl_optab
3557 || unoptab
== bswap_optab
)
3558 && mclass
== MODE_INT
);
3560 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3563 /* If we are generating clz using wider mode, adjust the
3564 result. Similarly for clrsb. */
3565 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3568 scalar_int_mode wider_int_mode
3569 = as_a
<scalar_int_mode
> (wider_mode
);
3570 int_mode
= as_a
<scalar_int_mode
> (mode
);
3572 (wider_mode
, sub_optab
, temp
,
3573 gen_int_mode (GET_MODE_PRECISION (wider_int_mode
)
3574 - GET_MODE_PRECISION (int_mode
),
3576 target
, true, OPTAB_DIRECT
);
3579 /* Likewise for bswap. */
3580 if (unoptab
== bswap_optab
&& temp
!= 0)
3582 scalar_int_mode wider_int_mode
3583 = as_a
<scalar_int_mode
> (wider_mode
);
3584 int_mode
= as_a
<scalar_int_mode
> (mode
);
3585 gcc_assert (GET_MODE_PRECISION (wider_int_mode
)
3586 == GET_MODE_BITSIZE (wider_int_mode
)
3587 && GET_MODE_PRECISION (int_mode
)
3588 == GET_MODE_BITSIZE (int_mode
));
3590 temp
= expand_shift (RSHIFT_EXPR
, wider_int_mode
, temp
,
3591 GET_MODE_BITSIZE (wider_int_mode
)
3592 - GET_MODE_BITSIZE (int_mode
),
3598 if (mclass
!= MODE_INT
)
3601 target
= gen_reg_rtx (mode
);
3602 convert_move (target
, temp
, 0);
3606 return gen_lowpart (mode
, temp
);
3609 delete_insns_since (last
);
3614 /* One final attempt at implementing negation via subtraction,
3615 this time allowing widening of the operand. */
3616 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3619 temp
= expand_binop (mode
,
3620 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3621 CONST0_RTX (mode
), op0
,
3622 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3630 /* Emit code to compute the absolute value of OP0, with result to
3631 TARGET if convenient. (TARGET may be 0.) The return value says
3632 where the result actually is to be found.
3634 MODE is the mode of the operand; the mode of the result is
3635 different but can be deduced from MODE.
3640 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3641 int result_unsignedp
)
3645 if (GET_MODE_CLASS (mode
) != MODE_INT
3647 result_unsignedp
= 1;
3649 /* First try to do it with a special abs instruction. */
3650 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3655 /* For floating point modes, try clearing the sign bit. */
3656 scalar_float_mode float_mode
;
3657 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3659 temp
= expand_absneg_bit (ABS
, float_mode
, op0
, target
);
3664 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3665 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3666 && !HONOR_SIGNED_ZEROS (mode
))
3668 rtx_insn
*last
= get_last_insn ();
3670 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3673 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3679 delete_insns_since (last
);
3682 /* If this machine has expensive jumps, we can do integer absolute
3683 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3684 where W is the width of MODE. */
3686 scalar_int_mode int_mode
;
3687 if (is_int_mode (mode
, &int_mode
)
3688 && BRANCH_COST (optimize_insn_for_speed_p (),
3691 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3692 GET_MODE_PRECISION (int_mode
) - 1,
3695 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3698 temp
= expand_binop (int_mode
,
3699 result_unsignedp
? sub_optab
: subv_optab
,
3700 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3710 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3711 int result_unsignedp
, int safe
)
3714 rtx_code_label
*op1
;
3716 if (GET_MODE_CLASS (mode
) != MODE_INT
3718 result_unsignedp
= 1;
3720 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3724 /* If that does not win, use conditional jump and negate. */
3726 /* It is safe to use the target if it is the same
3727 as the source if this is also a pseudo register */
3728 if (op0
== target
&& REG_P (op0
)
3729 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3732 op1
= gen_label_rtx ();
3733 if (target
== 0 || ! safe
3734 || GET_MODE (target
) != mode
3735 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3737 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3738 target
= gen_reg_rtx (mode
);
3740 emit_move_insn (target
, op0
);
3743 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3744 NULL_RTX
, NULL
, op1
,
3745 profile_probability::uninitialized ());
3747 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3750 emit_move_insn (target
, op0
);
3756 /* Emit code to compute the one's complement absolute value of OP0
3757 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3758 (TARGET may be NULL_RTX.) The return value says where the result
3759 actually is to be found.
3761 MODE is the mode of the operand; the mode of the result is
3762 different but can be deduced from MODE. */
3765 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3769 /* Not applicable for floating point modes. */
3770 if (FLOAT_MODE_P (mode
))
3773 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3774 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3776 rtx_insn
*last
= get_last_insn ();
3778 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3780 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3786 delete_insns_since (last
);
3789 /* If this machine has expensive jumps, we can do one's complement
3790 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3792 scalar_int_mode int_mode
;
3793 if (is_int_mode (mode
, &int_mode
)
3794 && BRANCH_COST (optimize_insn_for_speed_p (),
3797 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3798 GET_MODE_PRECISION (int_mode
) - 1,
3801 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3811 /* A subroutine of expand_copysign, perform the copysign operation using the
3812 abs and neg primitives advertised to exist on the target. The assumption
3813 is that we have a split register file, and leaving op0 in fp registers,
3814 and not playing with subregs so much, will help the register allocator. */
3817 expand_copysign_absneg (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3818 int bitpos
, bool op0_is_abs
)
3820 scalar_int_mode imode
;
3821 enum insn_code icode
;
3823 rtx_code_label
*label
;
3828 /* Check if the back end provides an insn that handles signbit for the
3830 icode
= optab_handler (signbit_optab
, mode
);
3831 if (icode
!= CODE_FOR_nothing
)
3833 imode
= as_a
<scalar_int_mode
> (insn_data
[(int) icode
].operand
[0].mode
);
3834 sign
= gen_reg_rtx (imode
);
3835 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3839 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3841 if (!int_mode_for_mode (mode
).exists (&imode
))
3843 op1
= gen_lowpart (imode
, op1
);
3850 if (FLOAT_WORDS_BIG_ENDIAN
)
3851 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3853 word
= bitpos
/ BITS_PER_WORD
;
3854 bitpos
= bitpos
% BITS_PER_WORD
;
3855 op1
= operand_subword_force (op1
, word
, mode
);
3858 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3859 sign
= expand_binop (imode
, and_optab
, op1
,
3860 immed_wide_int_const (mask
, imode
),
3861 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3866 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3873 if (target
== NULL_RTX
)
3874 target
= copy_to_reg (op0
);
3876 emit_move_insn (target
, op0
);
3879 label
= gen_label_rtx ();
3880 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3882 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3883 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3885 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3887 emit_move_insn (target
, op0
);
3895 /* A subroutine of expand_copysign, perform the entire copysign operation
3896 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3897 is true if op0 is known to have its sign bit clear. */
3900 expand_copysign_bit (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3901 int bitpos
, bool op0_is_abs
)
3903 scalar_int_mode imode
;
3904 int word
, nwords
, i
;
3908 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3910 if (!int_mode_for_mode (mode
).exists (&imode
))
3919 if (FLOAT_WORDS_BIG_ENDIAN
)
3920 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3922 word
= bitpos
/ BITS_PER_WORD
;
3923 bitpos
= bitpos
% BITS_PER_WORD
;
3924 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3927 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3932 || reg_overlap_mentioned_p (target
, op0
)
3933 || reg_overlap_mentioned_p (target
, op1
)
3934 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3935 target
= gen_reg_rtx (mode
);
3941 for (i
= 0; i
< nwords
; ++i
)
3943 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3944 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3950 = expand_binop (imode
, and_optab
, op0_piece
,
3951 immed_wide_int_const (~mask
, imode
),
3952 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3953 op1
= expand_binop (imode
, and_optab
,
3954 operand_subword_force (op1
, i
, mode
),
3955 immed_wide_int_const (mask
, imode
),
3956 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3958 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
3959 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3960 if (temp
!= targ_piece
)
3961 emit_move_insn (targ_piece
, temp
);
3964 emit_move_insn (targ_piece
, op0_piece
);
3967 insns
= get_insns ();
3974 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
3975 immed_wide_int_const (mask
, imode
),
3976 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3978 op0
= gen_lowpart (imode
, op0
);
3980 op0
= expand_binop (imode
, and_optab
, op0
,
3981 immed_wide_int_const (~mask
, imode
),
3982 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3984 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
3985 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3986 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3992 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3993 scalar floating point mode. Return NULL if we do not know how to
3994 expand the operation inline. */
3997 expand_copysign (rtx op0
, rtx op1
, rtx target
)
3999 scalar_float_mode mode
;
4000 const struct real_format
*fmt
;
4004 mode
= as_a
<scalar_float_mode
> (GET_MODE (op0
));
4005 gcc_assert (GET_MODE (op1
) == mode
);
4007 /* First try to do it with a special instruction. */
4008 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
4009 target
, 0, OPTAB_DIRECT
);
4013 fmt
= REAL_MODE_FORMAT (mode
);
4014 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
4018 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
4020 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
4021 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
4025 if (fmt
->signbit_ro
>= 0
4026 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
4027 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
4028 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
4030 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
4031 fmt
->signbit_ro
, op0_is_abs
);
4036 if (fmt
->signbit_rw
< 0)
4038 return expand_copysign_bit (mode
, op0
, op1
, target
,
4039 fmt
->signbit_rw
, op0_is_abs
);
4042 /* Generate an instruction whose insn-code is INSN_CODE,
4043 with two operands: an output TARGET and an input OP0.
4044 TARGET *must* be nonzero, and the output is always stored there.
4045 CODE is an rtx code such that (CODE OP0) is an rtx that describes
4046 the value that is stored into TARGET.
4048 Return false if expansion failed. */
4051 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
4054 class expand_operand ops
[2];
4057 create_output_operand (&ops
[0], target
, GET_MODE (target
));
4058 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
4059 pat
= maybe_gen_insn (icode
, 2, ops
);
4063 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
4065 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
,
4070 if (ops
[0].value
!= target
)
4071 emit_move_insn (target
, ops
[0].value
);
4074 /* Generate an instruction whose insn-code is INSN_CODE,
4075 with two operands: an output TARGET and an input OP0.
4076 TARGET *must* be nonzero, and the output is always stored there.
4077 CODE is an rtx code such that (CODE OP0) is an rtx that describes
4078 the value that is stored into TARGET. */
4081 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
4083 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
4087 struct no_conflict_data
4090 rtx_insn
*first
, *insn
;
4094 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
4095 the currently examined clobber / store has to stay in the list of
4096 insns that constitute the actual libcall block. */
4098 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
4100 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
4102 /* If this inns directly contributes to setting the target, it must stay. */
4103 if (reg_overlap_mentioned_p (p
->target
, dest
))
4104 p
->must_stay
= true;
4105 /* If we haven't committed to keeping any other insns in the list yet,
4106 there is nothing more to check. */
4107 else if (p
->insn
== p
->first
)
4109 /* If this insn sets / clobbers a register that feeds one of the insns
4110 already in the list, this insn has to stay too. */
4111 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
4112 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
4113 || reg_used_between_p (dest
, p
->first
, p
->insn
)
4114 /* Likewise if this insn depends on a register set by a previous
4115 insn in the list, or if it sets a result (presumably a hard
4116 register) that is set or clobbered by a previous insn.
4117 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4118 SET_DEST perform the former check on the address, and the latter
4119 check on the MEM. */
4120 || (GET_CODE (set
) == SET
4121 && (modified_in_p (SET_SRC (set
), p
->first
)
4122 || modified_in_p (SET_DEST (set
), p
->first
)
4123 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
4124 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
4125 p
->must_stay
= true;
4129 /* Emit code to make a call to a constant function or a library call.
4131 INSNS is a list containing all insns emitted in the call.
4132 These insns leave the result in RESULT. Our block is to copy RESULT
4133 to TARGET, which is logically equivalent to EQUIV.
4135 We first emit any insns that set a pseudo on the assumption that these are
4136 loading constants into registers; doing so allows them to be safely cse'ed
4137 between blocks. Then we emit all the other insns in the block, followed by
4138 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4139 note with an operand of EQUIV. */
4142 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
4143 bool equiv_may_trap
)
4145 rtx final_dest
= target
;
4146 rtx_insn
*next
, *last
, *insn
;
4148 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4149 into a MEM later. Protect the libcall block from this change. */
4150 if (! REG_P (target
) || REG_USERVAR_P (target
))
4151 target
= gen_reg_rtx (GET_MODE (target
));
4153 /* If we're using non-call exceptions, a libcall corresponding to an
4154 operation that may trap may also trap. */
4155 /* ??? See the comment in front of make_reg_eh_region_note. */
4156 if (cfun
->can_throw_non_call_exceptions
4157 && (equiv_may_trap
|| may_trap_p (equiv
)))
4159 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
4162 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
4165 int lp_nr
= INTVAL (XEXP (note
, 0));
4166 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
4167 remove_note (insn
, note
);
4173 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4174 reg note to indicate that this call cannot throw or execute a nonlocal
4175 goto (unless there is already a REG_EH_REGION note, in which case
4177 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
4179 make_reg_eh_region_note_nothrow_nononlocal (insn
);
4182 /* First emit all insns that set pseudos. Remove them from the list as
4183 we go. Avoid insns that set pseudos which were referenced in previous
4184 insns. These can be generated by move_by_pieces, for example,
4185 to update an address. Similarly, avoid insns that reference things
4186 set in previous insns. */
4188 for (insn
= insns
; insn
; insn
= next
)
4190 rtx set
= single_set (insn
);
4192 next
= NEXT_INSN (insn
);
4194 if (set
!= 0 && REG_P (SET_DEST (set
))
4195 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
4197 struct no_conflict_data data
;
4199 data
.target
= const0_rtx
;
4203 note_stores (insn
, no_conflict_move_test
, &data
);
4204 if (! data
.must_stay
)
4206 if (PREV_INSN (insn
))
4207 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
4212 SET_PREV_INSN (next
) = PREV_INSN (insn
);
4218 /* Some ports use a loop to copy large arguments onto the stack.
4219 Don't move anything outside such a loop. */
4224 /* Write the remaining insns followed by the final copy. */
4225 for (insn
= insns
; insn
; insn
= next
)
4227 next
= NEXT_INSN (insn
);
4232 last
= emit_move_insn (target
, result
);
4234 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
4236 if (final_dest
!= target
)
4237 emit_move_insn (final_dest
, target
);
4241 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
4243 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
4246 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
4247 PURPOSE describes how this comparison will be used. CODE is the rtx
4248 comparison code we will be using.
4250 ??? Actually, CODE is slightly weaker than that. A target is still
4251 required to implement all of the normal bcc operations, but not
4252 required to implement all (or any) of the unordered bcc operations. */
4255 can_compare_p (enum rtx_code code
, machine_mode mode
,
4256 enum can_compare_purpose purpose
)
4259 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
4262 enum insn_code icode
;
4264 if (purpose
== ccp_jump
4265 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
4266 && insn_operand_matches (icode
, 0, test
))
4268 if (purpose
== ccp_store_flag
4269 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
4270 && insn_operand_matches (icode
, 1, test
))
4272 if (purpose
== ccp_cmov
4273 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
4276 mode
= GET_MODE_WIDER_MODE (mode
).else_void ();
4277 PUT_MODE (test
, mode
);
4279 while (mode
!= VOIDmode
);
4284 /* Return whether RTL code CODE corresponds to an unsigned optab. */
4287 unsigned_optab_p (enum rtx_code code
)
4289 return code
== LTU
|| code
== LEU
|| code
== GTU
|| code
== GEU
;
4292 /* Return whether the backend-emitted comparison for code CODE, comparing
4293 operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4294 operand OPNO of pattern ICODE. */
4297 insn_predicate_matches_p (enum insn_code icode
, unsigned int opno
,
4298 enum rtx_code code
, machine_mode mask_mode
,
4299 machine_mode value_mode
)
4301 rtx reg1
= alloca_raw_REG (value_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4302 rtx reg2
= alloca_raw_REG (value_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4303 rtx test
= alloca_rtx_fmt_ee (code
, mask_mode
, reg1
, reg2
);
4304 return insn_operand_matches (icode
, opno
, test
);
4307 /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4308 for code CODE, comparing operands of mode VALUE_MODE and producing a result
4312 can_vec_cmp_compare_p (enum rtx_code code
, machine_mode value_mode
,
4313 machine_mode mask_mode
)
4315 enum insn_code icode
4316 = get_vec_cmp_icode (value_mode
, mask_mode
, unsigned_optab_p (code
));
4317 if (icode
== CODE_FOR_nothing
)
4320 return insn_predicate_matches_p (icode
, 1, code
, mask_mode
, value_mode
);
4323 /* Return whether the backend can emit a vector comparison (vcond/vcondu) for
4324 code CODE, comparing operands of mode CMP_OP_MODE and producing a result
4328 can_vcond_compare_p (enum rtx_code code
, machine_mode value_mode
,
4329 machine_mode cmp_op_mode
)
4331 enum insn_code icode
4332 = get_vcond_icode (value_mode
, cmp_op_mode
, unsigned_optab_p (code
));
4333 if (icode
== CODE_FOR_nothing
)
4336 return insn_predicate_matches_p (icode
, 3, code
, value_mode
, cmp_op_mode
);
4339 /* Return whether the backend can emit vector set instructions for inserting
4340 element into vector at variable index position. */
4343 can_vec_set_var_idx_p (machine_mode vec_mode
)
4345 if (!VECTOR_MODE_P (vec_mode
))
4348 machine_mode inner_mode
= GET_MODE_INNER (vec_mode
);
4349 rtx reg1
= alloca_raw_REG (vec_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4350 rtx reg2
= alloca_raw_REG (inner_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4351 rtx reg3
= alloca_raw_REG (VOIDmode
, LAST_VIRTUAL_REGISTER
+ 3);
4353 enum insn_code icode
= optab_handler (vec_set_optab
, vec_mode
);
4355 return icode
!= CODE_FOR_nothing
&& insn_operand_matches (icode
, 0, reg1
)
4356 && insn_operand_matches (icode
, 1, reg2
)
4357 && insn_operand_matches (icode
, 2, reg3
);
4360 /* This function is called when we are going to emit a compare instruction that
4361 compares the values found in X and Y, using the rtl operator COMPARISON.
4363 If they have mode BLKmode, then SIZE specifies the size of both operands.
4365 UNSIGNEDP nonzero says that the operands are unsigned;
4366 this matters if they need to be widened (as given by METHODS).
4368 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4369 if we failed to produce one.
4371 *PMODE is the mode of the inputs (in case they are const_int).
4373 This function performs all the setup necessary so that the caller only has
4374 to emit a single comparison insn. This setup can involve doing a BLKmode
4375 comparison or emitting a library call to perform the comparison if no insn
4376 is available to handle it.
4377 The values which are passed in through pointers can be modified; the caller
4378 should perform the comparison on the modified values. Constant
4379 comparisons must have already been folded. */
4382 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4383 int unsignedp
, enum optab_methods methods
,
4384 rtx
*ptest
, machine_mode
*pmode
)
4386 machine_mode mode
= *pmode
;
4388 machine_mode cmp_mode
;
4389 enum mode_class mclass
;
4391 /* The other methods are not needed. */
4392 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
4393 || methods
== OPTAB_LIB_WIDEN
);
4395 if (CONST_SCALAR_INT_P (y
))
4396 canonicalize_comparison (mode
, &comparison
, &y
);
4398 /* If we are optimizing, force expensive constants into a register. */
4399 if (CONSTANT_P (x
) && optimize
4400 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
4401 > COSTS_N_INSNS (1))
4402 && can_create_pseudo_p ())
4403 x
= force_reg (mode
, x
);
4405 if (CONSTANT_P (y
) && optimize
4406 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
4407 > COSTS_N_INSNS (1))
4408 && can_create_pseudo_p ())
4409 y
= force_reg (mode
, y
);
4411 /* Don't let both operands fail to indicate the mode. */
4412 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
4413 x
= force_reg (mode
, x
);
4414 if (mode
== VOIDmode
)
4415 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
4417 /* Handle all BLKmode compares. */
4419 if (mode
== BLKmode
)
4421 machine_mode result_mode
;
4422 enum insn_code cmp_code
;
4425 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
4429 /* Try to use a memory block compare insn - either cmpstr
4430 or cmpmem will do. */
4431 opt_scalar_int_mode cmp_mode_iter
;
4432 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
4434 scalar_int_mode cmp_mode
= cmp_mode_iter
.require ();
4435 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
4436 if (cmp_code
== CODE_FOR_nothing
)
4437 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
4438 if (cmp_code
== CODE_FOR_nothing
)
4439 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
4440 if (cmp_code
== CODE_FOR_nothing
)
4443 /* Must make sure the size fits the insn's mode. */
4444 if (CONST_INT_P (size
)
4445 ? UINTVAL (size
) > GET_MODE_MASK (cmp_mode
)
4446 : (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (size
)))
4447 > GET_MODE_BITSIZE (cmp_mode
)))
4450 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
4451 result
= gen_reg_rtx (result_mode
);
4452 size
= convert_to_mode (cmp_mode
, size
, 1);
4453 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
4455 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
4456 *pmode
= result_mode
;
4460 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
4463 /* Otherwise call a library function. */
4464 result
= emit_block_comp_via_libcall (x
, y
, size
);
4468 mode
= TYPE_MODE (integer_type_node
);
4469 methods
= OPTAB_LIB_WIDEN
;
4473 /* Don't allow operands to the compare to trap, as that can put the
4474 compare and branch in different basic blocks. */
4475 if (cfun
->can_throw_non_call_exceptions
)
4477 if (!can_create_pseudo_p () && (may_trap_p (x
) || may_trap_p (y
)))
4480 x
= copy_to_reg (x
);
4482 y
= copy_to_reg (y
);
4485 if (GET_MODE_CLASS (mode
) == MODE_CC
)
4487 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
4488 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4489 gcc_assert (icode
!= CODE_FOR_nothing
4490 && insn_operand_matches (icode
, 0, test
));
4495 mclass
= GET_MODE_CLASS (mode
);
4496 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4497 FOR_EACH_MODE_FROM (cmp_mode
, mode
)
4499 enum insn_code icode
;
4500 icode
= optab_handler (cbranch_optab
, cmp_mode
);
4501 if (icode
!= CODE_FOR_nothing
4502 && insn_operand_matches (icode
, 0, test
))
4504 rtx_insn
*last
= get_last_insn ();
4505 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
4506 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
4508 && insn_operand_matches (icode
, 1, op0
)
4509 && insn_operand_matches (icode
, 2, op1
))
4511 XEXP (test
, 0) = op0
;
4512 XEXP (test
, 1) = op1
;
4517 delete_insns_since (last
);
4520 if (methods
== OPTAB_DIRECT
|| !CLASS_HAS_WIDER_MODES_P (mclass
))
4524 if (methods
!= OPTAB_LIB_WIDEN
)
4527 if (SCALAR_FLOAT_MODE_P (mode
))
4529 /* Small trick if UNORDERED isn't implemented by the hardware. */
4530 if (comparison
== UNORDERED
&& rtx_equal_p (x
, y
))
4532 prepare_cmp_insn (x
, y
, UNLT
, NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4538 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
4543 machine_mode ret_mode
;
4545 /* Handle a libcall just for the mode we are using. */
4546 libfunc
= optab_libfunc (cmp_optab
, mode
);
4547 gcc_assert (libfunc
);
4549 /* If we want unsigned, and this mode has a distinct unsigned
4550 comparison routine, use that. */
4553 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
4558 ret_mode
= targetm
.libgcc_cmp_return_mode ();
4559 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4560 ret_mode
, x
, mode
, y
, mode
);
4562 /* There are two kinds of comparison routines. Biased routines
4563 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4564 of gcc expect that the comparison operation is equivalent
4565 to the modified comparison. For signed comparisons compare the
4566 result against 1 in the biased case, and zero in the unbiased
4567 case. For unsigned comparisons always compare against 1 after
4568 biasing the unbiased result by adding 1. This gives us a way to
4570 The comparisons in the fixed-point helper library are always
4575 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
4578 x
= plus_constant (ret_mode
, result
, 1);
4584 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
4594 /* Before emitting an insn with code ICODE, make sure that X, which is going
4595 to be used for operand OPNUM of the insn, is converted from mode MODE to
4596 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4597 that it is accepted by the operand predicate. Return the new value. */
4600 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
4601 machine_mode wider_mode
, int unsignedp
)
4603 if (mode
!= wider_mode
)
4604 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
4606 if (!insn_operand_matches (icode
, opnum
, x
))
4608 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
4609 if (reload_completed
)
4611 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
4613 x
= copy_to_mode_reg (op_mode
, x
);
4619 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4620 we can do the branch. */
4623 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
4624 profile_probability prob
)
4626 machine_mode optab_mode
;
4627 enum mode_class mclass
;
4628 enum insn_code icode
;
4631 mclass
= GET_MODE_CLASS (mode
);
4632 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
4633 icode
= optab_handler (cbranch_optab
, optab_mode
);
4635 gcc_assert (icode
!= CODE_FOR_nothing
);
4636 gcc_assert (insn_operand_matches (icode
, 0, test
));
4637 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
4638 XEXP (test
, 1), label
));
4639 if (prob
.initialized_p ()
4640 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4643 && any_condjump_p (insn
)
4644 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4645 add_reg_br_prob_note (insn
, prob
);
4648 /* Generate code to compare X with Y so that the condition codes are
4649 set and to jump to LABEL if the condition is true. If X is a
4650 constant and Y is not a constant, then the comparison is swapped to
4651 ensure that the comparison RTL has the canonical form.
4653 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4654 need to be widened. UNSIGNEDP is also used to select the proper
4655 branch condition code.
4657 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4659 MODE is the mode of the inputs (in case they are const_int).
4661 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4662 It will be potentially converted into an unsigned variant based on
4663 UNSIGNEDP to select a proper jump instruction.
4665 PROB is the probability of jumping to LABEL. */
4668 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4669 machine_mode mode
, int unsignedp
, rtx label
,
4670 profile_probability prob
)
4672 rtx op0
= x
, op1
= y
;
4675 /* Swap operands and condition to ensure canonical RTL. */
4676 if (swap_commutative_operands_p (x
, y
)
4677 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4680 comparison
= swap_condition (comparison
);
4683 /* If OP0 is still a constant, then both X and Y must be constants
4684 or the opposite comparison is not supported. Force X into a register
4685 to create canonical RTL. */
4686 if (CONSTANT_P (op0
))
4687 op0
= force_reg (mode
, op0
);
4690 comparison
= unsigned_condition (comparison
);
4692 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4694 emit_cmp_and_jump_insn_1 (test
, mode
, label
, prob
);
4698 /* Emit a library call comparison between floating point X and Y.
4699 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4702 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4703 rtx
*ptest
, machine_mode
*pmode
)
4705 enum rtx_code swapped
= swap_condition (comparison
);
4706 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4707 machine_mode orig_mode
= GET_MODE (x
);
4709 rtx true_rtx
, false_rtx
;
4710 rtx value
, target
, equiv
;
4713 bool reversed_p
= false;
4714 scalar_int_mode cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4716 FOR_EACH_MODE_FROM (mode
, orig_mode
)
4718 if (code_to_optab (comparison
)
4719 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4722 if (code_to_optab (swapped
)
4723 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4726 comparison
= swapped
;
4730 if (code_to_optab (reversed
)
4731 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4733 comparison
= reversed
;
4739 gcc_assert (mode
!= VOIDmode
);
4741 if (mode
!= orig_mode
)
4743 x
= convert_to_mode (mode
, x
, 0);
4744 y
= convert_to_mode (mode
, y
, 0);
4747 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4748 the RTL. The allows the RTL optimizers to delete the libcall if the
4749 condition can be determined at compile-time. */
4750 if (comparison
== UNORDERED
4751 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4753 true_rtx
= const_true_rtx
;
4754 false_rtx
= const0_rtx
;
4761 true_rtx
= const0_rtx
;
4762 false_rtx
= const_true_rtx
;
4766 true_rtx
= const_true_rtx
;
4767 false_rtx
= const0_rtx
;
4771 true_rtx
= const1_rtx
;
4772 false_rtx
= const0_rtx
;
4776 true_rtx
= const0_rtx
;
4777 false_rtx
= constm1_rtx
;
4781 true_rtx
= constm1_rtx
;
4782 false_rtx
= const0_rtx
;
4786 true_rtx
= const0_rtx
;
4787 false_rtx
= const1_rtx
;
4795 if (comparison
== UNORDERED
)
4797 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4798 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4799 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4800 temp
, const_true_rtx
, equiv
);
4804 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4805 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4806 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4807 equiv
, true_rtx
, false_rtx
);
4811 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4812 cmp_mode
, x
, mode
, y
, mode
);
4813 insns
= get_insns ();
4816 target
= gen_reg_rtx (cmp_mode
);
4817 emit_libcall_block (insns
, target
, value
, equiv
);
4819 if (comparison
== UNORDERED
4820 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
4822 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
4824 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
4829 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4832 emit_indirect_jump (rtx loc
)
4834 if (!targetm
.have_indirect_jump ())
4835 sorry ("indirect jumps are not available on this target");
4838 class expand_operand ops
[1];
4839 create_address_operand (&ops
[0], loc
);
4840 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
4846 /* Emit a conditional move instruction if the machine supports one for that
4847 condition and machine mode.
4849 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4850 the mode to use should they be constants. If it is VOIDmode, they cannot
4853 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4854 should be stored there. MODE is the mode to use should they be constants.
4855 If it is VOIDmode, they cannot both be constants.
4857 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4858 is not supported. */
4861 emit_conditional_move (rtx target
, struct rtx_comparison comp
,
4863 machine_mode mode
, int unsignedp
)
4867 enum insn_code icode
;
4868 enum rtx_code reversed
;
4870 /* If the two source operands are identical, that's just a move. */
4872 if (rtx_equal_p (op2
, op3
))
4875 target
= gen_reg_rtx (mode
);
4877 emit_move_insn (target
, op3
);
4881 /* If one operand is constant, make it the second one. Only do this
4882 if the other operand is not constant as well. */
4884 if (swap_commutative_operands_p (comp
.op0
, comp
.op1
))
4886 std::swap (comp
.op0
, comp
.op1
);
4887 comp
.code
= swap_condition (comp
.code
);
4890 /* get_condition will prefer to generate LT and GT even if the old
4891 comparison was against zero, so undo that canonicalization here since
4892 comparisons against zero are cheaper. */
4894 if (comp
.code
== LT
&& comp
.op1
== const1_rtx
)
4895 comp
.code
= LE
, comp
.op1
= const0_rtx
;
4896 else if (comp
.code
== GT
&& comp
.op1
== constm1_rtx
)
4897 comp
.code
= GE
, comp
.op1
= const0_rtx
;
4899 if (comp
.mode
== VOIDmode
)
4900 comp
.mode
= GET_MODE (comp
.op0
);
4902 enum rtx_code orig_code
= comp
.code
;
4903 bool swapped
= false;
4904 if (swap_commutative_operands_p (op2
, op3
)
4906 reversed_comparison_code_parts (comp
.code
, comp
.op0
, comp
.op1
, NULL
))
4909 std::swap (op2
, op3
);
4910 comp
.code
= reversed
;
4914 if (mode
== VOIDmode
)
4915 mode
= GET_MODE (op2
);
4917 icode
= direct_optab_handler (movcc_optab
, mode
);
4919 if (icode
== CODE_FOR_nothing
)
4923 target
= gen_reg_rtx (mode
);
4925 for (int pass
= 0; ; pass
++)
4927 comp
.code
= unsignedp
? unsigned_condition (comp
.code
) : comp
.code
;
4929 simplify_gen_relational (comp
.code
, VOIDmode
,
4930 comp
.mode
, comp
.op0
, comp
.op1
);
4932 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4933 punt and let the caller figure out how best to deal with this
4935 if (COMPARISON_P (comparison
))
4937 saved_pending_stack_adjust save
;
4938 save_pending_stack_adjust (&save
);
4939 last
= get_last_insn ();
4940 do_pending_stack_adjust ();
4941 machine_mode cmpmode
= comp
.mode
;
4942 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4943 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
4944 OPTAB_WIDEN
, &comparison
, &cmpmode
);
4947 rtx res
= emit_conditional_move_1 (target
, comparison
,
4949 if (res
!= NULL_RTX
)
4952 delete_insns_since (last
);
4953 restore_pending_stack_adjust (&save
);
4959 /* If the preferred op2/op3 order is not usable, retry with other
4960 operand order, perhaps it will expand successfully. */
4962 comp
.code
= orig_code
;
4963 else if ((reversed
=
4964 reversed_comparison_code_parts (orig_code
, comp
.op0
, comp
.op1
,
4967 comp
.code
= reversed
;
4970 std::swap (op2
, op3
);
4974 /* Helper function that, in addition to COMPARISON, also tries
4975 the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
4976 to when we pass the specific constituents of a comparison, no
4977 additional insns are emitted for it. It might still be necessary
4978 to emit more than one insn for the final conditional move, though. */
4981 emit_conditional_move (rtx target
, rtx comparison
, rtx rev_comparison
,
4982 rtx op2
, rtx op3
, machine_mode mode
)
4984 rtx res
= emit_conditional_move_1 (target
, comparison
, op2
, op3
, mode
);
4986 if (res
!= NULL_RTX
)
4989 return emit_conditional_move_1 (target
, rev_comparison
, op3
, op2
, mode
);
4992 /* Helper for emitting a conditional move. */
4995 emit_conditional_move_1 (rtx target
, rtx comparison
,
4996 rtx op2
, rtx op3
, machine_mode mode
)
4998 enum insn_code icode
;
5000 if (comparison
== NULL_RTX
|| !COMPARISON_P (comparison
))
5003 /* If the two source operands are identical, that's just a move.
5004 As the comparison comes in non-canonicalized, we must make
5005 sure not to discard any possible side effects. If there are
5006 side effects, just let the target handle it. */
5007 if (!side_effects_p (comparison
) && rtx_equal_p (op2
, op3
))
5010 target
= gen_reg_rtx (mode
);
5012 emit_move_insn (target
, op3
);
5016 if (mode
== VOIDmode
)
5017 mode
= GET_MODE (op2
);
5019 icode
= direct_optab_handler (movcc_optab
, mode
);
5021 if (icode
== CODE_FOR_nothing
)
5025 target
= gen_reg_rtx (mode
);
5027 class expand_operand ops
[4];
5029 create_output_operand (&ops
[0], target
, mode
);
5030 create_fixed_operand (&ops
[1], comparison
);
5031 create_input_operand (&ops
[2], op2
, mode
);
5032 create_input_operand (&ops
[3], op3
, mode
);
5034 if (maybe_expand_insn (icode
, 4, ops
))
5036 if (ops
[0].value
!= target
)
5037 convert_move (target
, ops
[0].value
, false);
5045 /* Emit a conditional negate or bitwise complement using the
5046 negcc or notcc optabs if available. Return NULL_RTX if such operations
5047 are not available. Otherwise return the RTX holding the result.
5048 TARGET is the desired destination of the result. COMP is the comparison
5049 on which to negate. If COND is true move into TARGET the negation
5050 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5051 CODE is either NEG or NOT. MODE is the machine mode in which the
5052 operation is performed. */
5055 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
5056 machine_mode mode
, rtx cond
, rtx op1
,
5059 optab op
= unknown_optab
;
5062 else if (code
== NOT
)
5067 insn_code icode
= direct_optab_handler (op
, mode
);
5069 if (icode
== CODE_FOR_nothing
)
5073 target
= gen_reg_rtx (mode
);
5075 rtx_insn
*last
= get_last_insn ();
5076 class expand_operand ops
[4];
5078 create_output_operand (&ops
[0], target
, mode
);
5079 create_fixed_operand (&ops
[1], cond
);
5080 create_input_operand (&ops
[2], op1
, mode
);
5081 create_input_operand (&ops
[3], op2
, mode
);
5083 if (maybe_expand_insn (icode
, 4, ops
))
5085 if (ops
[0].value
!= target
)
5086 convert_move (target
, ops
[0].value
, false);
5090 delete_insns_since (last
);
5094 /* Emit a conditional addition instruction if the machine supports one for that
5095 condition and machine mode.
5097 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5098 the mode to use should they be constants. If it is VOIDmode, they cannot
5101 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5102 should be stored there. MODE is the mode to use should they be constants.
5103 If it is VOIDmode, they cannot both be constants.
5105 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5106 is not supported. */
5109 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
5110 machine_mode cmode
, rtx op2
, rtx op3
,
5111 machine_mode mode
, int unsignedp
)
5115 enum insn_code icode
;
5117 /* If one operand is constant, make it the second one. Only do this
5118 if the other operand is not constant as well. */
5120 if (swap_commutative_operands_p (op0
, op1
))
5122 std::swap (op0
, op1
);
5123 code
= swap_condition (code
);
5126 /* get_condition will prefer to generate LT and GT even if the old
5127 comparison was against zero, so undo that canonicalization here since
5128 comparisons against zero are cheaper. */
5129 if (code
== LT
&& op1
== const1_rtx
)
5130 code
= LE
, op1
= const0_rtx
;
5131 else if (code
== GT
&& op1
== constm1_rtx
)
5132 code
= GE
, op1
= const0_rtx
;
5134 if (cmode
== VOIDmode
)
5135 cmode
= GET_MODE (op0
);
5137 if (mode
== VOIDmode
)
5138 mode
= GET_MODE (op2
);
5140 icode
= optab_handler (addcc_optab
, mode
);
5142 if (icode
== CODE_FOR_nothing
)
5146 target
= gen_reg_rtx (mode
);
5148 code
= unsignedp
? unsigned_condition (code
) : code
;
5149 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
5151 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5152 return NULL and let the caller figure out how best to deal with this
5154 if (!COMPARISON_P (comparison
))
5157 do_pending_stack_adjust ();
5158 last
= get_last_insn ();
5159 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
5160 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
5161 &comparison
, &cmode
);
5164 class expand_operand ops
[4];
5166 create_output_operand (&ops
[0], target
, mode
);
5167 create_fixed_operand (&ops
[1], comparison
);
5168 create_input_operand (&ops
[2], op2
, mode
);
5169 create_input_operand (&ops
[3], op3
, mode
);
5170 if (maybe_expand_insn (icode
, 4, ops
))
5172 if (ops
[0].value
!= target
)
5173 convert_move (target
, ops
[0].value
, false);
5177 delete_insns_since (last
);
5181 /* These functions attempt to generate an insn body, rather than
5182 emitting the insn, but if the gen function already emits them, we
5183 make no attempt to turn them back into naked patterns. */
5185 /* Generate and return an insn body to add Y to X. */
5188 gen_add2_insn (rtx x
, rtx y
)
5190 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
5192 gcc_assert (insn_operand_matches (icode
, 0, x
));
5193 gcc_assert (insn_operand_matches (icode
, 1, x
));
5194 gcc_assert (insn_operand_matches (icode
, 2, y
));
5196 return GEN_FCN (icode
) (x
, x
, y
);
5199 /* Generate and return an insn body to add r1 and c,
5200 storing the result in r0. */
5203 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
5205 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
5207 if (icode
== CODE_FOR_nothing
5208 || !insn_operand_matches (icode
, 0, r0
)
5209 || !insn_operand_matches (icode
, 1, r1
)
5210 || !insn_operand_matches (icode
, 2, c
))
5213 return GEN_FCN (icode
) (r0
, r1
, c
);
5217 have_add2_insn (rtx x
, rtx y
)
5219 enum insn_code icode
;
5221 gcc_assert (GET_MODE (x
) != VOIDmode
);
5223 icode
= optab_handler (add_optab
, GET_MODE (x
));
5225 if (icode
== CODE_FOR_nothing
)
5228 if (!insn_operand_matches (icode
, 0, x
)
5229 || !insn_operand_matches (icode
, 1, x
)
5230 || !insn_operand_matches (icode
, 2, y
))
5236 /* Generate and return an insn body to add Y to X. */
5239 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
5241 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
5243 gcc_assert (insn_operand_matches (icode
, 0, x
));
5244 gcc_assert (insn_operand_matches (icode
, 1, y
));
5245 gcc_assert (insn_operand_matches (icode
, 2, z
));
5247 return GEN_FCN (icode
) (x
, y
, z
);
5250 /* Return true if the target implements an addptr pattern and X, Y,
5251 and Z are valid for the pattern predicates. */
5254 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
5256 enum insn_code icode
;
5258 gcc_assert (GET_MODE (x
) != VOIDmode
);
5260 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
5262 if (icode
== CODE_FOR_nothing
)
5265 if (!insn_operand_matches (icode
, 0, x
)
5266 || !insn_operand_matches (icode
, 1, y
)
5267 || !insn_operand_matches (icode
, 2, z
))
5273 /* Generate and return an insn body to subtract Y from X. */
5276 gen_sub2_insn (rtx x
, rtx y
)
5278 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
5280 gcc_assert (insn_operand_matches (icode
, 0, x
));
5281 gcc_assert (insn_operand_matches (icode
, 1, x
));
5282 gcc_assert (insn_operand_matches (icode
, 2, y
));
5284 return GEN_FCN (icode
) (x
, x
, y
);
5287 /* Generate and return an insn body to subtract r1 and c,
5288 storing the result in r0. */
5291 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
5293 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
5295 if (icode
== CODE_FOR_nothing
5296 || !insn_operand_matches (icode
, 0, r0
)
5297 || !insn_operand_matches (icode
, 1, r1
)
5298 || !insn_operand_matches (icode
, 2, c
))
5301 return GEN_FCN (icode
) (r0
, r1
, c
);
5305 have_sub2_insn (rtx x
, rtx y
)
5307 enum insn_code icode
;
5309 gcc_assert (GET_MODE (x
) != VOIDmode
);
5311 icode
= optab_handler (sub_optab
, GET_MODE (x
));
5313 if (icode
== CODE_FOR_nothing
)
5316 if (!insn_operand_matches (icode
, 0, x
)
5317 || !insn_operand_matches (icode
, 1, x
)
5318 || !insn_operand_matches (icode
, 2, y
))
5324 /* Generate the body of an insn to extend Y (with mode MFROM)
5325 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5328 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
5329 machine_mode mfrom
, int unsignedp
)
5331 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
5332 return GEN_FCN (icode
) (x
, y
);
5335 /* Generate code to convert FROM to floating point
5336 and store in TO. FROM must be fixed point and not VOIDmode.
5337 UNSIGNEDP nonzero means regard FROM as unsigned.
5338 Normally this is done by correcting the final value
5339 if it is negative. */
5342 expand_float (rtx to
, rtx from
, int unsignedp
)
5344 enum insn_code icode
;
5346 scalar_mode from_mode
, to_mode
;
5347 machine_mode fmode
, imode
;
5348 bool can_do_signed
= false;
5350 /* Crash now, because we won't be able to decide which mode to use. */
5351 gcc_assert (GET_MODE (from
) != VOIDmode
);
5353 /* Look for an insn to do the conversion. Do it in the specified
5354 modes if possible; otherwise convert either input, output or both to
5355 wider mode. If the integer mode is wider than the mode of FROM,
5356 we can do the conversion signed even if the input is unsigned. */
5358 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
5359 FOR_EACH_MODE_FROM (imode
, GET_MODE (from
))
5361 int doing_unsigned
= unsignedp
;
5363 if (fmode
!= GET_MODE (to
)
5364 && (significand_size (fmode
)
5365 < GET_MODE_UNIT_PRECISION (GET_MODE (from
))))
5368 icode
= can_float_p (fmode
, imode
, unsignedp
);
5369 if (icode
== CODE_FOR_nothing
&& unsignedp
)
5371 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
5372 if (scode
!= CODE_FOR_nothing
)
5373 can_do_signed
= true;
5374 if (imode
!= GET_MODE (from
))
5375 icode
= scode
, doing_unsigned
= 0;
5378 if (icode
!= CODE_FOR_nothing
)
5380 if (imode
!= GET_MODE (from
))
5381 from
= convert_to_mode (imode
, from
, unsignedp
);
5383 if (fmode
!= GET_MODE (to
))
5384 target
= gen_reg_rtx (fmode
);
5386 emit_unop_insn (icode
, target
, from
,
5387 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
5390 convert_move (to
, target
, 0);
5395 /* Unsigned integer, and no way to convert directly. Convert as signed,
5396 then unconditionally adjust the result. */
5399 && is_a
<scalar_mode
> (GET_MODE (to
), &to_mode
)
5400 && is_a
<scalar_mode
> (GET_MODE (from
), &from_mode
))
5402 opt_scalar_mode fmode_iter
;
5403 rtx_code_label
*label
= gen_label_rtx ();
5405 REAL_VALUE_TYPE offset
;
5407 /* Look for a usable floating mode FMODE wider than the source and at
5408 least as wide as the target. Using FMODE will avoid rounding woes
5409 with unsigned values greater than the signed maximum value. */
5411 FOR_EACH_MODE_FROM (fmode_iter
, to_mode
)
5413 scalar_mode fmode
= fmode_iter
.require ();
5414 if (GET_MODE_PRECISION (from_mode
) < GET_MODE_BITSIZE (fmode
)
5415 && can_float_p (fmode
, from_mode
, 0) != CODE_FOR_nothing
)
5419 if (!fmode_iter
.exists (&fmode
))
5421 /* There is no such mode. Pretend the target is wide enough. */
5424 /* Avoid double-rounding when TO is narrower than FROM. */
5425 if ((significand_size (fmode
) + 1)
5426 < GET_MODE_PRECISION (from_mode
))
5429 rtx_code_label
*neglabel
= gen_label_rtx ();
5431 /* Don't use TARGET if it isn't a register, is a hard register,
5432 or is the wrong mode. */
5434 || REGNO (target
) < FIRST_PSEUDO_REGISTER
5435 || GET_MODE (target
) != fmode
)
5436 target
= gen_reg_rtx (fmode
);
5439 do_pending_stack_adjust ();
5441 /* Test whether the sign bit is set. */
5442 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
5445 /* The sign bit is not set. Convert as signed. */
5446 expand_float (target
, from
, 0);
5447 emit_jump_insn (targetm
.gen_jump (label
));
5450 /* The sign bit is set.
5451 Convert to a usable (positive signed) value by shifting right
5452 one bit, while remembering if a nonzero bit was shifted
5453 out; i.e., compute (from & 1) | (from >> 1). */
5455 emit_label (neglabel
);
5456 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
5457 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
5458 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
5459 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
5461 expand_float (target
, temp
, 0);
5463 /* Multiply by 2 to undo the shift above. */
5464 temp
= expand_binop (fmode
, add_optab
, target
, target
,
5465 target
, 0, OPTAB_LIB_WIDEN
);
5467 emit_move_insn (target
, temp
);
5469 do_pending_stack_adjust ();
5475 /* If we are about to do some arithmetic to correct for an
5476 unsigned operand, do it in a pseudo-register. */
5478 if (to_mode
!= fmode
5479 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
5480 target
= gen_reg_rtx (fmode
);
5482 /* Convert as signed integer to floating. */
5483 expand_float (target
, from
, 0);
5485 /* If FROM is negative (and therefore TO is negative),
5486 correct its value by 2**bitwidth. */
5488 do_pending_stack_adjust ();
5489 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, from_mode
,
5493 real_2expN (&offset
, GET_MODE_PRECISION (from_mode
), fmode
);
5494 temp
= expand_binop (fmode
, add_optab
, target
,
5495 const_double_from_real_value (offset
, fmode
),
5496 target
, 0, OPTAB_LIB_WIDEN
);
5498 emit_move_insn (target
, temp
);
5500 do_pending_stack_adjust ();
5505 /* No hardware instruction available; call a library routine. */
5510 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
5512 if (is_narrower_int_mode (GET_MODE (from
), SImode
))
5513 from
= convert_to_mode (SImode
, from
, unsignedp
);
5515 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5516 gcc_assert (libfunc
);
5520 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5521 GET_MODE (to
), from
, GET_MODE (from
));
5522 insns
= get_insns ();
5525 emit_libcall_block (insns
, target
, value
,
5526 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
5527 GET_MODE (to
), from
));
5532 /* Copy result to requested destination
5533 if we have been computing in a temp location. */
5537 if (GET_MODE (target
) == GET_MODE (to
))
5538 emit_move_insn (to
, target
);
5540 convert_move (to
, target
, 0);
5544 /* Generate code to convert FROM to fixed point and store in TO. FROM
5545 must be floating point. */
5548 expand_fix (rtx to
, rtx from
, int unsignedp
)
5550 enum insn_code icode
;
5552 machine_mode fmode
, imode
;
5553 opt_scalar_mode fmode_iter
;
5554 bool must_trunc
= false;
5556 /* We first try to find a pair of modes, one real and one integer, at
5557 least as wide as FROM and TO, respectively, in which we can open-code
5558 this conversion. If the integer mode is wider than the mode of TO,
5559 we can do the conversion either signed or unsigned. */
5561 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5562 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5564 int doing_unsigned
= unsignedp
;
5566 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
5567 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
5568 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
5570 if (icode
!= CODE_FOR_nothing
)
5572 rtx_insn
*last
= get_last_insn ();
5574 if (fmode
!= GET_MODE (from
))
5575 from1
= convert_to_mode (fmode
, from
, 0);
5579 rtx temp
= gen_reg_rtx (GET_MODE (from1
));
5580 from1
= expand_unop (GET_MODE (from1
), ftrunc_optab
, from1
,
5584 if (imode
!= GET_MODE (to
))
5585 target
= gen_reg_rtx (imode
);
5587 if (maybe_emit_unop_insn (icode
, target
, from1
,
5588 doing_unsigned
? UNSIGNED_FIX
: FIX
))
5591 convert_move (to
, target
, unsignedp
);
5594 delete_insns_since (last
);
5598 /* For an unsigned conversion, there is one more way to do it.
5599 If we have a signed conversion, we generate code that compares
5600 the real value to the largest representable positive number. If if
5601 is smaller, the conversion is done normally. Otherwise, subtract
5602 one plus the highest signed number, convert, and add it back.
5604 We only need to check all real modes, since we know we didn't find
5605 anything with a wider integer mode.
5607 This code used to extend FP value into mode wider than the destination.
5608 This is needed for decimal float modes which cannot accurately
5609 represent one plus the highest signed number of the same size, but
5610 not for binary modes. Consider, for instance conversion from SFmode
5613 The hot path through the code is dealing with inputs smaller than 2^63
5614 and doing just the conversion, so there is no bits to lose.
5616 In the other path we know the value is positive in the range 2^63..2^64-1
5617 inclusive. (as for other input overflow happens and result is undefined)
5618 So we know that the most important bit set in mantissa corresponds to
5619 2^63. The subtraction of 2^63 should not generate any rounding as it
5620 simply clears out that bit. The rest is trivial. */
5622 scalar_int_mode to_mode
;
5624 && is_a
<scalar_int_mode
> (GET_MODE (to
), &to_mode
)
5625 && HWI_COMPUTABLE_MODE_P (to_mode
))
5626 FOR_EACH_MODE_FROM (fmode_iter
, as_a
<scalar_mode
> (GET_MODE (from
)))
5628 scalar_mode fmode
= fmode_iter
.require ();
5629 if (CODE_FOR_nothing
!= can_fix_p (to_mode
, fmode
,
5631 && (!DECIMAL_FLOAT_MODE_P (fmode
)
5632 || (GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (to_mode
))))
5635 REAL_VALUE_TYPE offset
;
5637 rtx_code_label
*lab1
, *lab2
;
5640 bitsize
= GET_MODE_PRECISION (to_mode
);
5641 real_2expN (&offset
, bitsize
- 1, fmode
);
5642 limit
= const_double_from_real_value (offset
, fmode
);
5643 lab1
= gen_label_rtx ();
5644 lab2
= gen_label_rtx ();
5646 if (fmode
!= GET_MODE (from
))
5647 from
= convert_to_mode (fmode
, from
, 0);
5649 /* See if we need to do the subtraction. */
5650 do_pending_stack_adjust ();
5651 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
,
5652 GET_MODE (from
), 0, lab1
);
5654 /* If not, do the signed "fix" and branch around fixup code. */
5655 expand_fix (to
, from
, 0);
5656 emit_jump_insn (targetm
.gen_jump (lab2
));
5659 /* Otherwise, subtract 2**(N-1), convert to signed number,
5660 then add 2**(N-1). Do the addition using XOR since this
5661 will often generate better code. */
5663 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
5664 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
5665 expand_fix (to
, target
, 0);
5666 target
= expand_binop (to_mode
, xor_optab
, to
,
5668 (HOST_WIDE_INT_1
<< (bitsize
- 1),
5670 to
, 1, OPTAB_LIB_WIDEN
);
5673 emit_move_insn (to
, target
);
5677 if (optab_handler (mov_optab
, to_mode
) != CODE_FOR_nothing
)
5679 /* Make a place for a REG_NOTE and add it. */
5680 insn
= emit_move_insn (to
, to
);
5681 set_dst_reg_note (insn
, REG_EQUAL
,
5682 gen_rtx_fmt_e (UNSIGNED_FIX
, to_mode
,
5691 /* We can't do it with an insn, so use a library call. But first ensure
5692 that the mode of TO is at least as wide as SImode, since those are the
5693 only library calls we know about. */
5695 if (is_narrower_int_mode (GET_MODE (to
), SImode
))
5697 target
= gen_reg_rtx (SImode
);
5699 expand_fix (target
, from
, unsignedp
);
5707 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
5708 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5709 gcc_assert (libfunc
);
5713 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5714 GET_MODE (to
), from
, GET_MODE (from
));
5715 insns
= get_insns ();
5718 emit_libcall_block (insns
, target
, value
,
5719 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5720 GET_MODE (to
), from
));
5725 if (GET_MODE (to
) == GET_MODE (target
))
5726 emit_move_insn (to
, target
);
5728 convert_move (to
, target
, 0);
5733 /* Promote integer arguments for a libcall if necessary.
5734 emit_library_call_value cannot do the promotion because it does not
5735 know if it should do a signed or unsigned promotion. This is because
5736 there are no tree types defined for libcalls. */
5739 prepare_libcall_arg (rtx arg
, int uintp
)
5741 scalar_int_mode mode
;
5742 machine_mode arg_mode
;
5743 if (is_a
<scalar_int_mode
> (GET_MODE (arg
), &mode
))
5745 /* If we need to promote the integer function argument we need to do
5746 it here instead of inside emit_library_call_value because in
5747 emit_library_call_value we don't know if we should do a signed or
5748 unsigned promotion. */
5751 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
5752 &unsigned_p
, NULL_TREE
, 0);
5753 if (arg_mode
!= mode
)
5754 return convert_to_mode (arg_mode
, arg
, uintp
);
5759 /* Generate code to convert FROM or TO a fixed-point.
5760 If UINTP is true, either TO or FROM is an unsigned integer.
5761 If SATP is true, we need to saturate the result. */
5764 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
5766 machine_mode to_mode
= GET_MODE (to
);
5767 machine_mode from_mode
= GET_MODE (from
);
5769 enum rtx_code this_code
;
5770 enum insn_code code
;
5775 if (to_mode
== from_mode
)
5777 emit_move_insn (to
, from
);
5783 tab
= satp
? satfractuns_optab
: fractuns_optab
;
5784 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
5788 tab
= satp
? satfract_optab
: fract_optab
;
5789 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
5791 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
5792 if (code
!= CODE_FOR_nothing
)
5794 emit_unop_insn (code
, to
, from
, this_code
);
5798 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
5799 gcc_assert (libfunc
);
5801 from
= prepare_libcall_arg (from
, uintp
);
5802 from_mode
= GET_MODE (from
);
5805 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
5807 insns
= get_insns ();
5810 emit_libcall_block (insns
, to
, value
,
5811 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
5814 /* Generate code to convert FROM to fixed point and store in TO. FROM
5815 must be floating point, TO must be signed. Use the conversion optab
5816 TAB to do the conversion. */
5819 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
5821 enum insn_code icode
;
5823 machine_mode fmode
, imode
;
5825 /* We first try to find a pair of modes, one real and one integer, at
5826 least as wide as FROM and TO, respectively, in which we can open-code
5827 this conversion. If the integer mode is wider than the mode of TO,
5828 we can do the conversion either signed or unsigned. */
5830 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5831 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5833 icode
= convert_optab_handler (tab
, imode
, fmode
);
5834 if (icode
!= CODE_FOR_nothing
)
5836 rtx_insn
*last
= get_last_insn ();
5837 if (fmode
!= GET_MODE (from
))
5838 from
= convert_to_mode (fmode
, from
, 0);
5840 if (imode
!= GET_MODE (to
))
5841 target
= gen_reg_rtx (imode
);
5843 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
5845 delete_insns_since (last
);
5849 convert_move (to
, target
, 0);
5857 /* Report whether we have an instruction to perform the operation
5858 specified by CODE on operands of mode MODE. */
5860 have_insn_for (enum rtx_code code
, machine_mode mode
)
5862 return (code_to_optab (code
)
5863 && (optab_handler (code_to_optab (code
), mode
)
5864 != CODE_FOR_nothing
));
5867 /* Print information about the current contents of the optabs on
5871 debug_optab_libfuncs (void)
5875 /* Dump the arithmetic optabs. */
5876 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
5877 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5879 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
5882 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5883 fprintf (stderr
, "%s\t%s:\t%s\n",
5884 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5890 /* Dump the conversion optabs. */
5891 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
5892 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5893 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
5895 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
5899 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5900 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
5901 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5909 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5910 CODE. Return 0 on failure. */
5913 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
5915 machine_mode mode
= GET_MODE (op1
);
5916 enum insn_code icode
;
5920 if (mode
== VOIDmode
)
5923 icode
= optab_handler (ctrap_optab
, mode
);
5924 if (icode
== CODE_FOR_nothing
)
5927 /* Some targets only accept a zero trap code. */
5928 if (!insn_operand_matches (icode
, 3, tcode
))
5931 do_pending_stack_adjust ();
5933 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
5938 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
5941 /* If that failed, then give up. */
5949 insn
= get_insns ();
5954 /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
5955 or unsigned operation code. */
5958 get_rtx_code_1 (enum tree_code tcode
, bool unsignedp
)
5970 code
= unsignedp
? LTU
: LT
;
5973 code
= unsignedp
? LEU
: LE
;
5976 code
= unsignedp
? GTU
: GT
;
5979 code
= unsignedp
? GEU
: GE
;
5982 case UNORDERED_EXPR
:
6022 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6023 or unsigned operation code. */
6026 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
6028 enum rtx_code code
= get_rtx_code_1 (tcode
, unsignedp
);
6029 gcc_assert (code
!= UNKNOWN
);
6033 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6034 select signed or unsigned operators. OPNO holds the index of the
6035 first comparison operand for insn ICODE. Do not generate the
6036 compare instruction itself. */
6039 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
6040 tree t_op0
, tree t_op1
, bool unsignedp
,
6041 enum insn_code icode
, unsigned int opno
)
6043 class expand_operand ops
[2];
6044 rtx rtx_op0
, rtx_op1
;
6045 machine_mode m0
, m1
;
6046 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
6048 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
6050 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6051 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6052 cases, use the original mode. */
6053 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
6055 m0
= GET_MODE (rtx_op0
);
6057 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
6059 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
6061 m1
= GET_MODE (rtx_op1
);
6063 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
6065 create_input_operand (&ops
[0], rtx_op0
, m0
);
6066 create_input_operand (&ops
[1], rtx_op1
, m1
);
6067 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
6069 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
6072 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
6073 the first vec_perm operand, assuming the second operand (for left shift
6074 first operand) is a constant vector of zeros. Return the shift distance
6075 in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6076 mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6077 shift or vec_shl_optab for left shift. */
6079 shift_amt_for_vec_perm_mask (machine_mode mode
, const vec_perm_indices
&sel
,
6082 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (mode
);
6083 poly_int64 first
= sel
[0];
6084 if (maybe_ge (sel
[0], GET_MODE_NUNITS (mode
)))
6087 if (shift_optab
== vec_shl_optab
)
6090 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
6092 unsigned firstidx
= 0;
6093 for (unsigned int i
= 0; i
< nelt
; i
++)
6095 if (known_eq (sel
[i
], nelt
))
6097 if (i
== 0 || firstidx
)
6102 ? maybe_ne (sel
[i
], nelt
+ i
- firstidx
)
6103 : maybe_ge (sel
[i
], nelt
))
6111 else if (!sel
.series_p (0, 1, first
, 1))
6114 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
6116 for (unsigned int i
= 1; i
< nelt
; i
++)
6118 poly_int64 expected
= i
+ first
;
6119 /* Indices into the second vector are all equivalent. */
6120 if (maybe_lt (sel
[i
], nelt
)
6121 ? maybe_ne (sel
[i
], expected
)
6122 : maybe_lt (expected
, nelt
))
6127 return gen_int_shift_amount (mode
, first
* bitsize
);
6130 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6133 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
6134 rtx v0
, rtx v1
, rtx sel
)
6136 machine_mode tmode
= GET_MODE (target
);
6137 machine_mode smode
= GET_MODE (sel
);
6138 class expand_operand ops
[4];
6140 gcc_assert (GET_MODE_CLASS (smode
) == MODE_VECTOR_INT
6141 || related_int_vector_mode (tmode
).require () == smode
);
6142 create_output_operand (&ops
[0], target
, tmode
);
6143 create_input_operand (&ops
[3], sel
, smode
);
6145 /* Make an effort to preserve v0 == v1. The target expander is able to
6146 rely on this to determine if we're permuting a single input operand. */
6147 if (rtx_equal_p (v0
, v1
))
6149 if (!insn_operand_matches (icode
, 1, v0
))
6150 v0
= force_reg (tmode
, v0
);
6151 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
6152 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
6154 create_fixed_operand (&ops
[1], v0
);
6155 create_fixed_operand (&ops
[2], v0
);
6159 create_input_operand (&ops
[1], v0
, tmode
);
6160 create_input_operand (&ops
[2], v1
, tmode
);
6163 if (maybe_expand_insn (icode
, 4, ops
))
6164 return ops
[0].value
;
6168 /* Implement a permutation of vectors v0 and v1 using the permutation
6169 vector in SEL and return the result. Use TARGET to hold the result
6170 if nonnull and convenient.
6172 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
6173 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
6174 to have a particular mode. */
6177 expand_vec_perm_const (machine_mode mode
, rtx v0
, rtx v1
,
6178 const vec_perm_builder
&sel
, machine_mode sel_mode
,
6181 if (!target
|| !register_operand (target
, mode
))
6182 target
= gen_reg_rtx (mode
);
6184 /* Set QIMODE to a different vector mode with byte elements.
6185 If no such mode, or if MODE already has byte elements, use VOIDmode. */
6186 machine_mode qimode
;
6187 if (!qimode_for_vec_perm (mode
).exists (&qimode
))
6190 rtx_insn
*last
= get_last_insn ();
6192 bool single_arg_p
= rtx_equal_p (v0
, v1
);
6193 /* Always specify two input vectors here and leave the target to handle
6194 cases in which the inputs are equal. Not all backends can cope with
6195 the single-input representation when testing for a double-input
6196 target instruction. */
6197 vec_perm_indices
indices (sel
, 2, GET_MODE_NUNITS (mode
));
6199 /* See if this can be handled with a vec_shr or vec_shl. We only do this
6200 if the second (for vec_shr) or first (for vec_shl) vector is all
6202 insn_code shift_code
= CODE_FOR_nothing
;
6203 insn_code shift_code_qi
= CODE_FOR_nothing
;
6204 optab shift_optab
= unknown_optab
;
6206 if (v1
== CONST0_RTX (GET_MODE (v1
)))
6207 shift_optab
= vec_shr_optab
;
6208 else if (v0
== CONST0_RTX (GET_MODE (v0
)))
6210 shift_optab
= vec_shl_optab
;
6213 if (shift_optab
!= unknown_optab
)
6215 shift_code
= optab_handler (shift_optab
, mode
);
6216 shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
6217 ? optab_handler (shift_optab
, qimode
)
6218 : CODE_FOR_nothing
);
6220 if (shift_code
!= CODE_FOR_nothing
|| shift_code_qi
!= CODE_FOR_nothing
)
6222 rtx shift_amt
= shift_amt_for_vec_perm_mask (mode
, indices
, shift_optab
);
6225 class expand_operand ops
[3];
6226 if (shift_amt
== const0_rtx
)
6228 if (shift_code
!= CODE_FOR_nothing
)
6230 create_output_operand (&ops
[0], target
, mode
);
6231 create_input_operand (&ops
[1], v2
, mode
);
6232 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
6233 if (maybe_expand_insn (shift_code
, 3, ops
))
6234 return ops
[0].value
;
6236 if (shift_code_qi
!= CODE_FOR_nothing
)
6238 rtx tmp
= gen_reg_rtx (qimode
);
6239 create_output_operand (&ops
[0], tmp
, qimode
);
6240 create_input_operand (&ops
[1], gen_lowpart (qimode
, v2
), qimode
);
6241 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
6242 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
6243 return gen_lowpart (mode
, ops
[0].value
);
6248 if (targetm
.vectorize
.vec_perm_const
!= NULL
)
6253 if (targetm
.vectorize
.vec_perm_const (mode
, target
, v0
, v1
, indices
))
6257 /* Fall back to a constant byte-based permutation. */
6258 vec_perm_indices qimode_indices
;
6259 rtx target_qi
= NULL_RTX
, v0_qi
= NULL_RTX
, v1_qi
= NULL_RTX
;
6260 if (qimode
!= VOIDmode
)
6262 qimode_indices
.new_expanded_vector (indices
, GET_MODE_UNIT_SIZE (mode
));
6263 target_qi
= gen_reg_rtx (qimode
);
6264 v0_qi
= gen_lowpart (qimode
, v0
);
6265 v1_qi
= gen_lowpart (qimode
, v1
);
6266 if (targetm
.vectorize
.vec_perm_const
!= NULL
6267 && targetm
.vectorize
.vec_perm_const (qimode
, target_qi
, v0_qi
,
6268 v1_qi
, qimode_indices
))
6269 return gen_lowpart (mode
, target_qi
);
6272 v0
= force_reg (mode
, v0
);
6275 v1
= force_reg (mode
, v1
);
6277 /* Otherwise expand as a fully variable permuation. */
6279 /* The optabs are only defined for selectors with the same width
6280 as the values being permuted. */
6281 machine_mode required_sel_mode
;
6282 if (!related_int_vector_mode (mode
).exists (&required_sel_mode
))
6284 delete_insns_since (last
);
6288 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
6289 If that isn't the mode we want then we need to prove that using
6290 REQUIRED_SEL_MODE is OK. */
6291 if (sel_mode
!= required_sel_mode
)
6293 if (!selector_fits_mode_p (required_sel_mode
, indices
))
6295 delete_insns_since (last
);
6298 sel_mode
= required_sel_mode
;
6301 insn_code icode
= direct_optab_handler (vec_perm_optab
, mode
);
6302 if (icode
!= CODE_FOR_nothing
)
6304 rtx sel_rtx
= vec_perm_indices_to_rtx (sel_mode
, indices
);
6305 rtx tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel_rtx
);
6310 if (qimode
!= VOIDmode
6311 && selector_fits_mode_p (qimode
, qimode_indices
))
6313 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6314 if (icode
!= CODE_FOR_nothing
)
6316 rtx sel_qi
= vec_perm_indices_to_rtx (qimode
, qimode_indices
);
6317 rtx tmp
= expand_vec_perm_1 (icode
, target_qi
, v0_qi
, v1_qi
, sel_qi
);
6319 return gen_lowpart (mode
, tmp
);
6323 delete_insns_since (last
);
6327 /* Implement a permutation of vectors v0 and v1 using the permutation
6328 vector in SEL and return the result. Use TARGET to hold the result
6329 if nonnull and convenient.
6331 MODE is the mode of the vectors being permuted (V0 and V1).
6332 SEL must have the integer equivalent of MODE and is known to be
6333 unsuitable for permutes with a constant permutation vector. */
6336 expand_vec_perm_var (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
6338 enum insn_code icode
;
6342 u
= GET_MODE_UNIT_SIZE (mode
);
6344 if (!target
|| GET_MODE (target
) != mode
)
6345 target
= gen_reg_rtx (mode
);
6347 icode
= direct_optab_handler (vec_perm_optab
, mode
);
6348 if (icode
!= CODE_FOR_nothing
)
6350 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
6355 /* As a special case to aid several targets, lower the element-based
6356 permutation to a byte-based permutation and try again. */
6357 machine_mode qimode
;
6358 if (!qimode_for_vec_perm (mode
).exists (&qimode
)
6359 || maybe_gt (GET_MODE_NUNITS (qimode
), GET_MODE_MASK (QImode
) + 1))
6361 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6362 if (icode
== CODE_FOR_nothing
)
6365 /* Multiply each element by its byte size. */
6366 machine_mode selmode
= GET_MODE (sel
);
6368 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
6369 NULL
, 0, OPTAB_DIRECT
);
6371 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
6372 gen_int_shift_amount (selmode
, exact_log2 (u
)),
6373 NULL
, 0, OPTAB_DIRECT
);
6374 gcc_assert (sel
!= NULL
);
6376 /* Broadcast the low byte each element into each of its bytes.
6377 The encoding has U interleaved stepped patterns, one for each
6378 byte of an element. */
6379 vec_perm_builder
const_sel (GET_MODE_SIZE (mode
), u
, 3);
6380 unsigned int low_byte_in_u
= BYTES_BIG_ENDIAN
? u
- 1 : 0;
6381 for (i
= 0; i
< 3; ++i
)
6382 for (unsigned int j
= 0; j
< u
; ++j
)
6383 const_sel
.quick_push (i
* u
+ low_byte_in_u
);
6384 sel
= gen_lowpart (qimode
, sel
);
6385 sel
= expand_vec_perm_const (qimode
, sel
, sel
, const_sel
, qimode
, NULL
);
6386 gcc_assert (sel
!= NULL
);
6388 /* Add the byte offset to each byte element. */
6389 /* Note that the definition of the indicies here is memory ordering,
6390 so there should be no difference between big and little endian. */
6391 rtx_vector_builder
byte_indices (qimode
, u
, 1);
6392 for (i
= 0; i
< u
; ++i
)
6393 byte_indices
.quick_push (GEN_INT (i
));
6394 tmp
= byte_indices
.build ();
6395 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
6396 sel
, 0, OPTAB_DIRECT
);
6397 gcc_assert (sel_qi
!= NULL
);
6399 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
6400 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
6401 gen_lowpart (qimode
, v1
), sel_qi
);
6403 tmp
= gen_lowpart (mode
, tmp
);
6407 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
6408 Use TARGET for the result if nonnull and convenient. */
6411 expand_vec_series_expr (machine_mode vmode
, rtx op0
, rtx op1
, rtx target
)
6413 class expand_operand ops
[3];
6414 enum insn_code icode
;
6415 machine_mode emode
= GET_MODE_INNER (vmode
);
6417 icode
= direct_optab_handler (vec_series_optab
, vmode
);
6418 gcc_assert (icode
!= CODE_FOR_nothing
);
6420 create_output_operand (&ops
[0], target
, vmode
);
6421 create_input_operand (&ops
[1], op0
, emode
);
6422 create_input_operand (&ops
[2], op1
, emode
);
6424 expand_insn (icode
, 3, ops
);
6425 return ops
[0].value
;
6428 /* Generate insns for a vector comparison into a mask. */
6431 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
6433 class expand_operand ops
[4];
6434 enum insn_code icode
;
6436 machine_mode mask_mode
= TYPE_MODE (type
);
6440 enum tree_code tcode
;
6442 op0a
= TREE_OPERAND (exp
, 0);
6443 op0b
= TREE_OPERAND (exp
, 1);
6444 tcode
= TREE_CODE (exp
);
6446 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
6447 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
6449 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
6450 if (icode
== CODE_FOR_nothing
)
6452 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
6453 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
6454 if (icode
== CODE_FOR_nothing
)
6458 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
6459 unsignedp
, icode
, 2);
6460 create_output_operand (&ops
[0], target
, mask_mode
);
6461 create_fixed_operand (&ops
[1], comparison
);
6462 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
6463 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
6464 expand_insn (icode
, 4, ops
);
6465 return ops
[0].value
;
6468 /* Expand a highpart multiply. */
6471 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
6472 rtx target
, bool uns_p
)
6474 class expand_operand eops
[3];
6475 enum insn_code icode
;
6481 method
= can_mult_highpart_p (mode
, uns_p
);
6487 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
6488 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
6491 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
6492 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
6495 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
6496 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
6497 if (BYTES_BIG_ENDIAN
)
6498 std::swap (tab1
, tab2
);
6504 icode
= optab_handler (tab1
, mode
);
6505 wmode
= insn_data
[icode
].operand
[0].mode
;
6506 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode
),
6507 GET_MODE_NUNITS (mode
)));
6508 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode
), GET_MODE_SIZE (mode
)));
6510 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
6511 create_input_operand (&eops
[1], op0
, mode
);
6512 create_input_operand (&eops
[2], op1
, mode
);
6513 expand_insn (icode
, 3, eops
);
6514 m1
= gen_lowpart (mode
, eops
[0].value
);
6516 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
6517 create_input_operand (&eops
[1], op0
, mode
);
6518 create_input_operand (&eops
[2], op1
, mode
);
6519 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
6520 m2
= gen_lowpart (mode
, eops
[0].value
);
6522 vec_perm_builder sel
;
6525 /* The encoding has 2 interleaved stepped patterns. */
6526 sel
.new_vector (GET_MODE_NUNITS (mode
), 2, 3);
6527 for (i
= 0; i
< 6; ++i
)
6528 sel
.quick_push (!BYTES_BIG_ENDIAN
+ (i
& ~1)
6529 + ((i
& 1) ? GET_MODE_NUNITS (mode
) : 0));
6533 /* The encoding has a single interleaved stepped pattern. */
6534 sel
.new_vector (GET_MODE_NUNITS (mode
), 1, 3);
6535 for (i
= 0; i
< 3; ++i
)
6536 sel
.quick_push (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
6539 return expand_vec_perm_const (mode
, m1
, m2
, sel
, BLKmode
, target
);
6542 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6546 find_cc_set (rtx x
, const_rtx pat
, void *data
)
6548 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
6549 && GET_CODE (pat
) == SET
)
6551 rtx
*p_cc_reg
= (rtx
*) data
;
6552 gcc_assert (!*p_cc_reg
);
6557 /* This is a helper function for the other atomic operations. This function
6558 emits a loop that contains SEQ that iterates until a compare-and-swap
6559 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6560 a set of instructions that takes a value from OLD_REG as an input and
6561 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6562 set to the current contents of MEM. After SEQ, a compare-and-swap will
6563 attempt to update MEM with NEW_REG. The function returns true when the
6564 loop was generated successfully. */
6567 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
6569 machine_mode mode
= GET_MODE (mem
);
6570 rtx_code_label
*label
;
6571 rtx cmp_reg
, success
, oldval
;
6573 /* The loop we want to generate looks like
6579 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6583 Note that we only do the plain load from memory once. Subsequent
6584 iterations use the value loaded by the compare-and-swap pattern. */
6586 label
= gen_label_rtx ();
6587 cmp_reg
= gen_reg_rtx (mode
);
6589 emit_move_insn (cmp_reg
, mem
);
6591 emit_move_insn (old_reg
, cmp_reg
);
6597 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
6598 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
6602 if (oldval
!= cmp_reg
)
6603 emit_move_insn (cmp_reg
, oldval
);
6605 /* Mark this jump predicted not taken. */
6606 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
6607 GET_MODE (success
), 1, label
,
6608 profile_probability::guessed_never ());
6613 /* This function tries to emit an atomic_exchange intruction. VAL is written
6614 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6615 using TARGET if possible. */
6618 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6620 machine_mode mode
= GET_MODE (mem
);
6621 enum insn_code icode
;
6623 /* If the target supports the exchange directly, great. */
6624 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
6625 if (icode
!= CODE_FOR_nothing
)
6627 class expand_operand ops
[4];
6629 create_output_operand (&ops
[0], target
, mode
);
6630 create_fixed_operand (&ops
[1], mem
);
6631 create_input_operand (&ops
[2], val
, mode
);
6632 create_integer_operand (&ops
[3], model
);
6633 if (maybe_expand_insn (icode
, 4, ops
))
6634 return ops
[0].value
;
6640 /* This function tries to implement an atomic exchange operation using
6641 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6642 The previous contents of *MEM are returned, using TARGET if possible.
6643 Since this instructionn is an acquire barrier only, stronger memory
6644 models may require additional barriers to be emitted. */
6647 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
6648 enum memmodel model
)
6650 machine_mode mode
= GET_MODE (mem
);
6651 enum insn_code icode
;
6652 rtx_insn
*last_insn
= get_last_insn ();
6654 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
6656 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6657 exists, and the memory model is stronger than acquire, add a release
6658 barrier before the instruction. */
6660 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
6661 expand_mem_thread_fence (model
);
6663 if (icode
!= CODE_FOR_nothing
)
6665 class expand_operand ops
[3];
6666 create_output_operand (&ops
[0], target
, mode
);
6667 create_fixed_operand (&ops
[1], mem
);
6668 create_input_operand (&ops
[2], val
, mode
);
6669 if (maybe_expand_insn (icode
, 3, ops
))
6670 return ops
[0].value
;
6673 /* If an external test-and-set libcall is provided, use that instead of
6674 any external compare-and-swap that we might get from the compare-and-
6675 swap-loop expansion later. */
6676 if (!can_compare_and_swap_p (mode
, false))
6678 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
6679 if (libfunc
!= NULL
)
6683 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6684 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6685 mode
, addr
, ptr_mode
,
6690 /* If the test_and_set can't be emitted, eliminate any barrier that might
6691 have been emitted. */
6692 delete_insns_since (last_insn
);
6696 /* This function tries to implement an atomic exchange operation using a
6697 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6698 *MEM are returned, using TARGET if possible. No memory model is required
6699 since a compare_and_swap loop is seq-cst. */
6702 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
6704 machine_mode mode
= GET_MODE (mem
);
6706 if (can_compare_and_swap_p (mode
, true))
6708 if (!target
|| !register_operand (target
, mode
))
6709 target
= gen_reg_rtx (mode
);
6710 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
6717 /* This function tries to implement an atomic test-and-set operation
6718 using the atomic_test_and_set instruction pattern. A boolean value
6719 is returned from the operation, using TARGET if possible. */
6722 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6724 machine_mode pat_bool_mode
;
6725 class expand_operand ops
[3];
6727 if (!targetm
.have_atomic_test_and_set ())
6730 /* While we always get QImode from __atomic_test_and_set, we get
6731 other memory modes from __sync_lock_test_and_set. Note that we
6732 use no endian adjustment here. This matches the 4.6 behavior
6733 in the Sparc backend. */
6734 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
6735 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
6736 if (GET_MODE (mem
) != QImode
)
6737 mem
= adjust_address_nv (mem
, QImode
, 0);
6739 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
6740 create_output_operand (&ops
[0], target
, pat_bool_mode
);
6741 create_fixed_operand (&ops
[1], mem
);
6742 create_integer_operand (&ops
[2], model
);
6744 if (maybe_expand_insn (icode
, 3, ops
))
6745 return ops
[0].value
;
6749 /* This function expands the legacy _sync_lock test_and_set operation which is
6750 generally an atomic exchange. Some limited targets only allow the
6751 constant 1 to be stored. This is an ACQUIRE operation.
6753 TARGET is an optional place to stick the return value.
6754 MEM is where VAL is stored. */
6757 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
6761 /* Try an atomic_exchange first. */
6762 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
6766 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
6767 MEMMODEL_SYNC_ACQUIRE
);
6771 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6775 /* If there are no other options, try atomic_test_and_set if the value
6776 being stored is 1. */
6777 if (val
== const1_rtx
)
6778 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
6783 /* This function expands the atomic test_and_set operation:
6784 atomically store a boolean TRUE into MEM and return the previous value.
6786 MEMMODEL is the memory model variant to use.
6787 TARGET is an optional place to stick the return value. */
6790 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6792 machine_mode mode
= GET_MODE (mem
);
6793 rtx ret
, trueval
, subtarget
;
6795 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
6799 /* Be binary compatible with non-default settings of trueval, and different
6800 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6801 another only has atomic-exchange. */
6802 if (targetm
.atomic_test_and_set_trueval
== 1)
6804 trueval
= const1_rtx
;
6805 subtarget
= target
? target
: gen_reg_rtx (mode
);
6809 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
6810 subtarget
= gen_reg_rtx (mode
);
6813 /* Try the atomic-exchange optab... */
6814 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
6816 /* ... then an atomic-compare-and-swap loop ... */
6818 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
6820 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6822 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
6824 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6825 things with the value 1. Thus we try again without trueval. */
6826 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
6827 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
6829 /* Failing all else, assume a single threaded environment and simply
6830 perform the operation. */
6833 /* If the result is ignored skip the move to target. */
6834 if (subtarget
!= const0_rtx
)
6835 emit_move_insn (subtarget
, mem
);
6837 emit_move_insn (mem
, trueval
);
6841 /* Recall that have to return a boolean value; rectify if trueval
6842 is not exactly one. */
6843 if (targetm
.atomic_test_and_set_trueval
!= 1)
6844 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
6849 /* This function expands the atomic exchange operation:
6850 atomically store VAL in MEM and return the previous value in MEM.
6852 MEMMODEL is the memory model variant to use.
6853 TARGET is an optional place to stick the return value. */
6856 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6858 machine_mode mode
= GET_MODE (mem
);
6861 /* If loads are not atomic for the required size and we are not called to
6862 provide a __sync builtin, do not do anything so that we stay consistent
6863 with atomic loads of the same size. */
6864 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6867 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6869 /* Next try a compare-and-swap loop for the exchange. */
6871 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6876 /* This function expands the atomic compare exchange operation:
6878 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6879 *PTARGET_OVAL is an optional place to store the old value from memory.
6880 Both target parameters may be NULL or const0_rtx to indicate that we do
6881 not care about that return value. Both target parameters are updated on
6882 success to the actual location of the corresponding result.
6884 MEMMODEL is the memory model variant to use.
6886 The return value of the function is true for success. */
6889 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
6890 rtx mem
, rtx expected
, rtx desired
,
6891 bool is_weak
, enum memmodel succ_model
,
6892 enum memmodel fail_model
)
6894 machine_mode mode
= GET_MODE (mem
);
6895 class expand_operand ops
[8];
6896 enum insn_code icode
;
6897 rtx target_oval
, target_bool
= NULL_RTX
;
6900 /* If loads are not atomic for the required size and we are not called to
6901 provide a __sync builtin, do not do anything so that we stay consistent
6902 with atomic loads of the same size. */
6903 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
6906 /* Load expected into a register for the compare and swap. */
6907 if (MEM_P (expected
))
6908 expected
= copy_to_reg (expected
);
6910 /* Make sure we always have some place to put the return oldval.
6911 Further, make sure that place is distinct from the input expected,
6912 just in case we need that path down below. */
6913 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
6914 ptarget_oval
= NULL
;
6916 if (ptarget_oval
== NULL
6917 || (target_oval
= *ptarget_oval
) == NULL
6918 || reg_overlap_mentioned_p (expected
, target_oval
))
6919 target_oval
= gen_reg_rtx (mode
);
6921 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
6922 if (icode
!= CODE_FOR_nothing
)
6924 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
6926 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
6927 ptarget_bool
= NULL
;
6929 /* Make sure we always have a place for the bool operand. */
6930 if (ptarget_bool
== NULL
6931 || (target_bool
= *ptarget_bool
) == NULL
6932 || GET_MODE (target_bool
) != bool_mode
)
6933 target_bool
= gen_reg_rtx (bool_mode
);
6935 /* Emit the compare_and_swap. */
6936 create_output_operand (&ops
[0], target_bool
, bool_mode
);
6937 create_output_operand (&ops
[1], target_oval
, mode
);
6938 create_fixed_operand (&ops
[2], mem
);
6939 create_input_operand (&ops
[3], expected
, mode
);
6940 create_input_operand (&ops
[4], desired
, mode
);
6941 create_integer_operand (&ops
[5], is_weak
);
6942 create_integer_operand (&ops
[6], succ_model
);
6943 create_integer_operand (&ops
[7], fail_model
);
6944 if (maybe_expand_insn (icode
, 8, ops
))
6946 /* Return success/failure. */
6947 target_bool
= ops
[0].value
;
6948 target_oval
= ops
[1].value
;
6953 /* Otherwise fall back to the original __sync_val_compare_and_swap
6954 which is always seq-cst. */
6955 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
6956 if (icode
!= CODE_FOR_nothing
)
6960 create_output_operand (&ops
[0], target_oval
, mode
);
6961 create_fixed_operand (&ops
[1], mem
);
6962 create_input_operand (&ops
[2], expected
, mode
);
6963 create_input_operand (&ops
[3], desired
, mode
);
6964 if (!maybe_expand_insn (icode
, 4, ops
))
6967 target_oval
= ops
[0].value
;
6969 /* If the caller isn't interested in the boolean return value,
6970 skip the computation of it. */
6971 if (ptarget_bool
== NULL
)
6974 /* Otherwise, work out if the compare-and-swap succeeded. */
6976 if (have_insn_for (COMPARE
, CCmode
))
6977 note_stores (get_last_insn (), find_cc_set
, &cc_reg
);
6980 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
6981 const0_rtx
, VOIDmode
, 0, 1);
6984 goto success_bool_from_val
;
6987 /* Also check for library support for __sync_val_compare_and_swap. */
6988 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
6989 if (libfunc
!= NULL
)
6991 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6992 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6993 mode
, addr
, ptr_mode
,
6994 expected
, mode
, desired
, mode
);
6995 emit_move_insn (target_oval
, target
);
6997 /* Compute the boolean return value only if requested. */
6999 goto success_bool_from_val
;
7007 success_bool_from_val
:
7008 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
7009 expected
, VOIDmode
, 1, 1);
7011 /* Make sure that the oval output winds up where the caller asked. */
7013 *ptarget_oval
= target_oval
;
7015 *ptarget_bool
= target_bool
;
7019 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
7022 expand_asm_memory_blockage (void)
7026 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
7027 rtvec_alloc (0), rtvec_alloc (0),
7028 rtvec_alloc (0), UNKNOWN_LOCATION
);
7029 MEM_VOLATILE_P (asm_op
) = 1;
7031 clob
= gen_rtx_SCRATCH (VOIDmode
);
7032 clob
= gen_rtx_MEM (BLKmode
, clob
);
7033 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
7035 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
7038 /* Do not propagate memory accesses across this point. */
7041 expand_memory_blockage (void)
7043 if (targetm
.have_memory_blockage ())
7044 emit_insn (targetm
.gen_memory_blockage ());
7046 expand_asm_memory_blockage ();
7049 /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7050 same time clobbering the register set specified by REGS. */
7053 expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs
)
7055 rtx asm_op
, clob_mem
;
7057 unsigned int num_of_regs
= 0;
7058 for (unsigned int i
= 0; i
< FIRST_PSEUDO_REGISTER
; i
++)
7059 if (TEST_HARD_REG_BIT (regs
, i
))
7062 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
7063 rtvec_alloc (0), rtvec_alloc (0),
7064 rtvec_alloc (0), UNKNOWN_LOCATION
);
7065 MEM_VOLATILE_P (asm_op
) = 1;
7067 rtvec v
= rtvec_alloc (num_of_regs
+ 2);
7069 clob_mem
= gen_rtx_SCRATCH (VOIDmode
);
7070 clob_mem
= gen_rtx_MEM (BLKmode
, clob_mem
);
7071 clob_mem
= gen_rtx_CLOBBER (VOIDmode
, clob_mem
);
7073 RTVEC_ELT (v
, 0) = asm_op
;
7074 RTVEC_ELT (v
, 1) = clob_mem
;
7076 if (num_of_regs
> 0)
7079 for (unsigned int i
= 0; i
< FIRST_PSEUDO_REGISTER
; i
++)
7080 if (TEST_HARD_REG_BIT (regs
, i
))
7082 RTVEC_ELT (v
, j
) = gen_rtx_CLOBBER (VOIDmode
, regno_reg_rtx
[i
]);
7085 gcc_assert (j
== (num_of_regs
+ 2));
7088 emit_insn (gen_rtx_PARALLEL (VOIDmode
, v
));
7091 /* This routine will either emit the mem_thread_fence pattern or issue a
7092 sync_synchronize to generate a fence for memory model MEMMODEL. */
7095 expand_mem_thread_fence (enum memmodel model
)
7097 if (is_mm_relaxed (model
))
7099 if (targetm
.have_mem_thread_fence ())
7101 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
7102 expand_memory_blockage ();
7104 else if (targetm
.have_memory_barrier ())
7105 emit_insn (targetm
.gen_memory_barrier ());
7106 else if (synchronize_libfunc
!= NULL_RTX
)
7107 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
);
7109 expand_memory_blockage ();
7112 /* Emit a signal fence with given memory model. */
7115 expand_mem_signal_fence (enum memmodel model
)
7117 /* No machine barrier is required to implement a signal fence, but
7118 a compiler memory barrier must be issued, except for relaxed MM. */
7119 if (!is_mm_relaxed (model
))
7120 expand_memory_blockage ();
7123 /* This function expands the atomic load operation:
7124 return the atomically loaded value in MEM.
7126 MEMMODEL is the memory model variant to use.
7127 TARGET is an option place to stick the return value. */
7130 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
7132 machine_mode mode
= GET_MODE (mem
);
7133 enum insn_code icode
;
7135 /* If the target supports the load directly, great. */
7136 icode
= direct_optab_handler (atomic_load_optab
, mode
);
7137 if (icode
!= CODE_FOR_nothing
)
7139 class expand_operand ops
[3];
7140 rtx_insn
*last
= get_last_insn ();
7141 if (is_mm_seq_cst (model
))
7142 expand_memory_blockage ();
7144 create_output_operand (&ops
[0], target
, mode
);
7145 create_fixed_operand (&ops
[1], mem
);
7146 create_integer_operand (&ops
[2], model
);
7147 if (maybe_expand_insn (icode
, 3, ops
))
7149 if (!is_mm_relaxed (model
))
7150 expand_memory_blockage ();
7151 return ops
[0].value
;
7153 delete_insns_since (last
);
7156 /* If the size of the object is greater than word size on this target,
7157 then we assume that a load will not be atomic. We could try to
7158 emulate a load with a compare-and-swap operation, but the store that
7159 doing this could result in would be incorrect if this is a volatile
7160 atomic load or targetting read-only-mapped memory. */
7161 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
7162 /* If there is no atomic load, leave the library call. */
7165 /* Otherwise assume loads are atomic, and emit the proper barriers. */
7166 if (!target
|| target
== const0_rtx
)
7167 target
= gen_reg_rtx (mode
);
7169 /* For SEQ_CST, emit a barrier before the load. */
7170 if (is_mm_seq_cst (model
))
7171 expand_mem_thread_fence (model
);
7173 emit_move_insn (target
, mem
);
7175 /* Emit the appropriate barrier after the load. */
7176 expand_mem_thread_fence (model
);
7181 /* This function expands the atomic store operation:
7182 Atomically store VAL in MEM.
7183 MEMMODEL is the memory model variant to use.
7184 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7185 function returns const0_rtx if a pattern was emitted. */
7188 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
7190 machine_mode mode
= GET_MODE (mem
);
7191 enum insn_code icode
;
7192 class expand_operand ops
[3];
7194 /* If the target supports the store directly, great. */
7195 icode
= direct_optab_handler (atomic_store_optab
, mode
);
7196 if (icode
!= CODE_FOR_nothing
)
7198 rtx_insn
*last
= get_last_insn ();
7199 if (!is_mm_relaxed (model
))
7200 expand_memory_blockage ();
7201 create_fixed_operand (&ops
[0], mem
);
7202 create_input_operand (&ops
[1], val
, mode
);
7203 create_integer_operand (&ops
[2], model
);
7204 if (maybe_expand_insn (icode
, 3, ops
))
7206 if (is_mm_seq_cst (model
))
7207 expand_memory_blockage ();
7210 delete_insns_since (last
);
7213 /* If using __sync_lock_release is a viable alternative, try it.
7214 Note that this will not be set to true if we are expanding a generic
7215 __atomic_store_n. */
7218 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
7219 if (icode
!= CODE_FOR_nothing
)
7221 create_fixed_operand (&ops
[0], mem
);
7222 create_input_operand (&ops
[1], const0_rtx
, mode
);
7223 if (maybe_expand_insn (icode
, 2, ops
))
7225 /* lock_release is only a release barrier. */
7226 if (is_mm_seq_cst (model
))
7227 expand_mem_thread_fence (model
);
7233 /* If the size of the object is greater than word size on this target,
7234 a default store will not be atomic. */
7235 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
7237 /* If loads are atomic or we are called to provide a __sync builtin,
7238 we can try a atomic_exchange and throw away the result. Otherwise,
7239 don't do anything so that we do not create an inconsistency between
7240 loads and stores. */
7241 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
7243 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
7245 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
7253 /* Otherwise assume stores are atomic, and emit the proper barriers. */
7254 expand_mem_thread_fence (model
);
7256 emit_move_insn (mem
, val
);
7258 /* For SEQ_CST, also emit a barrier after the store. */
7259 if (is_mm_seq_cst (model
))
7260 expand_mem_thread_fence (model
);
7266 /* Structure containing the pointers and values required to process the
7267 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7269 struct atomic_op_functions
7271 direct_optab mem_fetch_before
;
7272 direct_optab mem_fetch_after
;
7273 direct_optab mem_no_result
;
7276 direct_optab no_result
;
7277 enum rtx_code reverse_code
;
7281 /* Fill in structure pointed to by OP with the various optab entries for an
7282 operation of type CODE. */
7285 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
7287 gcc_assert (op
!= NULL
);
7289 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7290 in the source code during compilation, and the optab entries are not
7291 computable until runtime. Fill in the values at runtime. */
7295 op
->mem_fetch_before
= atomic_fetch_add_optab
;
7296 op
->mem_fetch_after
= atomic_add_fetch_optab
;
7297 op
->mem_no_result
= atomic_add_optab
;
7298 op
->fetch_before
= sync_old_add_optab
;
7299 op
->fetch_after
= sync_new_add_optab
;
7300 op
->no_result
= sync_add_optab
;
7301 op
->reverse_code
= MINUS
;
7304 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
7305 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
7306 op
->mem_no_result
= atomic_sub_optab
;
7307 op
->fetch_before
= sync_old_sub_optab
;
7308 op
->fetch_after
= sync_new_sub_optab
;
7309 op
->no_result
= sync_sub_optab
;
7310 op
->reverse_code
= PLUS
;
7313 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
7314 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
7315 op
->mem_no_result
= atomic_xor_optab
;
7316 op
->fetch_before
= sync_old_xor_optab
;
7317 op
->fetch_after
= sync_new_xor_optab
;
7318 op
->no_result
= sync_xor_optab
;
7319 op
->reverse_code
= XOR
;
7322 op
->mem_fetch_before
= atomic_fetch_and_optab
;
7323 op
->mem_fetch_after
= atomic_and_fetch_optab
;
7324 op
->mem_no_result
= atomic_and_optab
;
7325 op
->fetch_before
= sync_old_and_optab
;
7326 op
->fetch_after
= sync_new_and_optab
;
7327 op
->no_result
= sync_and_optab
;
7328 op
->reverse_code
= UNKNOWN
;
7331 op
->mem_fetch_before
= atomic_fetch_or_optab
;
7332 op
->mem_fetch_after
= atomic_or_fetch_optab
;
7333 op
->mem_no_result
= atomic_or_optab
;
7334 op
->fetch_before
= sync_old_ior_optab
;
7335 op
->fetch_after
= sync_new_ior_optab
;
7336 op
->no_result
= sync_ior_optab
;
7337 op
->reverse_code
= UNKNOWN
;
7340 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
7341 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
7342 op
->mem_no_result
= atomic_nand_optab
;
7343 op
->fetch_before
= sync_old_nand_optab
;
7344 op
->fetch_after
= sync_new_nand_optab
;
7345 op
->no_result
= sync_nand_optab
;
7346 op
->reverse_code
= UNKNOWN
;
7353 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7354 using memory order MODEL. If AFTER is true the operation needs to return
7355 the value of *MEM after the operation, otherwise the previous value.
7356 TARGET is an optional place to place the result. The result is unused if
7358 Return the result if there is a better sequence, otherwise NULL_RTX. */
7361 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7362 enum memmodel model
, bool after
)
7364 /* If the value is prefetched, or not used, it may be possible to replace
7365 the sequence with a native exchange operation. */
7366 if (!after
|| target
== const0_rtx
)
7368 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7369 if (code
== AND
&& val
== const0_rtx
)
7371 if (target
== const0_rtx
)
7372 target
= gen_reg_rtx (GET_MODE (mem
));
7373 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7376 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7377 if (code
== IOR
&& val
== constm1_rtx
)
7379 if (target
== const0_rtx
)
7380 target
= gen_reg_rtx (GET_MODE (mem
));
7381 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7388 /* Try to emit an instruction for a specific operation varaition.
7389 OPTAB contains the OP functions.
7390 TARGET is an optional place to return the result. const0_rtx means unused.
7391 MEM is the memory location to operate on.
7392 VAL is the value to use in the operation.
7393 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7394 MODEL is the memory model, if used.
7395 AFTER is true if the returned result is the value after the operation. */
7398 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
7399 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
7401 machine_mode mode
= GET_MODE (mem
);
7402 class expand_operand ops
[4];
7403 enum insn_code icode
;
7407 /* Check to see if there is a result returned. */
7408 if (target
== const0_rtx
)
7412 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
7413 create_integer_operand (&ops
[2], model
);
7418 icode
= direct_optab_handler (optab
->no_result
, mode
);
7422 /* Otherwise, we need to generate a result. */
7427 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
7428 : optab
->mem_fetch_before
, mode
);
7429 create_integer_operand (&ops
[3], model
);
7434 icode
= optab_handler (after
? optab
->fetch_after
7435 : optab
->fetch_before
, mode
);
7438 create_output_operand (&ops
[op_counter
++], target
, mode
);
7440 if (icode
== CODE_FOR_nothing
)
7443 create_fixed_operand (&ops
[op_counter
++], mem
);
7444 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7445 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
7447 if (maybe_expand_insn (icode
, num_ops
, ops
))
7448 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
7454 /* This function expands an atomic fetch_OP or OP_fetch operation:
7455 TARGET is an option place to stick the return value. const0_rtx indicates
7456 the result is unused.
7457 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7458 CODE is the operation being performed (OP)
7459 MEMMODEL is the memory model variant to use.
7460 AFTER is true to return the result of the operation (OP_fetch).
7461 AFTER is false to return the value before the operation (fetch_OP).
7463 This function will *only* generate instructions if there is a direct
7464 optab. No compare and swap loops or libcalls will be generated. */
7467 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
7468 enum rtx_code code
, enum memmodel model
,
7471 machine_mode mode
= GET_MODE (mem
);
7472 struct atomic_op_functions optab
;
7474 bool unused_result
= (target
== const0_rtx
);
7476 get_atomic_op_for_code (&optab
, code
);
7478 /* Check to see if there are any better instructions. */
7479 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
7483 /* Check for the case where the result isn't used and try those patterns. */
7486 /* Try the memory model variant first. */
7487 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
7491 /* Next try the old style withuot a memory model. */
7492 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
7496 /* There is no no-result pattern, so try patterns with a result. */
7500 /* Try the __atomic version. */
7501 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
7505 /* Try the older __sync version. */
7506 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
7510 /* If the fetch value can be calculated from the other variation of fetch,
7511 try that operation. */
7512 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
7514 /* Try the __atomic version, then the older __sync version. */
7515 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
7517 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
7521 /* If the result isn't used, no need to do compensation code. */
7525 /* Issue compensation code. Fetch_after == fetch_before OP val.
7526 Fetch_before == after REVERSE_OP val. */
7528 code
= optab
.reverse_code
;
7531 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
7532 true, OPTAB_LIB_WIDEN
);
7533 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
7536 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
7537 true, OPTAB_LIB_WIDEN
);
7542 /* No direct opcode can be generated. */
7548 /* This function expands an atomic fetch_OP or OP_fetch operation:
7549 TARGET is an option place to stick the return value. const0_rtx indicates
7550 the result is unused.
7551 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7552 CODE is the operation being performed (OP)
7553 MEMMODEL is the memory model variant to use.
7554 AFTER is true to return the result of the operation (OP_fetch).
7555 AFTER is false to return the value before the operation (fetch_OP). */
7557 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7558 enum memmodel model
, bool after
)
7560 machine_mode mode
= GET_MODE (mem
);
7562 bool unused_result
= (target
== const0_rtx
);
7564 /* If loads are not atomic for the required size and we are not called to
7565 provide a __sync builtin, do not do anything so that we stay consistent
7566 with atomic loads of the same size. */
7567 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
7570 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
7576 /* Add/sub can be implemented by doing the reverse operation with -(val). */
7577 if (code
== PLUS
|| code
== MINUS
)
7580 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
7583 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
7584 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
7588 /* PLUS worked so emit the insns and return. */
7595 /* PLUS did not work, so throw away the negation code and continue. */
7599 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7600 if (!can_compare_and_swap_p (mode
, false))
7604 enum rtx_code orig_code
= code
;
7605 struct atomic_op_functions optab
;
7607 get_atomic_op_for_code (&optab
, code
);
7608 libfunc
= optab_libfunc (after
? optab
.fetch_after
7609 : optab
.fetch_before
, mode
);
7611 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
7615 code
= optab
.reverse_code
;
7616 libfunc
= optab_libfunc (after
? optab
.fetch_before
7617 : optab
.fetch_after
, mode
);
7619 if (libfunc
!= NULL
)
7621 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
7622 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
7623 addr
, ptr_mode
, val
, mode
);
7625 if (!unused_result
&& fixup
)
7626 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
7627 true, OPTAB_LIB_WIDEN
);
7631 /* We need the original code for any further attempts. */
7635 /* If nothing else has succeeded, default to a compare and swap loop. */
7636 if (can_compare_and_swap_p (mode
, true))
7639 rtx t0
= gen_reg_rtx (mode
), t1
;
7643 /* If the result is used, get a register for it. */
7646 if (!target
|| !register_operand (target
, mode
))
7647 target
= gen_reg_rtx (mode
);
7648 /* If fetch_before, copy the value now. */
7650 emit_move_insn (target
, t0
);
7653 target
= const0_rtx
;
7658 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
7659 true, OPTAB_LIB_WIDEN
);
7660 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
7663 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
7666 /* For after, copy the value now. */
7667 if (!unused_result
&& after
)
7668 emit_move_insn (target
, t1
);
7669 insn
= get_insns ();
7672 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
7679 /* Return true if OPERAND is suitable for operand number OPNO of
7680 instruction ICODE. */
7683 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
7685 return (!insn_data
[(int) icode
].operand
[opno
].predicate
7686 || (insn_data
[(int) icode
].operand
[opno
].predicate
7687 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
7690 /* TARGET is a target of a multiword operation that we are going to
7691 implement as a series of word-mode operations. Return true if
7692 TARGET is suitable for this purpose. */
7695 valid_multiword_target_p (rtx target
)
7700 mode
= GET_MODE (target
);
7701 if (!GET_MODE_SIZE (mode
).is_constant (&size
))
7703 for (i
= 0; i
< size
; i
+= UNITS_PER_WORD
)
7704 if (!validate_subreg (word_mode
, mode
, target
, i
))
7709 /* Make OP describe an input operand that has value INTVAL and that has
7710 no inherent mode. This function should only be used for operands that
7711 are always expand-time constants. The backend may request that INTVAL
7712 be copied into a different kind of rtx, but it must specify the mode
7713 of that rtx if so. */
7716 create_integer_operand (class expand_operand
*op
, poly_int64 intval
)
7718 create_expand_operand (op
, EXPAND_INTEGER
,
7719 gen_int_mode (intval
, MAX_MODE_INT
),
7720 VOIDmode
, false, intval
);
7723 /* Like maybe_legitimize_operand, but do not change the code of the
7724 current rtx value. */
7727 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
7728 class expand_operand
*op
)
7730 /* See if the operand matches in its current form. */
7731 if (insn_operand_matches (icode
, opno
, op
->value
))
7734 /* If the operand is a memory whose address has no side effects,
7735 try forcing the address into a non-virtual pseudo register.
7736 The check for side effects is important because copy_to_mode_reg
7737 cannot handle things like auto-modified addresses. */
7738 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
7743 addr
= XEXP (mem
, 0);
7744 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
7745 && !side_effects_p (addr
))
7750 last
= get_last_insn ();
7751 mode
= get_address_mode (mem
);
7752 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
7753 if (insn_operand_matches (icode
, opno
, mem
))
7758 delete_insns_since (last
);
7765 /* Try to make OP match operand OPNO of instruction ICODE. Return true
7766 on success, storing the new operand value back in OP. */
7769 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
7770 class expand_operand
*op
)
7772 machine_mode mode
, imode
, tmode
;
7779 temporary_volatile_ok
v (true);
7780 return maybe_legitimize_operand_same_code (icode
, opno
, op
);
7784 gcc_assert (mode
!= VOIDmode
);
7786 && op
->value
!= const0_rtx
7787 && GET_MODE (op
->value
) == mode
7788 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
7791 op
->value
= gen_reg_rtx (mode
);
7797 gcc_assert (mode
!= VOIDmode
);
7798 gcc_assert (GET_MODE (op
->value
) == VOIDmode
7799 || GET_MODE (op
->value
) == mode
);
7800 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
7803 op
->value
= copy_to_mode_reg (mode
, op
->value
);
7806 case EXPAND_CONVERT_TO
:
7807 gcc_assert (mode
!= VOIDmode
);
7808 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
7811 case EXPAND_CONVERT_FROM
:
7812 if (GET_MODE (op
->value
) != VOIDmode
)
7813 mode
= GET_MODE (op
->value
);
7815 /* The caller must tell us what mode this value has. */
7816 gcc_assert (mode
!= VOIDmode
);
7818 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7819 tmode
= (VECTOR_MODE_P (imode
) && !VECTOR_MODE_P (mode
)
7820 ? GET_MODE_INNER (imode
) : imode
);
7821 if (tmode
!= VOIDmode
&& tmode
!= mode
)
7823 op
->value
= convert_modes (tmode
, mode
, op
->value
, op
->unsigned_p
);
7826 if (imode
!= VOIDmode
&& imode
!= mode
)
7828 gcc_assert (VECTOR_MODE_P (imode
) && !VECTOR_MODE_P (mode
));
7829 op
->value
= expand_vector_broadcast (imode
, op
->value
);
7834 case EXPAND_ADDRESS
:
7835 op
->value
= convert_memory_address (as_a
<scalar_int_mode
> (mode
),
7839 case EXPAND_INTEGER
:
7840 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7841 if (mode
!= VOIDmode
7842 && known_eq (trunc_int_for_mode (op
->int_value
, mode
),
7845 op
->value
= gen_int_mode (op
->int_value
, mode
);
7850 return insn_operand_matches (icode
, opno
, op
->value
);
7853 /* Make OP describe an input operand that should have the same value
7854 as VALUE, after any mode conversion that the target might request.
7855 TYPE is the type of VALUE. */
7858 create_convert_operand_from_type (class expand_operand
*op
,
7859 rtx value
, tree type
)
7861 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
7862 TYPE_UNSIGNED (type
));
7865 /* Return true if the requirements on operands OP1 and OP2 of instruction
7866 ICODE are similar enough for the result of legitimizing OP1 to be
7867 reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
7868 with OP1 and OP2 respectively. */
7871 can_reuse_operands_p (enum insn_code icode
,
7872 unsigned int opno1
, unsigned int opno2
,
7873 const class expand_operand
*op1
,
7874 const class expand_operand
*op2
)
7876 /* Check requirements that are common to all types. */
7877 if (op1
->type
!= op2
->type
7878 || op1
->mode
!= op2
->mode
7879 || (insn_data
[(int) icode
].operand
[opno1
].mode
7880 != insn_data
[(int) icode
].operand
[opno2
].mode
))
7883 /* Check the requirements for specific types. */
7887 /* Outputs must remain distinct. */
7892 case EXPAND_ADDRESS
:
7893 case EXPAND_INTEGER
:
7896 case EXPAND_CONVERT_TO
:
7897 case EXPAND_CONVERT_FROM
:
7898 return op1
->unsigned_p
== op2
->unsigned_p
;
7903 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7904 of instruction ICODE. Return true on success, leaving the new operand
7905 values in the OPS themselves. Emit no code on failure. */
7908 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
7909 unsigned int nops
, class expand_operand
*ops
)
7911 rtx_insn
*last
= get_last_insn ();
7912 rtx
*orig_values
= XALLOCAVEC (rtx
, nops
);
7913 for (unsigned int i
= 0; i
< nops
; i
++)
7915 orig_values
[i
] = ops
[i
].value
;
7917 /* First try reusing the result of an earlier legitimization.
7918 This avoids duplicate rtl and ensures that tied operands
7921 This search is linear, but NOPS is bounded at compile time
7922 to a small number (current a single digit). */
7925 if (can_reuse_operands_p (icode
, opno
+ j
, opno
+ i
, &ops
[j
], &ops
[i
])
7926 && rtx_equal_p (orig_values
[j
], orig_values
[i
])
7928 && insn_operand_matches (icode
, opno
+ i
, ops
[j
].value
))
7930 ops
[i
].value
= copy_rtx (ops
[j
].value
);
7934 /* Otherwise try legitimizing the operand on its own. */
7935 if (j
== i
&& !maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
7937 delete_insns_since (last
);
7944 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7945 as its operands. Return the instruction pattern on success,
7946 and emit any necessary set-up code. Return null and emit no
7950 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
7951 class expand_operand
*ops
)
7953 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
7954 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
7960 return GEN_FCN (icode
) (ops
[0].value
);
7962 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
7964 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
7966 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7969 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7970 ops
[3].value
, ops
[4].value
);
7972 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7973 ops
[3].value
, ops
[4].value
, ops
[5].value
);
7975 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7976 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7979 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7980 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7981 ops
[6].value
, ops
[7].value
);
7983 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7984 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7985 ops
[6].value
, ops
[7].value
, ops
[8].value
);
7990 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7991 as its operands. Return true on success and emit no code on failure. */
7994 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
7995 class expand_operand
*ops
)
7997 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
8006 /* Like maybe_expand_insn, but for jumps. */
8009 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
8010 class expand_operand
*ops
)
8012 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
8015 emit_jump_insn (pat
);
8021 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8025 expand_insn (enum insn_code icode
, unsigned int nops
,
8026 class expand_operand
*ops
)
8028 if (!maybe_expand_insn (icode
, nops
, ops
))
8032 /* Like expand_insn, but for jumps. */
8035 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
8036 class expand_operand
*ops
)
8038 if (!maybe_expand_jump_insn (icode
, nops
, ops
))