1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2024 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
45 #include "optabs-tree.h"
47 #include "internal-fn.h"
48 #include "langhooks.h"
52 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
54 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
55 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
57 static rtx
emit_conditional_move_1 (rtx
, rtx
, rtx
, rtx
, machine_mode
);
59 /* Debug facility for use in GDB. */
60 void debug_optab_libfuncs (void);
62 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
63 the result of operation CODE applied to OP0 (and OP1 if it is a binary
64 operation). OP0_MODE is OP0's mode.
66 If the last insn does not set TARGET, don't do anything, but return true.
68 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
69 or OP1, don't add the REG_EQUAL note but return false. Our caller can then
70 try again, ensuring that TARGET is not one of the operands. */
73 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
,
74 rtx op1
, machine_mode op0_mode
)
80 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
82 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
83 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
84 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
85 && GET_RTX_CLASS (code
) != RTX_COMPARE
86 && GET_RTX_CLASS (code
) != RTX_UNARY
)
89 if (GET_CODE (target
) == ZERO_EXTRACT
)
92 for (last_insn
= insns
;
93 NEXT_INSN (last_insn
) != NULL_RTX
;
94 last_insn
= NEXT_INSN (last_insn
))
97 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
98 a value changing in the insn, so the note would be invalid for CSE. */
99 if (reg_overlap_mentioned_p (target
, op0
)
100 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
103 && (rtx_equal_p (target
, op0
)
104 || (op1
&& rtx_equal_p (target
, op1
))))
106 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
107 over expanding it as temp = MEM op X, MEM = temp. If the target
108 supports MEM = MEM op X instructions, it is sometimes too hard
109 to reconstruct that form later, especially if X is also a memory,
110 and due to multiple occurrences of addresses the address might
111 be forced into register unnecessarily.
112 Note that not emitting the REG_EQUIV note might inhibit
113 CSE in some cases. */
114 set
= single_set (last_insn
);
116 && GET_CODE (SET_SRC (set
)) == code
117 && MEM_P (SET_DEST (set
))
118 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
119 || (op1
&& rtx_equal_p (SET_DEST (set
),
120 XEXP (SET_SRC (set
), 1)))))
126 set
= set_for_reg_notes (last_insn
);
130 if (! rtx_equal_p (SET_DEST (set
), target
)
131 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
132 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
133 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
136 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
146 if (op0_mode
!= VOIDmode
&& GET_MODE (target
) != op0_mode
)
148 note
= gen_rtx_fmt_e (code
, op0_mode
, copy_rtx (op0
));
149 if (GET_MODE_UNIT_SIZE (op0_mode
)
150 > GET_MODE_UNIT_SIZE (GET_MODE (target
)))
151 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
154 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
160 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
164 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
166 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
171 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
172 for a widening operation would be. In most cases this would be OP0, but if
173 that's a constant it'll be VOIDmode, which isn't useful. */
176 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
178 machine_mode m0
= GET_MODE (op0
);
179 machine_mode m1
= GET_MODE (op1
);
182 if (m0
== VOIDmode
&& m1
== VOIDmode
)
184 else if (m0
== VOIDmode
|| GET_MODE_UNIT_SIZE (m0
) < GET_MODE_UNIT_SIZE (m1
))
189 if (GET_MODE_UNIT_SIZE (result
) > GET_MODE_UNIT_SIZE (to_mode
))
195 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
196 says whether OP is signed or unsigned. NO_EXTEND is true if we need
197 not actually do a sign-extend or zero-extend, but can leave the
198 higher-order bits of the result rtx undefined, for example, in the case
199 of logical operations, but not right shifts. */
202 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
203 int unsignedp
, bool no_extend
)
206 scalar_int_mode int_mode
;
208 /* If we don't have to extend and this is a constant, return it. */
209 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
212 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
213 extend since it will be more efficient to do so unless the signedness of
214 a promoted object differs from our extension. */
216 || !is_a
<scalar_int_mode
> (mode
, &int_mode
)
217 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
218 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
219 return convert_modes (mode
, oldmode
, op
, unsignedp
);
221 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
223 if (GET_MODE_SIZE (int_mode
) <= UNITS_PER_WORD
)
224 return gen_lowpart (int_mode
, force_reg (GET_MODE (op
), op
));
226 /* Otherwise, get an object of MODE, clobber it, and set the low-order
229 result
= gen_reg_rtx (int_mode
);
230 emit_clobber (result
);
231 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
235 /* Expand vector widening operations.
237 There are two different classes of operations handled here:
238 1) Operations whose result is wider than all the arguments to the operation.
239 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
240 In this case OP0 and optionally OP1 would be initialized,
241 but WIDE_OP wouldn't (not relevant for this case).
242 2) Operations whose result is of the same size as the last argument to the
243 operation, but wider than all the other arguments to the operation.
244 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
245 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
247 E.g, when called to expand the following operations, this is how
248 the arguments will be initialized:
250 widening-sum 2 oprnd0 - oprnd1
251 widening-dot-product 3 oprnd0 oprnd1 oprnd2
252 widening-mult 2 oprnd0 oprnd1 -
253 type-promotion (vec-unpack) 1 oprnd0 - - */
256 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
257 rtx target
, int unsignedp
)
259 class expand_operand eops
[4];
260 tree oprnd0
, oprnd1
, oprnd2
;
261 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
262 optab widen_pattern_optab
;
263 enum insn_code icode
;
264 int nops
= TREE_CODE_LENGTH (ops
->code
);
269 oprnd1
= nops
>= 2 ? ops
->op1
: NULL_TREE
;
270 oprnd2
= nops
>= 3 ? ops
->op2
: NULL_TREE
;
272 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
273 if (ops
->code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
274 || ops
->code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
)
275 /* The sign is from the result type rather than operand's type
278 = optab_for_tree_code (ops
->code
, ops
->type
, optab_default
);
279 else if ((ops
->code
== VEC_UNPACK_HI_EXPR
280 || ops
->code
== VEC_UNPACK_LO_EXPR
)
281 && VECTOR_BOOLEAN_TYPE_P (ops
->type
)
282 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0
))
283 && TYPE_MODE (ops
->type
) == TYPE_MODE (TREE_TYPE (oprnd0
))
284 && SCALAR_INT_MODE_P (TYPE_MODE (ops
->type
)))
286 /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
287 the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
288 vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
289 the pattern number of elements in the wider vector. */
291 = (ops
->code
== VEC_UNPACK_HI_EXPR
292 ? vec_unpacks_sbool_hi_optab
: vec_unpacks_sbool_lo_optab
);
295 else if (ops
->code
== DOT_PROD_EXPR
)
297 enum optab_subtype subtype
= optab_default
;
298 signop sign1
= TYPE_SIGN (TREE_TYPE (oprnd0
));
299 signop sign2
= TYPE_SIGN (TREE_TYPE (oprnd1
));
302 else if (sign1
== SIGNED
&& sign2
== UNSIGNED
)
304 subtype
= optab_vector_mixed_sign
;
305 /* Same as optab_vector_mixed_sign but flip the operands. */
306 std::swap (op0
, op1
);
308 else if (sign1
== UNSIGNED
&& sign2
== SIGNED
)
309 subtype
= optab_vector_mixed_sign
;
314 = optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), subtype
);
318 = optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
319 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
320 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
321 icode
= find_widening_optab_handler (widen_pattern_optab
,
322 TYPE_MODE (TREE_TYPE (ops
->op2
)),
325 icode
= optab_handler (widen_pattern_optab
, tmode0
);
326 gcc_assert (icode
!= CODE_FOR_nothing
);
329 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
333 op1
= GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0
)).to_constant ());
337 /* The last operand is of a wider mode than the rest of the operands. */
342 gcc_assert (tmode1
== tmode0
);
344 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
348 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
349 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
351 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
353 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
354 expand_insn (icode
, op
, eops
);
355 return eops
[0].value
;
358 /* Generate code to perform an operation specified by TERNARY_OPTAB
359 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
361 UNSIGNEDP is for the case where we have to widen the operands
362 to perform the operation. It says to use zero-extension.
364 If TARGET is nonzero, the value
365 is generated there, if it is convenient to do so.
366 In all cases an rtx is returned for the locus of the value;
367 this may or may not be TARGET. */
370 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
371 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
373 class expand_operand ops
[4];
374 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
376 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
378 create_output_operand (&ops
[0], target
, mode
);
379 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
380 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
381 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
382 expand_insn (icode
, 4, ops
);
387 /* Like expand_binop, but return a constant rtx if the result can be
388 calculated at compile time. The arguments and return value are
389 otherwise the same as for expand_binop. */
392 simplify_expand_binop (machine_mode mode
, optab binoptab
,
393 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
394 enum optab_methods methods
)
396 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
398 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
404 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
407 /* Like simplify_expand_binop, but always put the result in TARGET.
408 Return true if the expansion succeeded. */
411 force_expand_binop (machine_mode mode
, optab binoptab
,
412 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
413 enum optab_methods methods
)
415 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
416 target
, unsignedp
, methods
);
420 emit_move_insn (target
, x
);
424 /* Create a new vector value in VMODE with all elements set to OP. The
425 mode of OP must be the element mode of VMODE. If OP is a constant,
426 then the return value will be a constant. */
429 expand_vector_broadcast (machine_mode vmode
, rtx op
)
434 gcc_checking_assert (VECTOR_MODE_P (vmode
));
436 if (valid_for_const_vector_p (vmode
, op
))
437 return gen_const_vec_duplicate (vmode
, op
);
439 insn_code icode
= optab_handler (vec_duplicate_optab
, vmode
);
440 if (icode
!= CODE_FOR_nothing
)
442 class expand_operand ops
[2];
443 create_output_operand (&ops
[0], NULL_RTX
, vmode
);
444 create_input_operand (&ops
[1], op
, GET_MODE (op
));
445 expand_insn (icode
, 2, ops
);
449 if (!GET_MODE_NUNITS (vmode
).is_constant (&n
))
452 /* ??? If the target doesn't have a vec_init, then we have no easy way
453 of performing this operation. Most of this sort of generic support
454 is hidden away in the vector lowering support in gimple. */
455 icode
= convert_optab_handler (vec_init_optab
, vmode
,
456 GET_MODE_INNER (vmode
));
457 if (icode
== CODE_FOR_nothing
)
460 vec
= rtvec_alloc (n
);
461 for (int i
= 0; i
< n
; ++i
)
462 RTVEC_ELT (vec
, i
) = op
;
463 rtx ret
= gen_reg_rtx (vmode
);
464 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
469 /* This subroutine of expand_doubleword_shift handles the cases in which
470 the effective shift value is >= BITS_PER_WORD. The arguments and return
471 value are the same as for the parent routine, except that SUPERWORD_OP1
472 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
473 INTO_TARGET may be null if the caller has decided to calculate it. */
476 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
477 rtx outof_target
, rtx into_target
,
478 int unsignedp
, enum optab_methods methods
)
480 if (into_target
!= 0)
481 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
482 into_target
, unsignedp
, methods
))
485 if (outof_target
!= 0)
487 /* For a signed right shift, we must fill OUTOF_TARGET with copies
488 of the sign bit, otherwise we must fill it with zeros. */
489 if (binoptab
!= ashr_optab
)
490 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
492 if (!force_expand_binop (word_mode
, binoptab
, outof_input
,
493 gen_int_shift_amount (word_mode
,
495 outof_target
, unsignedp
, methods
))
501 /* This subroutine of expand_doubleword_shift handles the cases in which
502 the effective shift value is < BITS_PER_WORD. The arguments and return
503 value are the same as for the parent routine. */
506 expand_subword_shift (scalar_int_mode op1_mode
, optab binoptab
,
507 rtx outof_input
, rtx into_input
, rtx op1
,
508 rtx outof_target
, rtx into_target
,
509 int unsignedp
, enum optab_methods methods
,
510 unsigned HOST_WIDE_INT shift_mask
)
512 optab reverse_unsigned_shift
, unsigned_shift
;
515 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
516 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
518 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
519 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
520 the opposite direction to BINOPTAB. */
521 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
523 carries
= outof_input
;
524 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
525 op1_mode
), op1_mode
);
526 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
531 /* We must avoid shifting by BITS_PER_WORD bits since that is either
532 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
533 has unknown behavior. Do a single shift first, then shift by the
534 remainder. It's OK to use ~OP1 as the remainder if shift counts
535 are truncated to the mode size. */
536 carries
= simplify_expand_binop (word_mode
, reverse_unsigned_shift
,
537 outof_input
, const1_rtx
, 0,
539 if (carries
== const0_rtx
)
541 else if (shift_mask
== BITS_PER_WORD
- 1)
542 tmp
= expand_unop (op1_mode
, one_cmpl_optab
, op1
, 0, true);
545 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
546 op1_mode
), op1_mode
);
547 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
551 if (tmp
== 0 || carries
== 0)
553 if (carries
!= const0_rtx
&& tmp
!= const0_rtx
)
554 carries
= simplify_expand_binop (word_mode
, reverse_unsigned_shift
,
555 carries
, tmp
, 0, unsignedp
, methods
);
559 if (into_input
!= const0_rtx
)
561 /* Shift INTO_INPUT logically by OP1. This is the last use of
562 INTO_INPUT so the result can go directly into INTO_TARGET if
564 tmp
= simplify_expand_binop (word_mode
, unsigned_shift
, into_input
,
565 op1
, into_target
, unsignedp
, methods
);
569 /* Now OR in the bits carried over from OUTOF_INPUT. */
570 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
571 into_target
, unsignedp
, methods
))
575 emit_move_insn (into_target
, carries
);
577 /* Use a standard word_mode shift for the out-of half. */
578 if (outof_target
!= 0)
579 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
580 outof_target
, unsignedp
, methods
))
587 /* Try implementing expand_doubleword_shift using conditional moves.
588 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
589 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
590 are the shift counts to use in the former and latter case. All other
591 arguments are the same as the parent routine. */
594 expand_doubleword_shift_condmove (scalar_int_mode op1_mode
, optab binoptab
,
595 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
596 rtx outof_input
, rtx into_input
,
597 rtx subword_op1
, rtx superword_op1
,
598 rtx outof_target
, rtx into_target
,
599 int unsignedp
, enum optab_methods methods
,
600 unsigned HOST_WIDE_INT shift_mask
)
602 rtx outof_superword
, into_superword
;
604 /* Put the superword version of the output into OUTOF_SUPERWORD and
606 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
607 if (outof_target
!= 0 && subword_op1
== superword_op1
)
609 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
610 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
611 into_superword
= outof_target
;
612 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
613 outof_superword
, 0, unsignedp
, methods
))
618 into_superword
= gen_reg_rtx (word_mode
);
619 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
620 outof_superword
, into_superword
,
625 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
626 if (!expand_subword_shift (op1_mode
, binoptab
,
627 outof_input
, into_input
, subword_op1
,
628 outof_target
, into_target
,
629 unsignedp
, methods
, shift_mask
))
632 /* Select between them. Do the INTO half first because INTO_SUPERWORD
633 might be the current value of OUTOF_TARGET. */
634 if (!emit_conditional_move (into_target
, { cmp_code
, cmp1
, cmp2
, op1_mode
},
635 into_target
, into_superword
, word_mode
, false))
638 if (outof_target
!= 0)
639 if (!emit_conditional_move (outof_target
,
640 { cmp_code
, cmp1
, cmp2
, op1_mode
},
641 outof_target
, outof_superword
,
648 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
649 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
650 input operand; the shift moves bits in the direction OUTOF_INPUT->
651 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
652 of the target. OP1 is the shift count and OP1_MODE is its mode.
653 If OP1 is constant, it will have been truncated as appropriate
654 and is known to be nonzero.
656 If SHIFT_MASK is zero, the result of word shifts is undefined when the
657 shift count is outside the range [0, BITS_PER_WORD). This routine must
658 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
660 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
661 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
662 fill with zeros or sign bits as appropriate.
664 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
665 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
666 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
667 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
670 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
671 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
672 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
673 function wants to calculate it itself.
675 Return true if the shift could be successfully synthesized. */
678 expand_doubleword_shift (scalar_int_mode op1_mode
, optab binoptab
,
679 rtx outof_input
, rtx into_input
, rtx op1
,
680 rtx outof_target
, rtx into_target
,
681 int unsignedp
, enum optab_methods methods
,
682 unsigned HOST_WIDE_INT shift_mask
)
684 rtx superword_op1
, tmp
, cmp1
, cmp2
;
685 enum rtx_code cmp_code
;
687 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
688 fill the result with sign or zero bits as appropriate. If so, the value
689 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
690 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
691 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
693 This isn't worthwhile for constant shifts since the optimizers will
694 cope better with in-range shift counts. */
695 if (shift_mask
>= BITS_PER_WORD
697 && !CONSTANT_P (op1
))
699 if (!expand_doubleword_shift (op1_mode
, binoptab
,
700 outof_input
, into_input
, op1
,
702 unsignedp
, methods
, shift_mask
))
704 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
705 outof_target
, unsignedp
, methods
))
710 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
711 is true when the effective shift value is less than BITS_PER_WORD.
712 Set SUPERWORD_OP1 to the shift count that should be used to shift
713 OUTOF_INPUT into INTO_TARGET when the condition is false. */
714 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
715 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
717 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
718 is a subword shift count. */
719 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
721 cmp2
= CONST0_RTX (op1_mode
);
727 /* Set CMP1 to OP1 - BITS_PER_WORD. */
728 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
730 cmp2
= CONST0_RTX (op1_mode
);
732 superword_op1
= cmp1
;
737 /* If we can compute the condition at compile time, pick the
738 appropriate subroutine. */
739 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
740 if (tmp
!= 0 && CONST_INT_P (tmp
))
742 if (tmp
== const0_rtx
)
743 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
744 outof_target
, into_target
,
747 return expand_subword_shift (op1_mode
, binoptab
,
748 outof_input
, into_input
, op1
,
749 outof_target
, into_target
,
750 unsignedp
, methods
, shift_mask
);
753 /* Try using conditional moves to generate straight-line code. */
754 if (HAVE_conditional_move
)
756 rtx_insn
*start
= get_last_insn ();
757 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
758 cmp_code
, cmp1
, cmp2
,
759 outof_input
, into_input
,
761 outof_target
, into_target
,
762 unsignedp
, methods
, shift_mask
))
764 delete_insns_since (start
);
767 /* As a last resort, use branches to select the correct alternative. */
768 rtx_code_label
*subword_label
= gen_label_rtx ();
769 rtx_code_label
*done_label
= gen_label_rtx ();
772 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
774 profile_probability::uninitialized ());
777 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
778 outof_target
, into_target
,
782 emit_jump_insn (targetm
.gen_jump (done_label
));
784 emit_label (subword_label
);
786 if (!expand_subword_shift (op1_mode
, binoptab
,
787 outof_input
, into_input
, op1
,
788 outof_target
, into_target
,
789 unsignedp
, methods
, shift_mask
))
792 emit_label (done_label
);
796 /* Subroutine of expand_binop. Perform a double word multiplication of
797 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
798 as the target's word_mode. This function return NULL_RTX if anything
799 goes wrong, in which case it may have already emitted instructions
800 which need to be deleted.
802 If we want to multiply two two-word values and have normal and widening
803 multiplies of single-word values, we can do this with three smaller
806 The multiplication proceeds as follows:
807 _______________________
808 [__op0_high_|__op0_low__]
809 _______________________
810 * [__op1_high_|__op1_low__]
811 _______________________________________________
812 _______________________
813 (1) [__op0_low__*__op1_low__]
814 _______________________
815 (2a) [__op0_low__*__op1_high_]
816 _______________________
817 (2b) [__op0_high_*__op1_low__]
818 _______________________
819 (3) [__op0_high_*__op1_high_]
822 This gives a 4-word result. Since we are only interested in the
823 lower 2 words, partial result (3) and the upper words of (2a) and
824 (2b) don't need to be calculated. Hence (2a) and (2b) can be
825 calculated using non-widening multiplication.
827 (1), however, needs to be calculated with an unsigned widening
828 multiplication. If this operation is not directly supported we
829 try using a signed widening multiplication and adjust the result.
830 This adjustment works as follows:
832 If both operands are positive then no adjustment is needed.
834 If the operands have different signs, for example op0_low < 0 and
835 op1_low >= 0, the instruction treats the most significant bit of
836 op0_low as a sign bit instead of a bit with significance
837 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
838 with 2**BITS_PER_WORD - op0_low, and two's complements the
839 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
842 Similarly, if both operands are negative, we need to add
843 (op0_low + op1_low) * 2**BITS_PER_WORD.
845 We use a trick to adjust quickly. We logically shift op0_low right
846 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
847 op0_high (op1_high) before it is used to calculate 2b (2a). If no
848 logical shift exists, we do an arithmetic right shift and subtract
852 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
853 bool umulp
, enum optab_methods methods
)
855 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
856 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
857 rtx wordm1
= (umulp
? NULL_RTX
858 : gen_int_shift_amount (word_mode
, BITS_PER_WORD
- 1));
859 rtx product
, adjust
, product_high
, temp
;
861 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
862 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
863 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
864 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
866 /* If we're using an unsigned multiply to directly compute the product
867 of the low-order words of the operands and perform any required
868 adjustments of the operands, we begin by trying two more multiplications
869 and then computing the appropriate sum.
871 We have checked above that the required addition is provided.
872 Full-word addition will normally always succeed, especially if
873 it is provided at all, so we don't worry about its failure. The
874 multiplication may well fail, however, so we do handle that. */
878 /* ??? This could be done with emit_store_flag where available. */
879 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
880 NULL_RTX
, 1, methods
);
882 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
883 NULL_RTX
, 0, OPTAB_DIRECT
);
886 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
887 NULL_RTX
, 0, methods
);
890 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
891 NULL_RTX
, 0, OPTAB_DIRECT
);
898 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
899 NULL_RTX
, 0, OPTAB_DIRECT
);
903 /* OP0_HIGH should now be dead. */
907 /* ??? This could be done with emit_store_flag where available. */
908 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
909 NULL_RTX
, 1, methods
);
911 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
912 NULL_RTX
, 0, OPTAB_DIRECT
);
915 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
916 NULL_RTX
, 0, methods
);
919 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
920 NULL_RTX
, 0, OPTAB_DIRECT
);
927 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
928 NULL_RTX
, 0, OPTAB_DIRECT
);
932 /* OP1_HIGH should now be dead. */
934 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
935 NULL_RTX
, 0, OPTAB_DIRECT
);
937 if (target
&& !REG_P (target
))
940 /* *_widen_optab needs to determine operand mode, make sure at least
941 one operand has non-VOID mode. */
942 if (GET_MODE (op0_low
) == VOIDmode
&& GET_MODE (op1_low
) == VOIDmode
)
943 op0_low
= force_reg (word_mode
, op0_low
);
946 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
947 target
, 1, OPTAB_DIRECT
);
949 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
950 target
, 1, OPTAB_DIRECT
);
955 product_high
= operand_subword (product
, high
, 1, mode
);
956 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
957 NULL_RTX
, 0, OPTAB_DIRECT
);
958 emit_move_insn (product_high
, adjust
);
962 /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
963 constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
964 (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
965 computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
966 + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
967 depends on the bit value, if 2, then carry from the addition needs to be
968 added too, i.e. like:
969 sum += __builtin_add_overflow (low, high, &sum)
971 Optimize signed double-word OP0 % OP1 similarly, just apply some correction
972 factor to the sum before doing unsigned remainder, in the form of
973 sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
974 then perform unsigned
975 remainder = sum % OP1;
977 remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
980 expand_doubleword_mod (machine_mode mode
, rtx op0
, rtx op1
, bool unsignedp
)
982 if (INTVAL (op1
) <= 1 || (INTVAL (op1
) & 1) == 0)
985 rtx_insn
*last
= get_last_insn ();
986 for (int bit
= BITS_PER_WORD
; bit
>= BITS_PER_WORD
/ 2; bit
--)
988 wide_int w
= wi::shifted_mask (bit
, 1, false, 2 * BITS_PER_WORD
);
989 if (wi::ne_p (wi::umod_trunc (w
, INTVAL (op1
)), 1))
991 rtx sum
= NULL_RTX
, mask
= NULL_RTX
;
992 if (bit
== BITS_PER_WORD
)
994 /* For signed modulo we need to add correction to the sum
995 and that might again overflow. */
998 if (optab_handler (uaddv4_optab
, word_mode
) == CODE_FOR_nothing
)
1000 tree wtype
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
1001 if (wtype
== NULL_TREE
)
1003 tree ctype
= build_complex_type (wtype
);
1004 if (TYPE_MODE (ctype
) != GET_MODE_COMPLEX_MODE (word_mode
))
1006 machine_mode cmode
= TYPE_MODE (ctype
);
1007 rtx op00
= operand_subword_force (op0
, 0, mode
);
1008 rtx op01
= operand_subword_force (op0
, 1, mode
);
1009 rtx cres
= gen_rtx_CONCAT (cmode
, gen_reg_rtx (word_mode
),
1010 gen_reg_rtx (word_mode
));
1011 tree lhs
= make_tree (ctype
, cres
);
1012 tree arg0
= make_tree (wtype
, op00
);
1013 tree arg1
= make_tree (wtype
, op01
);
1014 expand_addsub_overflow (UNKNOWN_LOCATION
, PLUS_EXPR
, lhs
, arg0
,
1015 arg1
, true, true, true, false, NULL
);
1016 sum
= expand_simple_binop (word_mode
, PLUS
, XEXP (cres
, 0),
1017 XEXP (cres
, 1), NULL_RTX
, 1,
1019 if (sum
== NULL_RTX
)
1024 /* Code below uses GEN_INT, so we need the masks to be representable
1025 in HOST_WIDE_INTs. */
1026 if (bit
>= HOST_BITS_PER_WIDE_INT
)
1028 /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1029 overflow. Consider 64-bit -1ULL for word size 32, if we add
1030 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1031 if (bit
== BITS_PER_WORD
- 1)
1034 int count
= (2 * BITS_PER_WORD
+ bit
- 1) / bit
;
1035 rtx sum_corr
= NULL_RTX
;
1039 /* For signed modulo, compute it as unsigned modulo of
1040 sum with a correction added to it if OP0 is negative,
1041 such that the result can be computed as unsigned
1042 remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1043 w
= wi::min_value (2 * BITS_PER_WORD
, SIGNED
);
1044 wide_int wmod1
= wi::umod_trunc (w
, INTVAL (op1
));
1045 wide_int wmod2
= wi::smod_trunc (w
, INTVAL (op1
));
1046 /* wmod2 == -wmod1. */
1047 wmod2
= wmod2
+ (INTVAL (op1
) - 1);
1048 if (wi::ne_p (wmod1
, wmod2
))
1050 wide_int wcorr
= wmod2
- wmod1
;
1052 wcorr
= wcorr
+ INTVAL (op1
);
1053 /* Now verify if the count sums can't overflow, and punt
1055 w
= wi::mask (bit
, false, 2 * BITS_PER_WORD
);
1056 w
= w
* (count
- 1);
1057 w
= w
+ wi::mask (2 * BITS_PER_WORD
- (count
- 1) * bit
,
1058 false, 2 * BITS_PER_WORD
);
1060 w
= wi::lrshift (w
, BITS_PER_WORD
);
1061 if (wi::ne_p (w
, 0))
1064 mask
= operand_subword_force (op0
, WORDS_BIG_ENDIAN
? 0 : 1,
1066 mask
= expand_simple_binop (word_mode
, ASHIFTRT
, mask
,
1067 GEN_INT (BITS_PER_WORD
- 1),
1068 NULL_RTX
, 0, OPTAB_DIRECT
);
1069 if (mask
== NULL_RTX
)
1071 sum_corr
= immed_wide_int_const (wcorr
, word_mode
);
1072 sum_corr
= expand_simple_binop (word_mode
, AND
, mask
,
1073 sum_corr
, NULL_RTX
, 1,
1075 if (sum_corr
== NULL_RTX
)
1080 for (int i
= 0; i
< count
; i
++)
1084 v
= expand_simple_binop (mode
, LSHIFTRT
, v
, GEN_INT (i
* bit
),
1085 NULL_RTX
, 1, OPTAB_DIRECT
);
1088 v
= lowpart_subreg (word_mode
, v
, mode
);
1092 v
= expand_simple_binop (word_mode
, AND
, v
,
1093 GEN_INT ((HOST_WIDE_INT_1U
<< bit
)
1098 if (sum
== NULL_RTX
)
1101 sum
= expand_simple_binop (word_mode
, PLUS
, sum
, v
, NULL_RTX
,
1103 if (sum
== NULL_RTX
)
1108 sum
= expand_simple_binop (word_mode
, PLUS
, sum
, sum_corr
,
1109 NULL_RTX
, 1, OPTAB_DIRECT
);
1110 if (sum
== NULL_RTX
)
1114 rtx remainder
= expand_divmod (1, TRUNC_MOD_EXPR
, word_mode
, sum
,
1115 gen_int_mode (INTVAL (op1
), word_mode
),
1116 NULL_RTX
, 1, OPTAB_DIRECT
);
1117 if (remainder
== NULL_RTX
)
1122 if (mask
== NULL_RTX
)
1124 mask
= operand_subword_force (op0
, WORDS_BIG_ENDIAN
? 0 : 1,
1126 mask
= expand_simple_binop (word_mode
, ASHIFTRT
, mask
,
1127 GEN_INT (BITS_PER_WORD
- 1),
1128 NULL_RTX
, 0, OPTAB_DIRECT
);
1129 if (mask
== NULL_RTX
)
1132 mask
= expand_simple_binop (word_mode
, AND
, mask
,
1133 gen_int_mode (1 - INTVAL (op1
),
1135 NULL_RTX
, 1, OPTAB_DIRECT
);
1136 if (mask
== NULL_RTX
)
1138 remainder
= expand_simple_binop (word_mode
, PLUS
, remainder
,
1139 mask
, NULL_RTX
, 1, OPTAB_DIRECT
);
1140 if (remainder
== NULL_RTX
)
1144 remainder
= convert_modes (mode
, word_mode
, remainder
, unsignedp
);
1145 /* Punt if we need any library calls. */
1147 last
= NEXT_INSN (last
);
1149 last
= get_insns ();
1150 for (; last
; last
= NEXT_INSN (last
))
1158 /* Similarly to the above function, but compute both quotient and remainder.
1159 Quotient can be computed from the remainder as:
1160 rem = op0 % op1; // Handled using expand_doubleword_mod
1161 quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1162 // 2 * BITS_PER_WORD
1164 We can also handle cases where op1 is a multiple of power of two constant
1165 and constant handled by expand_doubleword_mod.
1166 op11 = 1 << __builtin_ctz (op1);
1168 rem1 = op0 % op12; // Handled using expand_doubleword_mod
1169 quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1170 // 2 * BITS_PER_WORD
1171 rem = (quot1 % op11) * op12 + rem1;
1172 quot = quot1 / op11; */
1175 expand_doubleword_divmod (machine_mode mode
, rtx op0
, rtx op1
, rtx
*rem
,
1180 /* Negative dividend should have been optimized into positive,
1181 similarly modulo by 1 and modulo by power of two is optimized
1183 if (INTVAL (op1
) <= 1 || pow2p_hwi (INTVAL (op1
)))
1186 rtx op11
= const1_rtx
;
1188 if ((INTVAL (op1
) & 1) == 0)
1190 int bit
= ctz_hwi (INTVAL (op1
));
1191 op11
= GEN_INT (HOST_WIDE_INT_1
<< bit
);
1192 op12
= GEN_INT (INTVAL (op1
) >> bit
);
1195 rtx rem1
= expand_doubleword_mod (mode
, op0
, op12
, unsignedp
);
1196 if (rem1
== NULL_RTX
)
1199 int prec
= 2 * BITS_PER_WORD
;
1200 wide_int a
= wide_int::from (INTVAL (op12
), prec
+ 1, UNSIGNED
);
1201 wide_int b
= wi::shifted_mask (prec
, 1, false, prec
+ 1);
1202 wide_int m
= wide_int::from (wi::mod_inv (a
, b
), prec
, UNSIGNED
);
1203 rtx inv
= immed_wide_int_const (m
, mode
);
1205 rtx_insn
*last
= get_last_insn ();
1206 rtx quot1
= expand_simple_binop (mode
, MINUS
, op0
, rem1
,
1207 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1208 if (quot1
== NULL_RTX
)
1211 quot1
= expand_simple_binop (mode
, MULT
, quot1
, inv
,
1212 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1213 if (quot1
== NULL_RTX
)
1216 if (op11
!= const1_rtx
)
1218 rtx rem2
= expand_divmod (1, TRUNC_MOD_EXPR
, mode
, quot1
, op11
,
1219 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1220 if (rem2
== NULL_RTX
)
1223 rem2
= expand_simple_binop (mode
, MULT
, rem2
, op12
, NULL_RTX
,
1224 unsignedp
, OPTAB_DIRECT
);
1225 if (rem2
== NULL_RTX
)
1228 rem2
= expand_simple_binop (mode
, PLUS
, rem2
, rem1
, NULL_RTX
,
1229 unsignedp
, OPTAB_DIRECT
);
1230 if (rem2
== NULL_RTX
)
1233 rtx quot2
= expand_divmod (0, TRUNC_DIV_EXPR
, mode
, quot1
, op11
,
1234 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1235 if (quot2
== NULL_RTX
)
1242 /* Punt if we need any library calls. */
1244 last
= NEXT_INSN (last
);
1246 last
= get_insns ();
1247 for (; last
; last
= NEXT_INSN (last
))
1255 /* Wrapper around expand_binop which takes an rtx code to specify
1256 the operation to perform, not an optab pointer. All other
1257 arguments are the same. */
1259 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
1260 rtx op1
, rtx target
, int unsignedp
,
1261 enum optab_methods methods
)
1263 optab binop
= code_to_optab (code
);
1266 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
1269 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1270 binop. Order them according to commutative_operand_precedence and, if
1271 possible, try to put TARGET or a pseudo first. */
1273 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
1275 int op0_prec
= commutative_operand_precedence (op0
);
1276 int op1_prec
= commutative_operand_precedence (op1
);
1278 if (op0_prec
< op1_prec
)
1281 if (op0_prec
> op1_prec
)
1284 /* With equal precedence, both orders are ok, but it is better if the
1285 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1286 if (target
== 0 || REG_P (target
))
1287 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
1289 return rtx_equal_p (op1
, target
);
1292 /* Return true if BINOPTAB implements a shift operation. */
1295 shift_optab_p (optab binoptab
)
1297 switch (optab_to_code (binoptab
))
1313 /* Return true if BINOPTAB implements a commutative binary operation. */
1316 commutative_optab_p (optab binoptab
)
1318 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
1319 || binoptab
== smul_widen_optab
1320 || binoptab
== umul_widen_optab
1321 || binoptab
== smul_highpart_optab
1322 || binoptab
== umul_highpart_optab
1323 || binoptab
== vec_widen_sadd_optab
1324 || binoptab
== vec_widen_uadd_optab
1325 || binoptab
== vec_widen_sadd_hi_optab
1326 || binoptab
== vec_widen_sadd_lo_optab
1327 || binoptab
== vec_widen_uadd_hi_optab
1328 || binoptab
== vec_widen_uadd_lo_optab
1329 || binoptab
== vec_widen_sadd_even_optab
1330 || binoptab
== vec_widen_sadd_odd_optab
1331 || binoptab
== vec_widen_uadd_even_optab
1332 || binoptab
== vec_widen_uadd_odd_optab
);
1335 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1336 optimizing, and if the operand is a constant that costs more than
1337 1 instruction, force the constant into a register and return that
1338 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1341 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
1342 int opn
, rtx x
, bool unsignedp
)
1344 bool speed
= optimize_insn_for_speed_p ();
1346 if (mode
!= VOIDmode
1349 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
1350 > set_src_cost (x
, mode
, speed
)))
1352 if (CONST_INT_P (x
))
1354 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
1355 if (intval
!= INTVAL (x
))
1356 x
= GEN_INT (intval
);
1359 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
1360 x
= force_reg (mode
, x
);
1365 /* Helper function for expand_binop: handle the case where there
1366 is an insn ICODE that directly implements the indicated operation.
1367 Returns null if this is not possible. */
1369 expand_binop_directly (enum insn_code icode
, machine_mode mode
, optab binoptab
,
1371 rtx target
, int unsignedp
, enum optab_methods methods
,
1374 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
1375 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
1376 machine_mode mode0
, mode1
, tmp_mode
;
1377 class expand_operand ops
[3];
1380 rtx xop0
= op0
, xop1
= op1
;
1381 bool canonicalize_op1
= false;
1383 /* If it is a commutative operator and the modes would match
1384 if we would swap the operands, we can save the conversions. */
1385 commutative_p
= commutative_optab_p (binoptab
);
1387 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1388 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode0
)
1389 std::swap (xop0
, xop1
);
1391 /* If we are optimizing, force expensive constants into a register. */
1392 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1393 if (!shift_optab_p (binoptab
))
1394 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1396 /* Shifts and rotates often use a different mode for op1 from op0;
1397 for VOIDmode constants we don't know the mode, so force it
1398 to be canonicalized using convert_modes. */
1399 canonicalize_op1
= true;
1401 /* In case the insn wants input operands in modes different from
1402 those of the actual operands, convert the operands. It would
1403 seem that we don't need to convert CONST_INTs, but we do, so
1404 that they're properly zero-extended, sign-extended or truncated
1407 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1408 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1410 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1414 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1415 ? GET_MODE (xop1
) : mode
);
1416 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1418 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1422 /* If operation is commutative,
1423 try to make the first operand a register.
1424 Even better, try to make it the same as the target.
1425 Also try to make the last operand a constant. */
1427 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1428 std::swap (xop0
, xop1
);
1430 /* Now, if insn's predicates don't allow our operands, put them into
1433 if (binoptab
== vec_pack_trunc_optab
1434 || binoptab
== vec_pack_usat_optab
1435 || binoptab
== vec_pack_ssat_optab
1436 || binoptab
== vec_pack_ufix_trunc_optab
1437 || binoptab
== vec_pack_sfix_trunc_optab
1438 || binoptab
== vec_packu_float_optab
1439 || binoptab
== vec_packs_float_optab
)
1441 /* The mode of the result is different then the mode of the
1443 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1444 if (VECTOR_MODE_P (mode
)
1445 && maybe_ne (GET_MODE_NUNITS (tmp_mode
), 2 * GET_MODE_NUNITS (mode
)))
1447 delete_insns_since (last
);
1454 create_output_operand (&ops
[0], target
, tmp_mode
);
1455 create_input_operand (&ops
[1], xop0
, mode0
);
1456 create_input_operand (&ops
[2], xop1
, mode1
);
1457 pat
= maybe_gen_insn (icode
, 3, ops
);
1460 /* If PAT is composed of more than one insn, try to add an appropriate
1461 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1462 operand, call expand_binop again, this time without a target. */
1463 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1464 && ! add_equal_note (pat
, ops
[0].value
,
1465 optab_to_code (binoptab
),
1466 ops
[1].value
, ops
[2].value
, mode0
))
1468 delete_insns_since (last
);
1469 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1470 unsignedp
, methods
);
1474 return ops
[0].value
;
1476 delete_insns_since (last
);
1480 /* Generate code to perform an operation specified by BINOPTAB
1481 on operands OP0 and OP1, with result having machine-mode MODE.
1483 UNSIGNEDP is for the case where we have to widen the operands
1484 to perform the operation. It says to use zero-extension.
1486 If TARGET is nonzero, the value
1487 is generated there, if it is convenient to do so.
1488 In all cases an rtx is returned for the locus of the value;
1489 this may or may not be TARGET. */
1492 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1493 rtx target
, int unsignedp
, enum optab_methods methods
)
1495 enum optab_methods next_methods
1496 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1497 ? OPTAB_WIDEN
: methods
);
1498 enum mode_class mclass
;
1499 enum insn_code icode
;
1500 machine_mode wider_mode
;
1501 scalar_int_mode int_mode
;
1504 rtx_insn
*entry_last
= get_last_insn ();
1507 mclass
= GET_MODE_CLASS (mode
);
1509 /* If subtracting an integer constant, convert this into an addition of
1510 the negated constant. */
1512 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1514 op1
= negate_rtx (mode
, op1
);
1515 binoptab
= add_optab
;
1517 /* For shifts, constant invalid op1 might be expanded from different
1518 mode than MODE. As those are invalid, force them to a register
1519 to avoid further problems during expansion. */
1520 else if (CONST_INT_P (op1
)
1521 && shift_optab_p (binoptab
)
1522 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1524 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1525 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1528 /* Record where to delete back to if we backtrack. */
1529 last
= get_last_insn ();
1531 /* If we can do it with a three-operand insn, do so. */
1533 if (methods
!= OPTAB_MUST_WIDEN
)
1535 if (convert_optab_p (binoptab
))
1537 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
1538 icode
= find_widening_optab_handler (binoptab
, mode
, from_mode
);
1541 icode
= optab_handler (binoptab
, mode
);
1542 if (icode
!= CODE_FOR_nothing
)
1544 temp
= expand_binop_directly (icode
, mode
, binoptab
, op0
, op1
,
1545 target
, unsignedp
, methods
, last
);
1551 /* If we were trying to rotate, and that didn't work, try rotating
1552 the other direction before falling back to shifts and bitwise-or. */
1553 if (((binoptab
== rotl_optab
1554 && (icode
= optab_handler (rotr_optab
, mode
)) != CODE_FOR_nothing
)
1555 || (binoptab
== rotr_optab
1556 && (icode
= optab_handler (rotl_optab
, mode
)) != CODE_FOR_nothing
))
1557 && is_int_mode (mode
, &int_mode
))
1559 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1561 unsigned int bits
= GET_MODE_PRECISION (int_mode
);
1563 if (CONST_INT_P (op1
))
1564 newop1
= gen_int_shift_amount (int_mode
, bits
- INTVAL (op1
));
1565 else if (targetm
.shift_truncation_mask (int_mode
) == bits
- 1)
1566 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1568 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1569 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1570 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1572 temp
= expand_binop_directly (icode
, int_mode
, otheroptab
, op0
, newop1
,
1573 target
, unsignedp
, methods
, last
);
1578 /* If this is a multiply, see if we can do a widening operation that
1579 takes operands of this mode and makes a wider mode. */
1581 if (binoptab
== smul_optab
1582 && GET_MODE_2XWIDER_MODE (mode
).exists (&wider_mode
)
1583 && (convert_optab_handler ((unsignedp
1585 : smul_widen_optab
),
1586 wider_mode
, mode
) != CODE_FOR_nothing
))
1588 /* *_widen_optab needs to determine operand mode, make sure at least
1589 one operand has non-VOID mode. */
1590 if (GET_MODE (op0
) == VOIDmode
&& GET_MODE (op1
) == VOIDmode
)
1591 op0
= force_reg (mode
, op0
);
1592 temp
= expand_binop (wider_mode
,
1593 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1594 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1598 if (GET_MODE_CLASS (mode
) == MODE_INT
1599 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1600 return gen_lowpart (mode
, temp
);
1602 return convert_to_mode (mode
, temp
, unsignedp
);
1606 /* If this is a vector shift by a scalar, see if we can do a vector
1607 shift by a vector. If so, broadcast the scalar into a vector. */
1608 if (mclass
== MODE_VECTOR_INT
)
1610 optab otheroptab
= unknown_optab
;
1612 if (binoptab
== ashl_optab
)
1613 otheroptab
= vashl_optab
;
1614 else if (binoptab
== ashr_optab
)
1615 otheroptab
= vashr_optab
;
1616 else if (binoptab
== lshr_optab
)
1617 otheroptab
= vlshr_optab
;
1618 else if (binoptab
== rotl_optab
)
1619 otheroptab
= vrotl_optab
;
1620 else if (binoptab
== rotr_optab
)
1621 otheroptab
= vrotr_optab
;
1624 && (icode
= optab_handler (otheroptab
, mode
)) != CODE_FOR_nothing
)
1626 /* The scalar may have been extended to be too wide. Truncate
1627 it back to the proper size to fit in the broadcast vector. */
1628 scalar_mode inner_mode
= GET_MODE_INNER (mode
);
1629 if (!CONST_INT_P (op1
)
1630 && (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (op1
)))
1631 > GET_MODE_BITSIZE (inner_mode
)))
1632 op1
= force_reg (inner_mode
,
1633 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1635 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1638 temp
= expand_binop_directly (icode
, mode
, otheroptab
, op0
, vop1
,
1639 target
, unsignedp
, methods
, last
);
1646 /* Look for a wider mode of the same class for which we think we
1647 can open-code the operation. Check for a widening multiply at the
1648 wider mode as well. */
1650 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1651 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1652 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1654 machine_mode next_mode
;
1655 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1656 || (binoptab
== smul_optab
1657 && GET_MODE_WIDER_MODE (wider_mode
).exists (&next_mode
)
1658 && (find_widening_optab_handler ((unsignedp
1660 : smul_widen_optab
),
1662 != CODE_FOR_nothing
)))
1664 rtx xop0
= op0
, xop1
= op1
;
1665 bool no_extend
= false;
1667 /* For certain integer operations, we need not actually extend
1668 the narrow operands, as long as we will truncate
1669 the results to the same narrowness. */
1671 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1672 || binoptab
== xor_optab
1673 || binoptab
== add_optab
|| binoptab
== sub_optab
1674 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1675 && mclass
== MODE_INT
)
1678 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1680 if (binoptab
!= ashl_optab
)
1681 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1685 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1687 /* The second operand of a shift must always be extended. */
1688 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1689 no_extend
&& binoptab
!= ashl_optab
);
1691 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1692 unsignedp
, OPTAB_DIRECT
);
1695 if (mclass
!= MODE_INT
1696 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1699 target
= gen_reg_rtx (mode
);
1700 convert_move (target
, temp
, 0);
1704 return gen_lowpart (mode
, temp
);
1707 delete_insns_since (last
);
1711 /* If operation is commutative,
1712 try to make the first operand a register.
1713 Even better, try to make it the same as the target.
1714 Also try to make the last operand a constant. */
1715 if (commutative_optab_p (binoptab
)
1716 && swap_commutative_operands_with_target (target
, op0
, op1
))
1717 std::swap (op0
, op1
);
1719 /* These can be done a word at a time. */
1720 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1721 && is_int_mode (mode
, &int_mode
)
1722 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
1723 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1728 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1729 won't be accurate, so use a new target. */
1733 || reg_overlap_mentioned_p (target
, op0
)
1734 || reg_overlap_mentioned_p (target
, op1
)
1735 || !valid_multiword_target_p (target
))
1736 target
= gen_reg_rtx (int_mode
);
1740 /* Do the actual arithmetic. */
1741 machine_mode op0_mode
= GET_MODE (op0
);
1742 machine_mode op1_mode
= GET_MODE (op1
);
1743 if (op0_mode
== VOIDmode
)
1744 op0_mode
= int_mode
;
1745 if (op1_mode
== VOIDmode
)
1746 op1_mode
= int_mode
;
1747 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
1749 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
1750 rtx x
= expand_binop (word_mode
, binoptab
,
1751 operand_subword_force (op0
, i
, op0_mode
),
1752 operand_subword_force (op1
, i
, op1_mode
),
1753 target_piece
, unsignedp
, next_methods
);
1758 if (target_piece
!= x
)
1759 emit_move_insn (target_piece
, x
);
1762 insns
= get_insns ();
1765 if (i
== GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
)
1772 /* Synthesize double word shifts from single word shifts. */
1773 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1774 || binoptab
== ashr_optab
)
1775 && is_int_mode (mode
, &int_mode
)
1776 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1777 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1778 && GET_MODE_PRECISION (int_mode
) == GET_MODE_BITSIZE (int_mode
)
1779 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1780 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1781 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1783 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1784 scalar_int_mode op1_mode
;
1786 double_shift_mask
= targetm
.shift_truncation_mask (int_mode
);
1787 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1788 op1_mode
= (GET_MODE (op1
) != VOIDmode
1789 ? as_a
<scalar_int_mode
> (GET_MODE (op1
))
1792 /* Apply the truncation to constant shifts. */
1793 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1794 op1
= gen_int_mode (INTVAL (op1
) & double_shift_mask
, op1_mode
);
1796 if (op1
== CONST0_RTX (op1_mode
))
1799 /* Make sure that this is a combination that expand_doubleword_shift
1800 can handle. See the comments there for details. */
1801 if (double_shift_mask
== 0
1802 || (shift_mask
== BITS_PER_WORD
- 1
1803 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1806 rtx into_target
, outof_target
;
1807 rtx into_input
, outof_input
;
1808 int left_shift
, outof_word
;
1810 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1811 won't be accurate, so use a new target. */
1815 || reg_overlap_mentioned_p (target
, op0
)
1816 || reg_overlap_mentioned_p (target
, op1
)
1817 || !valid_multiword_target_p (target
))
1818 target
= gen_reg_rtx (int_mode
);
1822 /* OUTOF_* is the word we are shifting bits away from, and
1823 INTO_* is the word that we are shifting bits towards, thus
1824 they differ depending on the direction of the shift and
1825 WORDS_BIG_ENDIAN. */
1827 left_shift
= binoptab
== ashl_optab
;
1828 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1830 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1831 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1833 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1834 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1836 if (expand_doubleword_shift (op1_mode
, binoptab
,
1837 outof_input
, into_input
, op1
,
1838 outof_target
, into_target
,
1839 unsignedp
, next_methods
, shift_mask
))
1841 insns
= get_insns ();
1851 /* Synthesize double word rotates from single word shifts. */
1852 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1853 && is_int_mode (mode
, &int_mode
)
1854 && CONST_INT_P (op1
)
1855 && GET_MODE_PRECISION (int_mode
) == 2 * BITS_PER_WORD
1856 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1857 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1860 rtx into_target
, outof_target
;
1861 rtx into_input
, outof_input
;
1863 int shift_count
, left_shift
, outof_word
;
1865 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1866 won't be accurate, so use a new target. Do this also if target is not
1867 a REG, first because having a register instead may open optimization
1868 opportunities, and second because if target and op0 happen to be MEMs
1869 designating the same location, we would risk clobbering it too early
1870 in the code sequence we generate below. */
1875 || reg_overlap_mentioned_p (target
, op0
)
1876 || reg_overlap_mentioned_p (target
, op1
)
1877 || !valid_multiword_target_p (target
))
1878 target
= gen_reg_rtx (int_mode
);
1882 shift_count
= INTVAL (op1
);
1884 /* OUTOF_* is the word we are shifting bits away from, and
1885 INTO_* is the word that we are shifting bits towards, thus
1886 they differ depending on the direction of the shift and
1887 WORDS_BIG_ENDIAN. */
1889 left_shift
= (binoptab
== rotl_optab
);
1890 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1892 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1893 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1895 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1896 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1898 if (shift_count
== BITS_PER_WORD
)
1900 /* This is just a word swap. */
1901 emit_move_insn (outof_target
, into_input
);
1902 emit_move_insn (into_target
, outof_input
);
1907 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1908 HOST_WIDE_INT first_shift_count
, second_shift_count
;
1909 optab reverse_unsigned_shift
, unsigned_shift
;
1911 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1912 ? lshr_optab
: ashl_optab
);
1914 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1915 ? ashl_optab
: lshr_optab
);
1917 if (shift_count
> BITS_PER_WORD
)
1919 first_shift_count
= shift_count
- BITS_PER_WORD
;
1920 second_shift_count
= 2 * BITS_PER_WORD
- shift_count
;
1924 first_shift_count
= BITS_PER_WORD
- shift_count
;
1925 second_shift_count
= shift_count
;
1927 rtx first_shift_count_rtx
1928 = gen_int_shift_amount (word_mode
, first_shift_count
);
1929 rtx second_shift_count_rtx
1930 = gen_int_shift_amount (word_mode
, second_shift_count
);
1932 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1933 outof_input
, first_shift_count_rtx
,
1934 NULL_RTX
, unsignedp
, next_methods
);
1935 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1936 into_input
, second_shift_count_rtx
,
1937 NULL_RTX
, unsignedp
, next_methods
);
1939 if (into_temp1
!= 0 && into_temp2
!= 0)
1940 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1941 into_target
, unsignedp
, next_methods
);
1945 if (inter
!= 0 && inter
!= into_target
)
1946 emit_move_insn (into_target
, inter
);
1948 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1949 into_input
, first_shift_count_rtx
,
1950 NULL_RTX
, unsignedp
, next_methods
);
1951 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1952 outof_input
, second_shift_count_rtx
,
1953 NULL_RTX
, unsignedp
, next_methods
);
1955 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1956 inter
= expand_binop (word_mode
, ior_optab
,
1957 outof_temp1
, outof_temp2
,
1958 outof_target
, unsignedp
, next_methods
);
1960 if (inter
!= 0 && inter
!= outof_target
)
1961 emit_move_insn (outof_target
, inter
);
1964 insns
= get_insns ();
1974 /* These can be done a word at a time by propagating carries. */
1975 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1976 && is_int_mode (mode
, &int_mode
)
1977 && GET_MODE_SIZE (int_mode
) >= 2 * UNITS_PER_WORD
1978 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1981 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1982 const unsigned int nwords
= GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
;
1983 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1984 rtx xop0
, xop1
, xtarget
;
1986 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1987 value is one of those, use it. Otherwise, use 1 since it is the
1988 one easiest to get. */
1989 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1990 int normalizep
= STORE_FLAG_VALUE
;
1995 /* Prepare the operands. */
1996 xop0
= force_reg (int_mode
, op0
);
1997 xop1
= force_reg (int_mode
, op1
);
1999 xtarget
= gen_reg_rtx (int_mode
);
2001 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
2004 /* Indicate for flow that the entire target reg is being set. */
2006 emit_clobber (xtarget
);
2008 /* Do the actual arithmetic. */
2009 for (i
= 0; i
< nwords
; i
++)
2011 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
2012 rtx target_piece
= operand_subword (xtarget
, index
, 1, int_mode
);
2013 rtx op0_piece
= operand_subword_force (xop0
, index
, int_mode
);
2014 rtx op1_piece
= operand_subword_force (xop1
, index
, int_mode
);
2017 /* Main add/subtract of the input operands. */
2018 x
= expand_binop (word_mode
, binoptab
,
2019 op0_piece
, op1_piece
,
2020 target_piece
, unsignedp
, next_methods
);
2026 /* Store carry from main add/subtract. */
2027 carry_out
= gen_reg_rtx (word_mode
);
2028 carry_out
= emit_store_flag_force (carry_out
,
2029 (binoptab
== add_optab
2032 word_mode
, 1, normalizep
);
2039 /* Add/subtract previous carry to main result. */
2040 newx
= expand_binop (word_mode
,
2041 normalizep
== 1 ? binoptab
: otheroptab
,
2043 NULL_RTX
, 1, next_methods
);
2047 /* Get out carry from adding/subtracting carry in. */
2048 rtx carry_tmp
= gen_reg_rtx (word_mode
);
2049 carry_tmp
= emit_store_flag_force (carry_tmp
,
2050 (binoptab
== add_optab
2053 word_mode
, 1, normalizep
);
2055 /* Logical-ior the two poss. carry together. */
2056 carry_out
= expand_binop (word_mode
, ior_optab
,
2057 carry_out
, carry_tmp
,
2058 carry_out
, 0, next_methods
);
2062 emit_move_insn (target_piece
, newx
);
2066 if (x
!= target_piece
)
2067 emit_move_insn (target_piece
, x
);
2070 carry_in
= carry_out
;
2073 if (i
== GET_MODE_BITSIZE (int_mode
) / (unsigned) BITS_PER_WORD
)
2075 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
2076 || ! rtx_equal_p (target
, xtarget
))
2078 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
2080 set_dst_reg_note (temp
, REG_EQUAL
,
2081 gen_rtx_fmt_ee (optab_to_code (binoptab
),
2082 int_mode
, copy_rtx (xop0
),
2093 delete_insns_since (last
);
2096 /* Attempt to synthesize double word multiplies using a sequence of word
2097 mode multiplications. We first attempt to generate a sequence using a
2098 more efficient unsigned widening multiply, and if that fails we then
2099 try using a signed widening multiply. */
2101 if (binoptab
== smul_optab
2102 && is_int_mode (mode
, &int_mode
)
2103 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2104 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
2105 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
2107 rtx product
= NULL_RTX
;
2108 if (convert_optab_handler (umul_widen_optab
, int_mode
, word_mode
)
2109 != CODE_FOR_nothing
)
2111 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
2114 delete_insns_since (last
);
2117 if (product
== NULL_RTX
2118 && (convert_optab_handler (smul_widen_optab
, int_mode
, word_mode
)
2119 != CODE_FOR_nothing
))
2121 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
2124 delete_insns_since (last
);
2127 if (product
!= NULL_RTX
)
2129 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
2131 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
2133 set_dst_reg_note (move
,
2135 gen_rtx_fmt_ee (MULT
, int_mode
,
2138 target
? target
: product
);
2144 /* Attempt to synthetize double word modulo by constant divisor. */
2145 if ((binoptab
== umod_optab
2146 || binoptab
== smod_optab
2147 || binoptab
== udiv_optab
2148 || binoptab
== sdiv_optab
)
2150 && CONST_INT_P (op1
)
2151 && is_int_mode (mode
, &int_mode
)
2152 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2153 && optab_handler ((binoptab
== umod_optab
|| binoptab
== udiv_optab
)
2154 ? udivmod_optab
: sdivmod_optab
,
2155 int_mode
) == CODE_FOR_nothing
2156 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
2157 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
2158 && optimize_insn_for_speed_p ())
2161 if ((binoptab
== umod_optab
|| binoptab
== smod_optab
)
2162 && (INTVAL (op1
) & 1) == 0)
2163 res
= expand_doubleword_mod (int_mode
, op0
, op1
,
2164 binoptab
== umod_optab
);
2167 rtx quot
= expand_doubleword_divmod (int_mode
, op0
, op1
, &res
,
2168 binoptab
== umod_optab
2169 || binoptab
== udiv_optab
);
2170 if (quot
== NULL_RTX
)
2172 else if (binoptab
== udiv_optab
|| binoptab
== sdiv_optab
)
2175 if (res
!= NULL_RTX
)
2177 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
2179 rtx_insn
*move
= emit_move_insn (target
? target
: res
,
2181 set_dst_reg_note (move
, REG_EQUAL
,
2182 gen_rtx_fmt_ee (optab_to_code (binoptab
),
2183 int_mode
, copy_rtx (op0
), op1
),
2184 target
? target
: res
);
2189 delete_insns_since (last
);
2192 /* It can't be open-coded in this mode.
2193 Use a library call if one is available and caller says that's ok. */
2195 libfunc
= optab_libfunc (binoptab
, mode
);
2197 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
2201 machine_mode op1_mode
= mode
;
2206 if (shift_optab_p (binoptab
))
2208 op1_mode
= targetm
.libgcc_shift_count_mode ();
2209 /* Specify unsigned here,
2210 since negative shift counts are meaningless. */
2211 op1x
= convert_to_mode (op1_mode
, op1
, 1);
2214 if (GET_MODE (op0
) != VOIDmode
2215 && GET_MODE (op0
) != mode
)
2216 op0
= convert_to_mode (mode
, op0
, unsignedp
);
2218 /* Pass 1 for NO_QUEUE so we don't lose any increments
2219 if the libcall is cse'd or moved. */
2220 value
= emit_library_call_value (libfunc
,
2221 NULL_RTX
, LCT_CONST
, mode
,
2222 op0
, mode
, op1x
, op1_mode
);
2224 insns
= get_insns ();
2227 bool trapv
= trapv_binoptab_p (binoptab
);
2228 target
= gen_reg_rtx (mode
);
2229 emit_libcall_block_1 (insns
, target
, value
,
2231 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
2232 mode
, op0
, op1
), trapv
);
2237 delete_insns_since (last
);
2239 /* It can't be done in this mode. Can we do it in a wider mode? */
2241 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
2242 || methods
== OPTAB_MUST_WIDEN
))
2244 /* Caller says, don't even try. */
2245 delete_insns_since (entry_last
);
2249 /* Compute the value of METHODS to pass to recursive calls.
2250 Don't allow widening to be tried recursively. */
2252 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
2254 /* Look for a wider mode of the same class for which it appears we can do
2257 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2259 /* This code doesn't make sense for conversion optabs, since we
2260 wouldn't then want to extend the operands to be the same size
2262 gcc_assert (!convert_optab_p (binoptab
));
2263 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2265 if (optab_handler (binoptab
, wider_mode
)
2266 || (methods
== OPTAB_LIB
2267 && optab_libfunc (binoptab
, wider_mode
)))
2269 rtx xop0
= op0
, xop1
= op1
;
2270 bool no_extend
= false;
2272 /* For certain integer operations, we need not actually extend
2273 the narrow operands, as long as we will truncate
2274 the results to the same narrowness. */
2276 if ((binoptab
== ior_optab
|| binoptab
== and_optab
2277 || binoptab
== xor_optab
2278 || binoptab
== add_optab
|| binoptab
== sub_optab
2279 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
2280 && mclass
== MODE_INT
)
2283 xop0
= widen_operand (xop0
, wider_mode
, mode
,
2284 unsignedp
, no_extend
);
2286 /* The second operand of a shift must always be extended. */
2287 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
2288 no_extend
&& binoptab
!= ashl_optab
);
2290 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
2291 unsignedp
, methods
);
2294 if (mclass
!= MODE_INT
2295 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2298 target
= gen_reg_rtx (mode
);
2299 convert_move (target
, temp
, 0);
2303 return gen_lowpart (mode
, temp
);
2306 delete_insns_since (last
);
2311 delete_insns_since (entry_last
);
2315 /* Expand a binary operator which has both signed and unsigned forms.
2316 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2319 If we widen unsigned operands, we may use a signed wider operation instead
2320 of an unsigned wider operation, since the result would be the same. */
2323 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
2324 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
2325 enum optab_methods methods
)
2328 optab direct_optab
= unsignedp
? uoptab
: soptab
;
2331 /* Do it without widening, if possible. */
2332 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2333 unsignedp
, OPTAB_DIRECT
);
2334 if (temp
|| methods
== OPTAB_DIRECT
)
2337 /* Try widening to a signed int. Disable any direct use of any
2338 signed insn in the current mode. */
2339 save_enable
= swap_optab_enable (soptab
, mode
, false);
2341 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2342 unsignedp
, OPTAB_WIDEN
);
2344 /* For unsigned operands, try widening to an unsigned int. */
2345 if (!temp
&& unsignedp
)
2346 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2347 unsignedp
, OPTAB_WIDEN
);
2348 if (temp
|| methods
== OPTAB_WIDEN
)
2351 /* Use the right width libcall if that exists. */
2352 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2353 unsignedp
, OPTAB_LIB
);
2354 if (temp
|| methods
== OPTAB_LIB
)
2357 /* Must widen and use a libcall, use either signed or unsigned. */
2358 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2359 unsignedp
, methods
);
2360 if (!temp
&& unsignedp
)
2361 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2362 unsignedp
, methods
);
2365 /* Undo the fiddling above. */
2367 swap_optab_enable (soptab
, mode
, true);
2371 /* Generate code to perform an operation specified by UNOPPTAB
2372 on operand OP0, with two results to TARG0 and TARG1.
2373 We assume that the order of the operands for the instruction
2374 is TARG0, TARG1, OP0.
2376 Either TARG0 or TARG1 may be zero, but what that means is that
2377 the result is not actually wanted. We will generate it into
2378 a dummy pseudo-reg and discard it. They may not both be zero.
2380 Returns true if this operation can be performed; false if not. */
2383 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
2386 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2387 enum mode_class mclass
;
2388 machine_mode wider_mode
;
2389 rtx_insn
*entry_last
= get_last_insn ();
2392 mclass
= GET_MODE_CLASS (mode
);
2395 targ0
= gen_reg_rtx (mode
);
2397 targ1
= gen_reg_rtx (mode
);
2399 /* Record where to go back to if we fail. */
2400 last
= get_last_insn ();
2402 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2404 class expand_operand ops
[3];
2405 enum insn_code icode
= optab_handler (unoptab
, mode
);
2407 create_fixed_operand (&ops
[0], targ0
);
2408 create_fixed_operand (&ops
[1], targ1
);
2409 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
2410 if (maybe_expand_insn (icode
, 3, ops
))
2414 /* It can't be done in this mode. Can we do it in a wider mode? */
2416 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2418 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2420 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2422 rtx t0
= gen_reg_rtx (wider_mode
);
2423 rtx t1
= gen_reg_rtx (wider_mode
);
2424 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2426 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
2428 convert_move (targ0
, t0
, unsignedp
);
2429 convert_move (targ1
, t1
, unsignedp
);
2433 delete_insns_since (last
);
2438 delete_insns_since (entry_last
);
2442 /* Generate code to perform an operation specified by BINOPTAB
2443 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2444 We assume that the order of the operands for the instruction
2445 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2446 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2448 Either TARG0 or TARG1 may be zero, but what that means is that
2449 the result is not actually wanted. We will generate it into
2450 a dummy pseudo-reg and discard it. They may not both be zero.
2452 Returns true if this operation can be performed; false if not. */
2455 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
2458 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2459 enum mode_class mclass
;
2460 machine_mode wider_mode
;
2461 rtx_insn
*entry_last
= get_last_insn ();
2464 mclass
= GET_MODE_CLASS (mode
);
2467 targ0
= gen_reg_rtx (mode
);
2469 targ1
= gen_reg_rtx (mode
);
2471 /* Record where to go back to if we fail. */
2472 last
= get_last_insn ();
2474 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2476 class expand_operand ops
[4];
2477 enum insn_code icode
= optab_handler (binoptab
, mode
);
2478 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2479 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2480 rtx xop0
= op0
, xop1
= op1
;
2482 /* If we are optimizing, force expensive constants into a register. */
2483 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2484 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2486 create_fixed_operand (&ops
[0], targ0
);
2487 create_convert_operand_from (&ops
[1], xop0
, mode
, unsignedp
);
2488 create_convert_operand_from (&ops
[2], xop1
, mode
, unsignedp
);
2489 create_fixed_operand (&ops
[3], targ1
);
2490 if (maybe_expand_insn (icode
, 4, ops
))
2492 delete_insns_since (last
);
2495 /* It can't be done in this mode. Can we do it in a wider mode? */
2497 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2499 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2501 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2503 rtx t0
= gen_reg_rtx (wider_mode
);
2504 rtx t1
= gen_reg_rtx (wider_mode
);
2505 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2506 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2508 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2511 convert_move (targ0
, t0
, unsignedp
);
2512 convert_move (targ1
, t1
, unsignedp
);
2516 delete_insns_since (last
);
2521 delete_insns_since (entry_last
);
2525 /* Expand the two-valued library call indicated by BINOPTAB, but
2526 preserve only one of the values. If TARG0 is non-NULL, the first
2527 value is placed into TARG0; otherwise the second value is placed
2528 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2529 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2530 This routine assumes that the value returned by the library call is
2531 as if the return value was of an integral mode twice as wide as the
2532 mode of OP0. Returns 1 if the call was successful. */
2535 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2536 rtx targ0
, rtx targ1
, enum rtx_code code
)
2539 machine_mode libval_mode
;
2544 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2545 gcc_assert (!targ0
!= !targ1
);
2547 mode
= GET_MODE (op0
);
2548 libfunc
= optab_libfunc (binoptab
, mode
);
2552 /* The value returned by the library function will have twice as
2553 many bits as the nominal MODE. */
2554 libval_mode
= smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode
));
2556 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2560 /* Get the part of VAL containing the value that we want. */
2561 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2562 targ0
? 0 : GET_MODE_SIZE (mode
));
2563 insns
= get_insns ();
2565 /* Move the into the desired location. */
2566 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2567 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2573 /* Wrapper around expand_unop which takes an rtx code to specify
2574 the operation to perform, not an optab pointer. All other
2575 arguments are the same. */
2577 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2578 rtx target
, int unsignedp
)
2580 optab unop
= code_to_optab (code
);
2583 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2589 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2591 A similar operation can be used for clrsb. UNOPTAB says which operation
2592 we are trying to expand. */
2594 widen_leading (scalar_int_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2596 opt_scalar_int_mode wider_mode_iter
;
2597 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2599 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2600 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2605 last
= get_last_insn ();
2608 target
= gen_reg_rtx (mode
);
2609 xop0
= widen_operand (op0
, wider_mode
, mode
,
2610 unoptab
!= clrsb_optab
, false);
2611 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2612 unoptab
!= clrsb_optab
);
2615 (wider_mode
, sub_optab
, temp
,
2616 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2617 - GET_MODE_PRECISION (mode
),
2619 target
, true, OPTAB_DIRECT
);
2621 delete_insns_since (last
);
2629 /* Attempt to emit (clrsb:mode op0) as
2630 (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2632 if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2634 (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2635 (ashr:mode op0 (const_int prec-1)))
2640 expand_clrsb_using_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2642 if (optimize_insn_for_size_p ()
2643 || optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2647 HOST_WIDE_INT val
= 0;
2648 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) != 2
2649 || val
!= GET_MODE_PRECISION (mode
))
2657 temp2
= expand_binop (mode
, ashl_optab
, op0
, const1_rtx
,
2658 NULL_RTX
, 0, OPTAB_DIRECT
);
2667 rtx temp
= expand_binop (mode
, ashr_optab
, op0
,
2668 GEN_INT (GET_MODE_PRECISION (mode
) - 1),
2669 NULL_RTX
, 0, OPTAB_DIRECT
);
2673 temp
= expand_binop (mode
, xor_optab
, temp2
, temp
, NULL_RTX
, 0,
2680 temp
= expand_binop (mode
, ior_optab
, temp
, const1_rtx
,
2681 NULL_RTX
, 0, OPTAB_DIRECT
);
2685 temp
= expand_unop_direct (mode
, clz_optab
, temp
, val
? NULL_RTX
: target
,
2691 temp
= expand_binop (mode
, add_optab
, temp
, constm1_rtx
,
2692 target
, 0, OPTAB_DIRECT
);
2697 rtx_insn
*seq
= get_insns ();
2700 add_equal_note (seq
, temp
, CLRSB
, op0
, NULL_RTX
, mode
);
2705 static rtx
expand_ffs (scalar_int_mode
, rtx
, rtx
);
2707 /* Try calculating clz, ctz or ffs of a double-word quantity as two clz, ctz or
2708 ffs operations on word-sized quantities, choosing which based on whether the
2709 high (for clz) or low (for ctz and ffs) word is nonzero. */
2711 expand_doubleword_clz_ctz_ffs (scalar_int_mode mode
, rtx op0
, rtx target
,
2714 rtx xop0
= force_reg (mode
, op0
);
2715 rtx subhi
= gen_highpart (word_mode
, xop0
);
2716 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2717 rtx_code_label
*hi0_label
= gen_label_rtx ();
2718 rtx_code_label
*after_label
= gen_label_rtx ();
2723 /* If we were not given a target, use a word_mode register, not a
2724 'mode' register. The result will fit, and nobody is expecting
2725 anything bigger (the return type of __builtin_clz* is int). */
2727 target
= gen_reg_rtx (word_mode
);
2729 /* In any case, write to a word_mode scratch in both branches of the
2730 conditional, so we can ensure there is a single move insn setting
2731 'target' to tag a REG_EQUAL note on. */
2732 result
= gen_reg_rtx (word_mode
);
2734 if (unoptab
!= clz_optab
)
2735 std::swap (subhi
, sublo
);
2739 /* If the high word is not equal to zero,
2740 then clz of the full value is clz of the high word. */
2741 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2742 word_mode
, true, hi0_label
);
2744 if (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2745 temp
= expand_unop_direct (word_mode
, unoptab
, subhi
, result
, true);
2748 gcc_assert (unoptab
== ffs_optab
);
2749 temp
= expand_ffs (word_mode
, subhi
, result
);
2755 convert_move (result
, temp
, true);
2757 emit_jump_insn (targetm
.gen_jump (after_label
));
2760 /* Else clz of the full value is clz of the low word plus the number
2761 of bits in the high word. Similarly for ctz/ffs of the high word,
2762 except that ffs should be 0 when both words are zero. */
2763 emit_label (hi0_label
);
2765 if (unoptab
== ffs_optab
)
2767 convert_move (result
, const0_rtx
, true);
2768 emit_cmp_and_jump_insns (sublo
, CONST0_RTX (word_mode
), EQ
, 0,
2769 word_mode
, true, after_label
);
2772 if (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2773 temp
= expand_unop_direct (word_mode
, unoptab
, sublo
, NULL_RTX
, true);
2776 gcc_assert (unoptab
== ffs_optab
);
2777 temp
= expand_unop_direct (word_mode
, ctz_optab
, sublo
, NULL_RTX
, true);
2784 temp
= expand_binop (word_mode
, add_optab
, temp
,
2785 gen_int_mode (GET_MODE_BITSIZE (word_mode
) + addend
,
2787 result
, true, OPTAB_DIRECT
);
2791 convert_move (result
, temp
, true);
2793 emit_label (after_label
);
2794 convert_move (target
, result
, true);
2799 add_equal_note (seq
, target
, optab_to_code (unoptab
), xop0
, NULL_RTX
, mode
);
2808 /* Try calculating popcount of a double-word quantity as two popcount's of
2809 word-sized quantities and summing up the results. */
2811 expand_doubleword_popcount (scalar_int_mode mode
, rtx op0
, rtx target
)
2818 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2819 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2821 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2822 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2830 /* If we were not given a target, use a word_mode register, not a
2831 'mode' register. The result will fit, and nobody is expecting
2832 anything bigger (the return type of __builtin_popcount* is int). */
2834 target
= gen_reg_rtx (word_mode
);
2836 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2841 add_equal_note (seq
, t
, POPCOUNT
, op0
, NULL_RTX
, mode
);
2849 (parity:narrow (low (x) ^ high (x))) */
2851 expand_doubleword_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2853 rtx t
= expand_binop (word_mode
, xor_optab
,
2854 operand_subword_force (op0
, 0, mode
),
2855 operand_subword_force (op0
, 1, mode
),
2856 NULL_RTX
, 0, OPTAB_DIRECT
);
2857 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2863 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2865 widen_bswap (scalar_int_mode mode
, rtx op0
, rtx target
)
2869 opt_scalar_int_mode wider_mode_iter
;
2871 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2872 if (optab_handler (bswap_optab
, wider_mode_iter
.require ())
2873 != CODE_FOR_nothing
)
2876 if (!wider_mode_iter
.exists ())
2879 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2880 last
= get_last_insn ();
2882 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2883 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2885 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2886 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2888 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2889 GET_MODE_BITSIZE (wider_mode
)
2890 - GET_MODE_BITSIZE (mode
),
2896 target
= gen_reg_rtx (mode
);
2897 emit_move_insn (target
, gen_lowpart (mode
, x
));
2900 delete_insns_since (last
);
2905 /* Try calculating bswap as two bswaps of two word-sized operands. */
2908 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2912 t1
= expand_unop (word_mode
, bswap_optab
,
2913 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2914 t0
= expand_unop (word_mode
, bswap_optab
,
2915 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2917 if (target
== 0 || !valid_multiword_target_p (target
))
2918 target
= gen_reg_rtx (mode
);
2920 emit_clobber (target
);
2921 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2922 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2927 /* Try calculating (parity x) as (and (popcount x) 1), where
2928 popcount can also be done in a wider mode. */
2930 expand_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2932 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2933 opt_scalar_int_mode wider_mode_iter
;
2934 FOR_EACH_MODE_FROM (wider_mode_iter
, mode
)
2936 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2937 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2942 last
= get_last_insn ();
2944 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2945 target
= gen_reg_rtx (wider_mode
);
2947 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2948 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2951 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2952 target
, true, OPTAB_DIRECT
);
2956 if (mclass
!= MODE_INT
2957 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2958 return convert_to_mode (mode
, temp
, 0);
2960 return gen_lowpart (mode
, temp
);
2963 delete_insns_since (last
);
2969 /* Try calculating ctz(x) as K - clz(x & -x) ,
2970 where K is GET_MODE_PRECISION(mode) - 1.
2972 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2973 don't have to worry about what the hardware does in that case. (If
2974 the clz instruction produces the usual value at 0, which is K, the
2975 result of this code sequence will be -1; expand_ffs, below, relies
2976 on this. It might be nice to have it be K instead, for consistency
2977 with the (very few) processors that provide a ctz with a defined
2978 value, but that would take one more instruction, and it would be
2979 less convenient for expand_ffs anyway. */
2982 expand_ctz (scalar_int_mode mode
, rtx op0
, rtx target
)
2987 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2992 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2994 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2995 true, OPTAB_DIRECT
);
2997 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2999 temp
= expand_binop (mode
, sub_optab
,
3000 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
3002 true, OPTAB_DIRECT
);
3012 add_equal_note (seq
, temp
, CTZ
, op0
, NULL_RTX
, mode
);
3018 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
3019 else with the sequence used by expand_clz.
3021 The ffs builtin promises to return zero for a zero value and ctz/clz
3022 may have an undefined value in that case. If they do not give us a
3023 convenient value, we have to generate a test and branch. */
3025 expand_ffs (scalar_int_mode mode
, rtx op0
, rtx target
)
3027 HOST_WIDE_INT val
= 0;
3028 bool defined_at_zero
= false;
3032 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
3036 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
3040 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
3042 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
3045 temp
= expand_ctz (mode
, op0
, 0);
3049 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
3051 defined_at_zero
= true;
3052 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
3058 if (defined_at_zero
&& val
== -1)
3059 /* No correction needed at zero. */;
3062 /* We don't try to do anything clever with the situation found
3063 on some processors (eg Alpha) where ctz(0:mode) ==
3064 bitsize(mode). If someone can think of a way to send N to -1
3065 and leave alone all values in the range 0..N-1 (where N is a
3066 power of two), cheaper than this test-and-branch, please add it.
3068 The test-and-branch is done after the operation itself, in case
3069 the operation sets condition codes that can be recycled for this.
3070 (This is true on i386, for instance.) */
3072 rtx_code_label
*nonzero_label
= gen_label_rtx ();
3073 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
3074 mode
, true, nonzero_label
);
3076 convert_move (temp
, GEN_INT (-1), false);
3077 emit_label (nonzero_label
);
3080 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
3081 to produce a value in the range 0..bitsize. */
3082 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
3083 target
, false, OPTAB_DIRECT
);
3090 add_equal_note (seq
, temp
, FFS
, op0
, NULL_RTX
, mode
);
3099 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
3100 conditions, VAL may already be a SUBREG against which we cannot generate
3101 a further SUBREG. In this case, we expect forcing the value into a
3102 register will work around the situation. */
3105 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
3109 ret
= lowpart_subreg (omode
, val
, imode
);
3112 val
= force_reg (imode
, val
);
3113 ret
= lowpart_subreg (omode
, val
, imode
);
3114 gcc_assert (ret
!= NULL
);
3119 /* Expand a floating point absolute value or negation operation via a
3120 logical operation on the sign bit. */
3123 expand_absneg_bit (enum rtx_code code
, scalar_float_mode mode
,
3124 rtx op0
, rtx target
)
3126 const struct real_format
*fmt
;
3127 int bitpos
, word
, nwords
, i
;
3128 scalar_int_mode imode
;
3132 /* The format has to have a simple sign bit. */
3133 fmt
= REAL_MODE_FORMAT (mode
);
3137 bitpos
= fmt
->signbit_rw
;
3141 /* Don't create negative zeros if the format doesn't support them. */
3142 if (code
== NEG
&& !fmt
->has_signed_zero
)
3145 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3147 if (!int_mode_for_mode (mode
).exists (&imode
))
3156 if (FLOAT_WORDS_BIG_ENDIAN
)
3157 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3159 word
= bitpos
/ BITS_PER_WORD
;
3160 bitpos
= bitpos
% BITS_PER_WORD
;
3161 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3164 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3170 || reg_overlap_mentioned_p (target
, op0
)
3171 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3172 target
= gen_reg_rtx (mode
);
3178 for (i
= 0; i
< nwords
; ++i
)
3180 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3181 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3185 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
3187 immed_wide_int_const (mask
, imode
),
3188 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3189 if (temp
!= targ_piece
)
3190 emit_move_insn (targ_piece
, temp
);
3193 emit_move_insn (targ_piece
, op0_piece
);
3196 insns
= get_insns ();
3203 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
3204 gen_lowpart (imode
, op0
),
3205 immed_wide_int_const (mask
, imode
),
3206 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3207 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3209 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
3210 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
3217 /* As expand_unop, but will fail rather than attempt the operation in a
3218 different mode or with a libcall. */
3220 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
3223 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
3225 class expand_operand ops
[2];
3226 enum insn_code icode
= optab_handler (unoptab
, mode
);
3227 rtx_insn
*last
= get_last_insn ();
3230 create_output_operand (&ops
[0], target
, mode
);
3231 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
3232 pat
= maybe_gen_insn (icode
, 2, ops
);
3235 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3236 && ! add_equal_note (pat
, ops
[0].value
,
3237 optab_to_code (unoptab
),
3238 ops
[1].value
, NULL_RTX
, mode
))
3240 delete_insns_since (last
);
3241 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
3246 return ops
[0].value
;
3252 /* Generate code to perform an operation specified by UNOPTAB
3253 on operand OP0, with result having machine-mode MODE.
3255 UNSIGNEDP is for the case where we have to widen the operands
3256 to perform the operation. It says to use zero-extension.
3258 If TARGET is nonzero, the value
3259 is generated there, if it is convenient to do so.
3260 In all cases an rtx is returned for the locus of the value;
3261 this may or may not be TARGET. */
3264 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
3267 enum mode_class mclass
= GET_MODE_CLASS (mode
);
3268 machine_mode wider_mode
;
3269 scalar_int_mode int_mode
;
3270 scalar_float_mode float_mode
;
3274 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
3278 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3280 /* Widening (or narrowing) clz needs special treatment. */
3281 if (unoptab
== clz_optab
)
3283 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3285 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
3289 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3290 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3292 temp
= expand_doubleword_clz_ctz_ffs (int_mode
, op0
, target
,
3302 if (unoptab
== clrsb_optab
)
3304 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3306 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
3309 temp
= expand_clrsb_using_clz (int_mode
, op0
, target
);
3316 if (unoptab
== popcount_optab
3317 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3318 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3319 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3320 && optimize_insn_for_speed_p ())
3322 temp
= expand_doubleword_popcount (int_mode
, op0
, target
);
3327 if (unoptab
== parity_optab
3328 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3329 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3330 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3331 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
3332 && optimize_insn_for_speed_p ())
3334 temp
= expand_doubleword_parity (int_mode
, op0
, target
);
3339 /* Widening (or narrowing) bswap needs special treatment. */
3340 if (unoptab
== bswap_optab
)
3342 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3343 or ROTATERT. First try these directly; if this fails, then try the
3344 obvious pair of shifts with allowed widening, as this will probably
3345 be always more efficient than the other fallback methods. */
3351 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
3353 temp
= expand_binop (mode
, rotl_optab
, op0
,
3354 gen_int_shift_amount (mode
, 8),
3355 target
, unsignedp
, OPTAB_DIRECT
);
3360 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
3362 temp
= expand_binop (mode
, rotr_optab
, op0
,
3363 gen_int_shift_amount (mode
, 8),
3364 target
, unsignedp
, OPTAB_DIRECT
);
3369 last
= get_last_insn ();
3371 temp1
= expand_binop (mode
, ashl_optab
, op0
,
3372 gen_int_shift_amount (mode
, 8), NULL_RTX
,
3373 unsignedp
, OPTAB_WIDEN
);
3374 temp2
= expand_binop (mode
, lshr_optab
, op0
,
3375 gen_int_shift_amount (mode
, 8), NULL_RTX
,
3376 unsignedp
, OPTAB_WIDEN
);
3379 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
3380 unsignedp
, OPTAB_WIDEN
);
3385 delete_insns_since (last
);
3388 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3390 temp
= widen_bswap (int_mode
, op0
, target
);
3394 /* We do not provide a 128-bit bswap in libgcc so force the use of
3395 a double bswap for 64-bit targets. */
3396 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3397 && (UNITS_PER_WORD
== 8
3398 || optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
))
3400 temp
= expand_doubleword_bswap (mode
, op0
, target
);
3409 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3410 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3412 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
3415 rtx_insn
*last
= get_last_insn ();
3417 /* For certain operations, we need not actually extend
3418 the narrow operand, as long as we will truncate the
3419 results to the same narrowness. */
3421 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3422 (unoptab
== neg_optab
3423 || unoptab
== one_cmpl_optab
)
3424 && mclass
== MODE_INT
);
3426 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3431 if (mclass
!= MODE_INT
3432 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
3435 target
= gen_reg_rtx (mode
);
3436 convert_move (target
, temp
, 0);
3440 return gen_lowpart (mode
, temp
);
3443 delete_insns_since (last
);
3447 /* These can be done a word at a time. */
3448 if (unoptab
== one_cmpl_optab
3449 && is_int_mode (mode
, &int_mode
)
3450 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
3451 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3458 || reg_overlap_mentioned_p (target
, op0
)
3459 || !valid_multiword_target_p (target
))
3460 target
= gen_reg_rtx (int_mode
);
3464 /* Do the actual arithmetic. */
3465 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
3467 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
3468 rtx x
= expand_unop (word_mode
, unoptab
,
3469 operand_subword_force (op0
, i
, int_mode
),
3470 target_piece
, unsignedp
);
3472 if (target_piece
!= x
)
3473 emit_move_insn (target_piece
, x
);
3476 insns
= get_insns ();
3483 /* Emit ~op0 as op0 ^ -1. */
3484 if (unoptab
== one_cmpl_optab
3485 && (SCALAR_INT_MODE_P (mode
) || GET_MODE_CLASS (mode
) == MODE_VECTOR_INT
)
3486 && optab_handler (xor_optab
, mode
) != CODE_FOR_nothing
)
3488 temp
= expand_binop (mode
, xor_optab
, op0
, CONSTM1_RTX (mode
),
3489 target
, unsignedp
, OPTAB_DIRECT
);
3494 if (optab_to_code (unoptab
) == NEG
)
3496 /* Try negating floating point values by flipping the sign bit. */
3497 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3499 temp
= expand_absneg_bit (NEG
, float_mode
, op0
, target
);
3504 /* If there is no negation pattern, and we have no negative zero,
3505 try subtracting from zero. */
3506 if (!HONOR_SIGNED_ZEROS (mode
))
3508 temp
= expand_binop (mode
, (unoptab
== negv_optab
3509 ? subv_optab
: sub_optab
),
3510 CONST0_RTX (mode
), op0
, target
,
3511 unsignedp
, OPTAB_DIRECT
);
3517 /* Try calculating parity (x) as popcount (x) % 2. */
3518 if (unoptab
== parity_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3520 temp
= expand_parity (int_mode
, op0
, target
);
3525 /* Try implementing ffs (x) in terms of clz (x). */
3526 if (unoptab
== ffs_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3528 temp
= expand_ffs (int_mode
, op0
, target
);
3533 /* Try implementing ctz (x) in terms of clz (x). */
3534 if (unoptab
== ctz_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3536 temp
= expand_ctz (int_mode
, op0
, target
);
3541 if ((unoptab
== ctz_optab
|| unoptab
== ffs_optab
)
3542 && optimize_insn_for_speed_p ()
3543 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3544 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3545 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3546 || optab_handler (ctz_optab
, word_mode
) != CODE_FOR_nothing
))
3548 temp
= expand_doubleword_clz_ctz_ffs (int_mode
, op0
, target
, unoptab
);
3554 /* Now try a library call in this mode. */
3555 libfunc
= optab_libfunc (unoptab
, mode
);
3561 machine_mode outmode
= mode
;
3563 /* All of these functions return small values. Thus we choose to
3564 have them return something that isn't a double-word. */
3565 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
3566 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
3567 || unoptab
== parity_optab
)
3569 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
3570 optab_libfunc (unoptab
, mode
)));
3574 /* Pass 1 for NO_QUEUE so we don't lose any increments
3575 if the libcall is cse'd or moved. */
3576 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
3578 insns
= get_insns ();
3581 target
= gen_reg_rtx (outmode
);
3582 bool trapv
= trapv_unoptab_p (unoptab
);
3584 eq_value
= NULL_RTX
;
3587 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
3588 if (GET_MODE_UNIT_SIZE (outmode
) < GET_MODE_UNIT_SIZE (mode
))
3589 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
3590 else if (GET_MODE_UNIT_SIZE (outmode
) > GET_MODE_UNIT_SIZE (mode
))
3591 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
3592 outmode
, eq_value
, mode
);
3594 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
3599 /* It can't be done in this mode. Can we do it in a wider mode? */
3601 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3603 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3605 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
3606 || optab_libfunc (unoptab
, wider_mode
))
3609 rtx_insn
*last
= get_last_insn ();
3611 /* For certain operations, we need not actually extend
3612 the narrow operand, as long as we will truncate the
3613 results to the same narrowness. */
3614 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3615 (unoptab
== neg_optab
3616 || unoptab
== one_cmpl_optab
3617 || unoptab
== bswap_optab
)
3618 && mclass
== MODE_INT
);
3620 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3623 /* If we are generating clz using wider mode, adjust the
3624 result. Similarly for clrsb. */
3625 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3628 scalar_int_mode wider_int_mode
3629 = as_a
<scalar_int_mode
> (wider_mode
);
3630 int_mode
= as_a
<scalar_int_mode
> (mode
);
3632 (wider_mode
, sub_optab
, temp
,
3633 gen_int_mode (GET_MODE_PRECISION (wider_int_mode
)
3634 - GET_MODE_PRECISION (int_mode
),
3636 target
, true, OPTAB_DIRECT
);
3639 /* Likewise for bswap. */
3640 if (unoptab
== bswap_optab
&& temp
!= 0)
3642 scalar_int_mode wider_int_mode
3643 = as_a
<scalar_int_mode
> (wider_mode
);
3644 int_mode
= as_a
<scalar_int_mode
> (mode
);
3645 gcc_assert (GET_MODE_PRECISION (wider_int_mode
)
3646 == GET_MODE_BITSIZE (wider_int_mode
)
3647 && GET_MODE_PRECISION (int_mode
)
3648 == GET_MODE_BITSIZE (int_mode
));
3650 temp
= expand_shift (RSHIFT_EXPR
, wider_int_mode
, temp
,
3651 GET_MODE_BITSIZE (wider_int_mode
)
3652 - GET_MODE_BITSIZE (int_mode
),
3658 if (mclass
!= MODE_INT
)
3661 target
= gen_reg_rtx (mode
);
3662 convert_move (target
, temp
, 0);
3666 return gen_lowpart (mode
, temp
);
3669 delete_insns_since (last
);
3674 /* One final attempt at implementing negation via subtraction,
3675 this time allowing widening of the operand. */
3676 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3679 temp
= expand_binop (mode
,
3680 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3681 CONST0_RTX (mode
), op0
,
3682 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3690 /* Emit code to compute the absolute value of OP0, with result to
3691 TARGET if convenient. (TARGET may be 0.) The return value says
3692 where the result actually is to be found.
3694 MODE is the mode of the operand; the mode of the result is
3695 different but can be deduced from MODE.
3700 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3701 int result_unsignedp
)
3705 if (GET_MODE_CLASS (mode
) != MODE_INT
3707 result_unsignedp
= 1;
3709 /* First try to do it with a special abs instruction. */
3710 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3715 /* For floating point modes, try clearing the sign bit. */
3716 scalar_float_mode float_mode
;
3717 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3719 temp
= expand_absneg_bit (ABS
, float_mode
, op0
, target
);
3724 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3725 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3726 && !HONOR_SIGNED_ZEROS (mode
))
3728 rtx_insn
*last
= get_last_insn ();
3730 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3733 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3739 delete_insns_since (last
);
3742 /* If this machine has expensive jumps, we can do integer absolute
3743 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3744 where W is the width of MODE. */
3746 scalar_int_mode int_mode
;
3747 if (is_int_mode (mode
, &int_mode
)
3748 && BRANCH_COST (optimize_insn_for_speed_p (),
3751 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3752 GET_MODE_PRECISION (int_mode
) - 1,
3755 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3758 temp
= expand_binop (int_mode
,
3759 result_unsignedp
? sub_optab
: subv_optab
,
3760 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3770 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3771 int result_unsignedp
, int safe
)
3774 rtx_code_label
*op1
;
3776 if (GET_MODE_CLASS (mode
) != MODE_INT
3778 result_unsignedp
= 1;
3780 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3784 /* If that does not win, use conditional jump and negate. */
3786 /* It is safe to use the target if it is the same
3787 as the source if this is also a pseudo register */
3788 if (op0
== target
&& REG_P (op0
)
3789 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3792 op1
= gen_label_rtx ();
3793 if (target
== 0 || ! safe
3794 || GET_MODE (target
) != mode
3795 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3797 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3798 target
= gen_reg_rtx (mode
);
3800 emit_move_insn (target
, op0
);
3803 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3804 NULL_RTX
, NULL
, op1
,
3805 profile_probability::uninitialized ());
3807 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3810 emit_move_insn (target
, op0
);
3816 /* Emit code to compute the one's complement absolute value of OP0
3817 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3818 (TARGET may be NULL_RTX.) The return value says where the result
3819 actually is to be found.
3821 MODE is the mode of the operand; the mode of the result is
3822 different but can be deduced from MODE. */
3825 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3829 /* Not applicable for floating point modes. */
3830 if (FLOAT_MODE_P (mode
))
3833 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3834 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3836 rtx_insn
*last
= get_last_insn ();
3838 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3840 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3846 delete_insns_since (last
);
3849 /* If this machine has expensive jumps, we can do one's complement
3850 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3852 scalar_int_mode int_mode
;
3853 if (is_int_mode (mode
, &int_mode
)
3854 && BRANCH_COST (optimize_insn_for_speed_p (),
3857 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3858 GET_MODE_PRECISION (int_mode
) - 1,
3861 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3871 /* A subroutine of expand_copysign, perform the copysign operation using the
3872 abs and neg primitives advertised to exist on the target. The assumption
3873 is that we have a split register file, and leaving op0 in fp registers,
3874 and not playing with subregs so much, will help the register allocator. */
3877 expand_copysign_absneg (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3878 int bitpos
, bool op0_is_abs
)
3880 scalar_int_mode imode
;
3881 enum insn_code icode
;
3883 rtx_code_label
*label
;
3888 /* Check if the back end provides an insn that handles signbit for the
3890 icode
= optab_handler (signbit_optab
, mode
);
3891 if (icode
!= CODE_FOR_nothing
)
3893 imode
= as_a
<scalar_int_mode
> (insn_data
[(int) icode
].operand
[0].mode
);
3894 sign
= gen_reg_rtx (imode
);
3895 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3899 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3901 if (!int_mode_for_mode (mode
).exists (&imode
))
3903 op1
= gen_lowpart (imode
, op1
);
3910 if (FLOAT_WORDS_BIG_ENDIAN
)
3911 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3913 word
= bitpos
/ BITS_PER_WORD
;
3914 bitpos
= bitpos
% BITS_PER_WORD
;
3915 op1
= operand_subword_force (op1
, word
, mode
);
3918 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3919 sign
= expand_binop (imode
, and_optab
, op1
,
3920 immed_wide_int_const (mask
, imode
),
3921 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3926 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3933 if (target
== NULL_RTX
)
3934 target
= copy_to_reg (op0
);
3936 emit_move_insn (target
, op0
);
3939 label
= gen_label_rtx ();
3940 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3942 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3943 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3945 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3947 emit_move_insn (target
, op0
);
3955 /* A subroutine of expand_copysign, perform the entire copysign operation
3956 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3957 is true if op0 is known to have its sign bit clear. */
3960 expand_copysign_bit (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3961 int bitpos
, bool op0_is_abs
)
3963 scalar_int_mode imode
;
3964 int word
, nwords
, i
;
3968 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3970 if (!int_mode_for_mode (mode
).exists (&imode
))
3979 if (FLOAT_WORDS_BIG_ENDIAN
)
3980 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3982 word
= bitpos
/ BITS_PER_WORD
;
3983 bitpos
= bitpos
% BITS_PER_WORD
;
3984 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3987 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3992 || reg_overlap_mentioned_p (target
, op0
)
3993 || reg_overlap_mentioned_p (target
, op1
)
3994 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3995 target
= gen_reg_rtx (mode
);
4001 for (i
= 0; i
< nwords
; ++i
)
4003 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
4004 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
4010 = expand_binop (imode
, and_optab
, op0_piece
,
4011 immed_wide_int_const (~mask
, imode
),
4012 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4013 op1
= expand_binop (imode
, and_optab
,
4014 operand_subword_force (op1
, i
, mode
),
4015 immed_wide_int_const (mask
, imode
),
4016 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4018 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
4019 targ_piece
, 1, OPTAB_LIB_WIDEN
);
4020 if (temp
!= targ_piece
)
4021 emit_move_insn (targ_piece
, temp
);
4024 emit_move_insn (targ_piece
, op0_piece
);
4027 insns
= get_insns ();
4034 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
4035 immed_wide_int_const (mask
, imode
),
4036 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4038 op0
= gen_lowpart (imode
, op0
);
4040 op0
= expand_binop (imode
, and_optab
, op0
,
4041 immed_wide_int_const (~mask
, imode
),
4042 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4044 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
4045 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
4046 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
4052 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
4053 scalar floating point mode. Return NULL if we do not know how to
4054 expand the operation inline. */
4057 expand_copysign (rtx op0
, rtx op1
, rtx target
)
4059 scalar_float_mode mode
;
4060 const struct real_format
*fmt
;
4064 mode
= as_a
<scalar_float_mode
> (GET_MODE (op0
));
4065 gcc_assert (GET_MODE (op1
) == mode
);
4067 /* First try to do it with a special instruction. */
4068 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
4069 target
, 0, OPTAB_DIRECT
);
4073 fmt
= REAL_MODE_FORMAT (mode
);
4074 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
4078 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
4080 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
4081 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
4085 if (fmt
->signbit_ro
>= 0
4086 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
4087 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
4088 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
4090 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
4091 fmt
->signbit_ro
, op0_is_abs
);
4096 if (fmt
->signbit_rw
< 0)
4098 return expand_copysign_bit (mode
, op0
, op1
, target
,
4099 fmt
->signbit_rw
, op0_is_abs
);
4102 /* Generate an instruction whose insn-code is INSN_CODE,
4103 with two operands: an output TARGET and an input OP0.
4104 TARGET *must* be nonzero, and the output is always stored there.
4105 CODE is an rtx code such that (CODE OP0) is an rtx that describes
4106 the value that is stored into TARGET.
4108 Return false if expansion failed. */
4111 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
4114 class expand_operand ops
[2];
4117 create_output_operand (&ops
[0], target
, GET_MODE (target
));
4118 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
4119 pat
= maybe_gen_insn (icode
, 2, ops
);
4123 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
4125 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
,
4130 if (ops
[0].value
!= target
)
4131 emit_move_insn (target
, ops
[0].value
);
4134 /* Generate an instruction whose insn-code is INSN_CODE,
4135 with two operands: an output TARGET and an input OP0.
4136 TARGET *must* be nonzero, and the output is always stored there.
4137 CODE is an rtx code such that (CODE OP0) is an rtx that describes
4138 the value that is stored into TARGET. */
4141 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
4143 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
4147 struct no_conflict_data
4150 rtx_insn
*first
, *insn
;
4154 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
4155 the currently examined clobber / store has to stay in the list of
4156 insns that constitute the actual libcall block. */
4158 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
4160 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
4162 /* If this inns directly contributes to setting the target, it must stay. */
4163 if (reg_overlap_mentioned_p (p
->target
, dest
))
4164 p
->must_stay
= true;
4165 /* If we haven't committed to keeping any other insns in the list yet,
4166 there is nothing more to check. */
4167 else if (p
->insn
== p
->first
)
4169 /* If this insn sets / clobbers a register that feeds one of the insns
4170 already in the list, this insn has to stay too. */
4171 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
4172 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
4173 || reg_used_between_p (dest
, p
->first
, p
->insn
)
4174 /* Likewise if this insn depends on a register set by a previous
4175 insn in the list, or if it sets a result (presumably a hard
4176 register) that is set or clobbered by a previous insn.
4177 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4178 SET_DEST perform the former check on the address, and the latter
4179 check on the MEM. */
4180 || (GET_CODE (set
) == SET
4181 && (modified_in_p (SET_SRC (set
), p
->first
)
4182 || modified_in_p (SET_DEST (set
), p
->first
)
4183 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
4184 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
4185 p
->must_stay
= true;
4189 /* Emit code to make a call to a constant function or a library call.
4191 INSNS is a list containing all insns emitted in the call.
4192 These insns leave the result in RESULT. Our block is to copy RESULT
4193 to TARGET, which is logically equivalent to EQUIV.
4195 We first emit any insns that set a pseudo on the assumption that these are
4196 loading constants into registers; doing so allows them to be safely cse'ed
4197 between blocks. Then we emit all the other insns in the block, followed by
4198 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4199 note with an operand of EQUIV. */
4202 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
4203 bool equiv_may_trap
)
4205 rtx final_dest
= target
;
4206 rtx_insn
*next
, *last
, *insn
;
4208 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4209 into a MEM later. Protect the libcall block from this change. */
4210 if (! REG_P (target
) || REG_USERVAR_P (target
))
4211 target
= gen_reg_rtx (GET_MODE (target
));
4213 /* If we're using non-call exceptions, a libcall corresponding to an
4214 operation that may trap may also trap. */
4215 /* ??? See the comment in front of make_reg_eh_region_note. */
4216 if (cfun
->can_throw_non_call_exceptions
4217 && (equiv_may_trap
|| may_trap_p (equiv
)))
4219 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
4222 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
4225 int lp_nr
= INTVAL (XEXP (note
, 0));
4226 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
4227 remove_note (insn
, note
);
4233 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4234 reg note to indicate that this call cannot throw or execute a nonlocal
4235 goto (unless there is already a REG_EH_REGION note, in which case
4237 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
4239 make_reg_eh_region_note_nothrow_nononlocal (insn
);
4242 /* First emit all insns that set pseudos. Remove them from the list as
4243 we go. Avoid insns that set pseudos which were referenced in previous
4244 insns. These can be generated by move_by_pieces, for example,
4245 to update an address. Similarly, avoid insns that reference things
4246 set in previous insns. */
4248 for (insn
= insns
; insn
; insn
= next
)
4250 rtx set
= single_set (insn
);
4252 next
= NEXT_INSN (insn
);
4254 if (set
!= 0 && REG_P (SET_DEST (set
))
4255 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
4257 struct no_conflict_data data
;
4259 data
.target
= const0_rtx
;
4263 note_stores (insn
, no_conflict_move_test
, &data
);
4264 if (! data
.must_stay
)
4266 if (PREV_INSN (insn
))
4267 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
4272 SET_PREV_INSN (next
) = PREV_INSN (insn
);
4278 /* Some ports use a loop to copy large arguments onto the stack.
4279 Don't move anything outside such a loop. */
4284 /* Write the remaining insns followed by the final copy. */
4285 for (insn
= insns
; insn
; insn
= next
)
4287 next
= NEXT_INSN (insn
);
4292 last
= emit_move_insn (target
, result
);
4294 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
4296 if (final_dest
!= target
)
4297 emit_move_insn (final_dest
, target
);
4301 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
4303 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
4306 /* True if we can perform a comparison of mode MODE straightforwardly.
4307 PURPOSE describes how this comparison will be used. CODE is the rtx
4308 comparison code we will be using.
4310 ??? Actually, CODE is slightly weaker than that. A target is still
4311 required to implement all of the normal bcc operations, but not
4312 required to implement all (or any) of the unordered bcc operations. */
4315 can_compare_p (enum rtx_code code
, machine_mode mode
,
4316 enum can_compare_purpose purpose
)
4319 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
4322 enum insn_code icode
;
4324 if (purpose
== ccp_jump
4325 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
4326 && insn_operand_matches (icode
, 0, test
))
4328 if (purpose
== ccp_store_flag
4329 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
4330 && insn_operand_matches (icode
, 1, test
))
4332 if (purpose
== ccp_cmov
4333 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
4336 mode
= GET_MODE_WIDER_MODE (mode
).else_void ();
4337 PUT_MODE (test
, mode
);
4339 while (mode
!= VOIDmode
);
4344 /* Return whether RTL code CODE corresponds to an unsigned optab. */
4347 unsigned_optab_p (enum rtx_code code
)
4349 return code
== LTU
|| code
== LEU
|| code
== GTU
|| code
== GEU
;
4352 /* Return whether the backend-emitted comparison for code CODE, comparing
4353 operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4354 operand OPNO of pattern ICODE. */
4357 insn_predicate_matches_p (enum insn_code icode
, unsigned int opno
,
4358 enum rtx_code code
, machine_mode mask_mode
,
4359 machine_mode value_mode
)
4361 rtx reg1
= alloca_raw_REG (value_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4362 rtx reg2
= alloca_raw_REG (value_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4363 rtx test
= alloca_rtx_fmt_ee (code
, mask_mode
, reg1
, reg2
);
4364 return insn_operand_matches (icode
, opno
, test
);
4367 /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4368 for code CODE, comparing operands of mode VALUE_MODE and producing a result
4372 can_vec_cmp_compare_p (enum rtx_code code
, machine_mode value_mode
,
4373 machine_mode mask_mode
)
4375 enum insn_code icode
4376 = get_vec_cmp_icode (value_mode
, mask_mode
, unsigned_optab_p (code
));
4377 if (icode
== CODE_FOR_nothing
)
4380 return insn_predicate_matches_p (icode
, 1, code
, mask_mode
, value_mode
);
4383 /* Return whether the backend can emit a vector comparison (vcond/vcondu) for
4384 code CODE, comparing operands of mode CMP_OP_MODE and producing a result
4388 can_vcond_compare_p (enum rtx_code code
, machine_mode value_mode
,
4389 machine_mode cmp_op_mode
)
4391 enum insn_code icode
4392 = get_vcond_icode (value_mode
, cmp_op_mode
, unsigned_optab_p (code
));
4393 if (icode
== CODE_FOR_nothing
)
4396 return insn_predicate_matches_p (icode
, 3, code
, value_mode
, cmp_op_mode
);
4399 /* Return whether the backend can emit vector set instructions for inserting
4400 element into vector at variable index position. */
4403 can_vec_set_var_idx_p (machine_mode vec_mode
)
4405 if (!VECTOR_MODE_P (vec_mode
))
4408 machine_mode inner_mode
= GET_MODE_INNER (vec_mode
);
4410 rtx reg1
= alloca_raw_REG (vec_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4411 rtx reg2
= alloca_raw_REG (inner_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4413 enum insn_code icode
= optab_handler (vec_set_optab
, vec_mode
);
4415 const struct insn_data_d
*data
= &insn_data
[icode
];
4416 machine_mode idx_mode
= data
->operand
[2].mode
;
4418 rtx reg3
= alloca_raw_REG (idx_mode
, LAST_VIRTUAL_REGISTER
+ 3);
4420 return icode
!= CODE_FOR_nothing
&& insn_operand_matches (icode
, 0, reg1
)
4421 && insn_operand_matches (icode
, 1, reg2
)
4422 && insn_operand_matches (icode
, 2, reg3
);
4425 /* Return whether the backend can emit a vec_extract instruction with
4426 a non-constant index. */
4428 can_vec_extract_var_idx_p (machine_mode vec_mode
, machine_mode extr_mode
)
4430 if (!VECTOR_MODE_P (vec_mode
))
4433 rtx reg1
= alloca_raw_REG (extr_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4434 rtx reg2
= alloca_raw_REG (vec_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4436 enum insn_code icode
= convert_optab_handler (vec_extract_optab
,
4437 vec_mode
, extr_mode
);
4439 const struct insn_data_d
*data
= &insn_data
[icode
];
4440 machine_mode idx_mode
= data
->operand
[2].mode
;
4442 rtx reg3
= alloca_raw_REG (idx_mode
, LAST_VIRTUAL_REGISTER
+ 3);
4444 return icode
!= CODE_FOR_nothing
&& insn_operand_matches (icode
, 0, reg1
)
4445 && insn_operand_matches (icode
, 1, reg2
)
4446 && insn_operand_matches (icode
, 2, reg3
);
4449 /* This function is called when we are going to emit a compare instruction that
4450 compares the values found in X and Y, using the rtl operator COMPARISON.
4452 If they have mode BLKmode, then SIZE specifies the size of both operands.
4454 UNSIGNEDP nonzero says that the operands are unsigned;
4455 this matters if they need to be widened (as given by METHODS).
4457 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4458 if we failed to produce one.
4460 *PMODE is the mode of the inputs (in case they are const_int).
4462 This function performs all the setup necessary so that the caller only has
4463 to emit a single comparison insn. This setup can involve doing a BLKmode
4464 comparison or emitting a library call to perform the comparison if no insn
4465 is available to handle it.
4466 The values which are passed in through pointers can be modified; the caller
4467 should perform the comparison on the modified values. Constant
4468 comparisons must have already been folded. */
4471 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4472 int unsignedp
, enum optab_methods methods
,
4473 rtx
*ptest
, machine_mode
*pmode
)
4475 machine_mode mode
= *pmode
;
4477 machine_mode cmp_mode
;
4479 /* The other methods are not needed. */
4480 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
4481 || methods
== OPTAB_LIB_WIDEN
);
4483 if (CONST_SCALAR_INT_P (y
))
4484 canonicalize_comparison (mode
, &comparison
, &y
);
4486 /* If we are optimizing, force expensive constants into a register. */
4487 if (CONSTANT_P (x
) && optimize
4488 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
4489 > COSTS_N_INSNS (1))
4490 && can_create_pseudo_p ())
4491 x
= force_reg (mode
, x
);
4493 if (CONSTANT_P (y
) && optimize
4494 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
4495 > COSTS_N_INSNS (1))
4496 && can_create_pseudo_p ())
4497 y
= force_reg (mode
, y
);
4499 /* Don't let both operands fail to indicate the mode. */
4500 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
4501 x
= force_reg (mode
, x
);
4502 if (mode
== VOIDmode
)
4503 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
4505 /* Handle all BLKmode compares. */
4507 if (mode
== BLKmode
)
4509 machine_mode result_mode
;
4510 enum insn_code cmp_code
;
4513 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
4517 /* Try to use a memory block compare insn - either cmpstr
4518 or cmpmem will do. */
4519 opt_scalar_int_mode cmp_mode_iter
;
4520 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
4522 scalar_int_mode cmp_mode
= cmp_mode_iter
.require ();
4523 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
4524 if (cmp_code
== CODE_FOR_nothing
)
4525 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
4526 if (cmp_code
== CODE_FOR_nothing
)
4527 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
4528 if (cmp_code
== CODE_FOR_nothing
)
4531 /* Must make sure the size fits the insn's mode. */
4532 if (CONST_INT_P (size
)
4533 ? UINTVAL (size
) > GET_MODE_MASK (cmp_mode
)
4534 : (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (size
)))
4535 > GET_MODE_BITSIZE (cmp_mode
)))
4538 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
4539 result
= gen_reg_rtx (result_mode
);
4540 size
= convert_to_mode (cmp_mode
, size
, 1);
4541 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
4543 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
4544 *pmode
= result_mode
;
4548 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
4551 /* Otherwise call a library function. */
4552 result
= emit_block_comp_via_libcall (x
, y
, size
);
4556 mode
= TYPE_MODE (integer_type_node
);
4557 methods
= OPTAB_LIB_WIDEN
;
4561 /* Don't allow operands to the compare to trap, as that can put the
4562 compare and branch in different basic blocks. */
4563 if (cfun
->can_throw_non_call_exceptions
)
4565 if (!can_create_pseudo_p () && (may_trap_p (x
) || may_trap_p (y
)))
4568 x
= copy_to_reg (x
);
4570 y
= copy_to_reg (y
);
4573 if (GET_MODE_CLASS (mode
) == MODE_CC
)
4575 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
4576 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4577 if (icode
!= CODE_FOR_nothing
4578 && insn_operand_matches (icode
, 0, test
))
4587 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4588 FOR_EACH_WIDER_MODE_FROM (cmp_mode
, mode
)
4590 enum insn_code icode
;
4591 icode
= optab_handler (cbranch_optab
, cmp_mode
);
4592 if (icode
!= CODE_FOR_nothing
4593 && insn_operand_matches (icode
, 0, test
))
4595 rtx_insn
*last
= get_last_insn ();
4596 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
4597 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
4599 && insn_operand_matches (icode
, 1, op0
)
4600 && insn_operand_matches (icode
, 2, op1
))
4602 XEXP (test
, 0) = op0
;
4603 XEXP (test
, 1) = op1
;
4608 delete_insns_since (last
);
4611 if (methods
== OPTAB_DIRECT
)
4615 if (methods
!= OPTAB_LIB_WIDEN
)
4618 if (SCALAR_FLOAT_MODE_P (mode
))
4620 /* Small trick if UNORDERED isn't implemented by the hardware. */
4621 if (comparison
== UNORDERED
&& rtx_equal_p (x
, y
))
4623 prepare_cmp_insn (x
, y
, UNLT
, NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4629 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
4634 machine_mode ret_mode
;
4636 /* Handle a libcall just for the mode we are using. */
4637 libfunc
= optab_libfunc (cmp_optab
, mode
);
4638 gcc_assert (libfunc
);
4640 /* If we want unsigned, and this mode has a distinct unsigned
4641 comparison routine, use that. */
4644 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
4649 ret_mode
= targetm
.libgcc_cmp_return_mode ();
4650 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4651 ret_mode
, x
, mode
, y
, mode
);
4653 /* There are two kinds of comparison routines. Biased routines
4654 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4655 of gcc expect that the comparison operation is equivalent
4656 to the modified comparison. For signed comparisons compare the
4657 result against 1 in the biased case, and zero in the unbiased
4658 case. For unsigned comparisons always compare against 1 after
4659 biasing the unbiased result by adding 1. This gives us a way to
4661 The comparisons in the fixed-point helper library are always
4666 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
4669 x
= plus_constant (ret_mode
, result
, 1);
4675 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
4685 /* Before emitting an insn with code ICODE, make sure that X, which is going
4686 to be used for operand OPNUM of the insn, is converted from mode MODE to
4687 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4688 that it is accepted by the operand predicate. Return the new value. */
4691 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
4692 machine_mode wider_mode
, int unsignedp
)
4694 if (mode
!= wider_mode
)
4695 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
4697 if (!insn_operand_matches (icode
, opnum
, x
))
4699 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
4700 if (reload_completed
)
4702 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
4704 x
= copy_to_mode_reg (op_mode
, x
);
4710 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4711 we can do the branch. */
4714 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
4715 direct_optab cmp_optab
, profile_probability prob
,
4718 machine_mode optab_mode
;
4719 enum mode_class mclass
;
4720 enum insn_code icode
;
4723 mclass
= GET_MODE_CLASS (mode
);
4724 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
4725 icode
= optab_handler (cmp_optab
, optab_mode
);
4727 gcc_assert (icode
!= CODE_FOR_nothing
);
4728 gcc_assert (test_branch
|| insn_operand_matches (icode
, 0, test
));
4730 insn
= emit_jump_insn (GEN_FCN (icode
) (XEXP (test
, 0),
4731 XEXP (test
, 1), label
));
4733 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
4734 XEXP (test
, 1), label
));
4736 if (prob
.initialized_p ()
4737 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4740 && any_condjump_p (insn
)
4741 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4742 add_reg_br_prob_note (insn
, prob
);
4745 /* PTEST points to a comparison that compares its first operand with zero.
4746 Check to see if it can be performed as a bit-test-and-branch instead.
4747 On success, return the instruction that performs the bit-test-and-branch
4748 and replace the second operand of *PTEST with the bit number to test.
4749 On failure, return CODE_FOR_nothing and leave *PTEST unchanged.
4751 Note that the comparison described by *PTEST should not be taken
4752 literally after a successful return. *PTEST is just a convenient
4753 place to store the two operands of the bit-and-test.
4755 VAL must contain the original tree expression for the first operand
4758 static enum insn_code
4759 validate_test_and_branch (tree val
, rtx
*ptest
, machine_mode
*pmode
, optab
*res
)
4761 if (!val
|| TREE_CODE (val
) != SSA_NAME
)
4762 return CODE_FOR_nothing
;
4764 machine_mode mode
= TYPE_MODE (TREE_TYPE (val
));
4768 if (GET_CODE (test
) == EQ
)
4769 optab
= tbranch_eq_optab
;
4770 else if (GET_CODE (test
) == NE
)
4771 optab
= tbranch_ne_optab
;
4773 return CODE_FOR_nothing
;
4777 /* If the target supports the testbit comparison directly, great. */
4778 auto icode
= direct_optab_handler (optab
, mode
);
4779 if (icode
== CODE_FOR_nothing
)
4782 if (tree_zero_one_valued_p (val
))
4784 auto pos
= BITS_BIG_ENDIAN
? GET_MODE_BITSIZE (mode
) - 1 : 0;
4785 XEXP (test
, 1) = gen_int_mode (pos
, mode
);
4791 wide_int wcst
= get_nonzero_bits (val
);
4793 return CODE_FOR_nothing
;
4797 if ((bitpos
= wi::exact_log2 (wcst
)) == -1)
4798 return CODE_FOR_nothing
;
4800 auto pos
= BITS_BIG_ENDIAN
? GET_MODE_BITSIZE (mode
) - 1 - bitpos
: bitpos
;
4801 XEXP (test
, 1) = gen_int_mode (pos
, mode
);
4807 /* Generate code to compare X with Y so that the condition codes are
4808 set and to jump to LABEL if the condition is true. If X is a
4809 constant and Y is not a constant, then the comparison is swapped to
4810 ensure that the comparison RTL has the canonical form.
4812 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4813 need to be widened. UNSIGNEDP is also used to select the proper
4814 branch condition code.
4816 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4818 MODE is the mode of the inputs (in case they are const_int).
4820 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4821 It will be potentially converted into an unsigned variant based on
4822 UNSIGNEDP to select a proper jump instruction.
4824 PROB is the probability of jumping to LABEL. If the comparison is against
4825 zero then VAL contains the expression from which the non-zero RTL is
4829 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4830 machine_mode mode
, int unsignedp
, tree val
, rtx label
,
4831 profile_probability prob
)
4833 rtx op0
= x
, op1
= y
;
4836 /* Swap operands and condition to ensure canonical RTL. */
4837 if (swap_commutative_operands_p (x
, y
)
4838 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4841 comparison
= swap_condition (comparison
);
4844 /* If OP0 is still a constant, then both X and Y must be constants
4845 or the opposite comparison is not supported. Force X into a register
4846 to create canonical RTL. */
4847 if (CONSTANT_P (op0
))
4848 op0
= force_reg (mode
, op0
);
4851 comparison
= unsigned_condition (comparison
);
4853 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4856 /* Check if we're comparing a truth type with 0, and if so check if
4857 the target supports tbranch. */
4858 machine_mode tmode
= mode
;
4860 if (op1
== CONST0_RTX (GET_MODE (op1
))
4861 && validate_test_and_branch (val
, &test
, &tmode
,
4862 &optab
) != CODE_FOR_nothing
)
4864 emit_cmp_and_jump_insn_1 (test
, tmode
, label
, optab
, prob
, true);
4868 emit_cmp_and_jump_insn_1 (test
, mode
, label
, cbranch_optab
, prob
, false);
4871 /* Overloaded version of emit_cmp_and_jump_insns in which VAL is unknown. */
4874 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4875 machine_mode mode
, int unsignedp
, rtx label
,
4876 profile_probability prob
)
4878 emit_cmp_and_jump_insns (x
, y
, comparison
, size
, mode
, unsignedp
, NULL
,
4883 /* Emit a library call comparison between floating point X and Y.
4884 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4887 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4888 rtx
*ptest
, machine_mode
*pmode
)
4890 enum rtx_code swapped
= swap_condition (comparison
);
4891 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4892 machine_mode orig_mode
= GET_MODE (x
);
4894 rtx true_rtx
, false_rtx
;
4895 rtx value
, target
, equiv
;
4898 bool reversed_p
= false;
4899 scalar_int_mode cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4901 FOR_EACH_WIDER_MODE_FROM (mode
, orig_mode
)
4903 if (code_to_optab (comparison
)
4904 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4907 if (code_to_optab (swapped
)
4908 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4911 comparison
= swapped
;
4915 if (code_to_optab (reversed
)
4916 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4918 comparison
= reversed
;
4924 gcc_assert (mode
!= VOIDmode
);
4926 if (mode
!= orig_mode
)
4928 x
= convert_to_mode (mode
, x
, 0);
4929 y
= convert_to_mode (mode
, y
, 0);
4932 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4933 the RTL. The allows the RTL optimizers to delete the libcall if the
4934 condition can be determined at compile-time. */
4935 if (comparison
== UNORDERED
4936 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4938 true_rtx
= const_true_rtx
;
4939 false_rtx
= const0_rtx
;
4946 true_rtx
= const0_rtx
;
4947 false_rtx
= const_true_rtx
;
4951 true_rtx
= const_true_rtx
;
4952 false_rtx
= const0_rtx
;
4956 true_rtx
= const1_rtx
;
4957 false_rtx
= const0_rtx
;
4961 true_rtx
= const0_rtx
;
4962 false_rtx
= constm1_rtx
;
4966 true_rtx
= constm1_rtx
;
4967 false_rtx
= const0_rtx
;
4971 true_rtx
= const0_rtx
;
4972 false_rtx
= const1_rtx
;
4980 if (comparison
== UNORDERED
)
4982 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4983 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4984 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4985 temp
, const_true_rtx
, equiv
);
4989 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4990 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4991 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4992 equiv
, true_rtx
, false_rtx
);
4996 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4997 cmp_mode
, x
, mode
, y
, mode
);
4998 insns
= get_insns ();
5001 target
= gen_reg_rtx (cmp_mode
);
5002 emit_libcall_block (insns
, target
, value
, equiv
);
5004 if (comparison
== UNORDERED
5005 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
5007 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
5009 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
5014 /* Generate code to indirectly jump to a location given in the rtx LOC. */
5017 emit_indirect_jump (rtx loc
)
5019 if (!targetm
.have_indirect_jump ())
5020 sorry ("indirect jumps are not available on this target");
5023 class expand_operand ops
[1];
5024 create_address_operand (&ops
[0], loc
);
5025 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
5031 /* Emit a conditional move instruction if the machine supports one for that
5032 condition and machine mode.
5034 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5035 the mode to use should they be constants. If it is VOIDmode, they cannot
5038 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
5039 should be stored there. MODE is the mode to use should they be constants.
5040 If it is VOIDmode, they cannot both be constants.
5042 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5043 is not supported. */
5046 emit_conditional_move (rtx target
, struct rtx_comparison comp
,
5048 machine_mode mode
, int unsignedp
)
5052 enum insn_code icode
;
5053 enum rtx_code reversed
;
5055 /* If the two source operands are identical, that's just a move. */
5057 if (rtx_equal_p (op2
, op3
))
5060 target
= gen_reg_rtx (mode
);
5062 emit_move_insn (target
, op3
);
5066 /* If one operand is constant, make it the second one. Only do this
5067 if the other operand is not constant as well. */
5069 if (swap_commutative_operands_p (comp
.op0
, comp
.op1
))
5071 std::swap (comp
.op0
, comp
.op1
);
5072 comp
.code
= swap_condition (comp
.code
);
5075 /* get_condition will prefer to generate LT and GT even if the old
5076 comparison was against zero, so undo that canonicalization here since
5077 comparisons against zero are cheaper. */
5079 if (comp
.code
== LT
&& comp
.op1
== const1_rtx
)
5080 comp
.code
= LE
, comp
.op1
= const0_rtx
;
5081 else if (comp
.code
== GT
&& comp
.op1
== constm1_rtx
)
5082 comp
.code
= GE
, comp
.op1
= const0_rtx
;
5084 if (comp
.mode
== VOIDmode
)
5085 comp
.mode
= GET_MODE (comp
.op0
);
5087 enum rtx_code orig_code
= comp
.code
;
5088 bool swapped
= false;
5089 if (swap_commutative_operands_p (op2
, op3
)
5091 reversed_comparison_code_parts (comp
.code
, comp
.op0
, comp
.op1
, NULL
))
5094 std::swap (op2
, op3
);
5095 comp
.code
= reversed
;
5099 if (mode
== VOIDmode
)
5100 mode
= GET_MODE (op2
);
5102 icode
= direct_optab_handler (movcc_optab
, mode
);
5104 if (icode
== CODE_FOR_nothing
)
5108 target
= gen_reg_rtx (mode
);
5110 for (int pass
= 0; ; pass
++)
5112 comp
.code
= unsignedp
? unsigned_condition (comp
.code
) : comp
.code
;
5114 simplify_gen_relational (comp
.code
, VOIDmode
,
5115 comp
.mode
, comp
.op0
, comp
.op1
);
5117 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5118 punt and let the caller figure out how best to deal with this
5120 if (COMPARISON_P (comparison
))
5122 saved_pending_stack_adjust save
;
5123 save_pending_stack_adjust (&save
);
5124 last
= get_last_insn ();
5125 do_pending_stack_adjust ();
5126 machine_mode cmpmode
= comp
.mode
;
5127 rtx orig_op0
= XEXP (comparison
, 0);
5128 rtx orig_op1
= XEXP (comparison
, 1);
5131 /* If we are optimizing, force expensive constants into a register
5132 but preserve an eventual equality with op2/op3. */
5133 if (CONSTANT_P (orig_op0
) && optimize
5135 && (rtx_cost (orig_op0
, mode
, COMPARE
, 0,
5136 optimize_insn_for_speed_p ())
5137 > COSTS_N_INSNS (1))
5138 && can_create_pseudo_p ())
5140 if (rtx_equal_p (orig_op0
, op2
))
5141 op2p
= XEXP (comparison
, 0) = force_reg (cmpmode
, orig_op0
);
5142 else if (rtx_equal_p (orig_op0
, op3
))
5143 op3p
= XEXP (comparison
, 0) = force_reg (cmpmode
, orig_op0
);
5145 if (CONSTANT_P (orig_op1
) && optimize
5147 && (rtx_cost (orig_op1
, mode
, COMPARE
, 0,
5148 optimize_insn_for_speed_p ())
5149 > COSTS_N_INSNS (1))
5150 && can_create_pseudo_p ())
5152 if (rtx_equal_p (orig_op1
, op2
))
5153 op2p
= XEXP (comparison
, 1) = force_reg (cmpmode
, orig_op1
);
5154 else if (rtx_equal_p (orig_op1
, op3
))
5155 op3p
= XEXP (comparison
, 1) = force_reg (cmpmode
, orig_op1
);
5157 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
5158 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
5159 OPTAB_WIDEN
, &comparison
, &cmpmode
);
5162 rtx res
= emit_conditional_move_1 (target
, comparison
,
5164 if (res
!= NULL_RTX
)
5167 delete_insns_since (last
);
5168 restore_pending_stack_adjust (&save
);
5174 /* If the preferred op2/op3 order is not usable, retry with other
5175 operand order, perhaps it will expand successfully. */
5177 comp
.code
= orig_code
;
5178 else if ((reversed
=
5179 reversed_comparison_code_parts (orig_code
, comp
.op0
, comp
.op1
,
5182 comp
.code
= reversed
;
5185 std::swap (op2
, op3
);
5189 /* Helper function that, in addition to COMPARISON, also tries
5190 the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
5191 to when we pass the specific constituents of a comparison, no
5192 additional insns are emitted for it. It might still be necessary
5193 to emit more than one insn for the final conditional move, though. */
5196 emit_conditional_move (rtx target
, rtx comparison
, rtx rev_comparison
,
5197 rtx op2
, rtx op3
, machine_mode mode
)
5199 rtx res
= emit_conditional_move_1 (target
, comparison
, op2
, op3
, mode
);
5201 if (res
!= NULL_RTX
)
5204 return emit_conditional_move_1 (target
, rev_comparison
, op3
, op2
, mode
);
5207 /* Helper for emitting a conditional move. */
5210 emit_conditional_move_1 (rtx target
, rtx comparison
,
5211 rtx op2
, rtx op3
, machine_mode mode
)
5213 enum insn_code icode
;
5215 if (comparison
== NULL_RTX
|| !COMPARISON_P (comparison
))
5218 /* If the two source operands are identical, that's just a move.
5219 As the comparison comes in non-canonicalized, we must make
5220 sure not to discard any possible side effects. If there are
5221 side effects, just let the target handle it. */
5222 if (!side_effects_p (comparison
) && rtx_equal_p (op2
, op3
))
5225 target
= gen_reg_rtx (mode
);
5227 emit_move_insn (target
, op3
);
5231 if (mode
== VOIDmode
)
5232 mode
= GET_MODE (op2
);
5234 icode
= direct_optab_handler (movcc_optab
, mode
);
5236 if (icode
== CODE_FOR_nothing
)
5240 target
= gen_reg_rtx (mode
);
5242 class expand_operand ops
[4];
5244 create_output_operand (&ops
[0], target
, mode
);
5245 create_fixed_operand (&ops
[1], comparison
);
5246 create_input_operand (&ops
[2], op2
, mode
);
5247 create_input_operand (&ops
[3], op3
, mode
);
5249 if (maybe_expand_insn (icode
, 4, ops
))
5251 if (ops
[0].value
!= target
)
5252 convert_move (target
, ops
[0].value
, false);
5260 /* Emit a conditional negate or bitwise complement using the
5261 negcc or notcc optabs if available. Return NULL_RTX if such operations
5262 are not available. Otherwise return the RTX holding the result.
5263 TARGET is the desired destination of the result. COMP is the comparison
5264 on which to negate. If COND is true move into TARGET the negation
5265 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5266 CODE is either NEG or NOT. MODE is the machine mode in which the
5267 operation is performed. */
5270 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
5271 machine_mode mode
, rtx cond
, rtx op1
,
5274 optab op
= unknown_optab
;
5277 else if (code
== NOT
)
5282 insn_code icode
= direct_optab_handler (op
, mode
);
5284 if (icode
== CODE_FOR_nothing
)
5288 target
= gen_reg_rtx (mode
);
5290 rtx_insn
*last
= get_last_insn ();
5291 class expand_operand ops
[4];
5293 create_output_operand (&ops
[0], target
, mode
);
5294 create_fixed_operand (&ops
[1], cond
);
5295 create_input_operand (&ops
[2], op1
, mode
);
5296 create_input_operand (&ops
[3], op2
, mode
);
5298 if (maybe_expand_insn (icode
, 4, ops
))
5300 if (ops
[0].value
!= target
)
5301 convert_move (target
, ops
[0].value
, false);
5305 delete_insns_since (last
);
5309 /* Emit a conditional addition instruction if the machine supports one for that
5310 condition and machine mode.
5312 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5313 the mode to use should they be constants. If it is VOIDmode, they cannot
5316 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5317 should be stored there. MODE is the mode to use should they be constants.
5318 If it is VOIDmode, they cannot both be constants.
5320 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5321 is not supported. */
5324 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
5325 machine_mode cmode
, rtx op2
, rtx op3
,
5326 machine_mode mode
, int unsignedp
)
5330 enum insn_code icode
;
5332 /* If one operand is constant, make it the second one. Only do this
5333 if the other operand is not constant as well. */
5335 if (swap_commutative_operands_p (op0
, op1
))
5337 std::swap (op0
, op1
);
5338 code
= swap_condition (code
);
5341 /* get_condition will prefer to generate LT and GT even if the old
5342 comparison was against zero, so undo that canonicalization here since
5343 comparisons against zero are cheaper. */
5344 if (code
== LT
&& op1
== const1_rtx
)
5345 code
= LE
, op1
= const0_rtx
;
5346 else if (code
== GT
&& op1
== constm1_rtx
)
5347 code
= GE
, op1
= const0_rtx
;
5349 if (cmode
== VOIDmode
)
5350 cmode
= GET_MODE (op0
);
5352 if (mode
== VOIDmode
)
5353 mode
= GET_MODE (op2
);
5355 icode
= optab_handler (addcc_optab
, mode
);
5357 if (icode
== CODE_FOR_nothing
)
5361 target
= gen_reg_rtx (mode
);
5363 code
= unsignedp
? unsigned_condition (code
) : code
;
5364 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
5366 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5367 return NULL and let the caller figure out how best to deal with this
5369 if (!COMPARISON_P (comparison
))
5372 do_pending_stack_adjust ();
5373 last
= get_last_insn ();
5374 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
5375 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
5376 &comparison
, &cmode
);
5379 class expand_operand ops
[4];
5381 create_output_operand (&ops
[0], target
, mode
);
5382 create_fixed_operand (&ops
[1], comparison
);
5383 create_input_operand (&ops
[2], op2
, mode
);
5384 create_input_operand (&ops
[3], op3
, mode
);
5385 if (maybe_expand_insn (icode
, 4, ops
))
5387 if (ops
[0].value
!= target
)
5388 convert_move (target
, ops
[0].value
, false);
5392 delete_insns_since (last
);
5396 /* These functions attempt to generate an insn body, rather than
5397 emitting the insn, but if the gen function already emits them, we
5398 make no attempt to turn them back into naked patterns. */
5400 /* Generate and return an insn body to add Y to X. */
5403 gen_add2_insn (rtx x
, rtx y
)
5405 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
5407 gcc_assert (insn_operand_matches (icode
, 0, x
));
5408 gcc_assert (insn_operand_matches (icode
, 1, x
));
5409 gcc_assert (insn_operand_matches (icode
, 2, y
));
5411 return GEN_FCN (icode
) (x
, x
, y
);
5414 /* Generate and return an insn body to add r1 and c,
5415 storing the result in r0. */
5418 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
5420 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
5422 if (icode
== CODE_FOR_nothing
5423 || !insn_operand_matches (icode
, 0, r0
)
5424 || !insn_operand_matches (icode
, 1, r1
)
5425 || !insn_operand_matches (icode
, 2, c
))
5428 return GEN_FCN (icode
) (r0
, r1
, c
);
5432 have_add2_insn (rtx x
, rtx y
)
5434 enum insn_code icode
;
5436 gcc_assert (GET_MODE (x
) != VOIDmode
);
5438 icode
= optab_handler (add_optab
, GET_MODE (x
));
5440 if (icode
== CODE_FOR_nothing
)
5443 if (!insn_operand_matches (icode
, 0, x
)
5444 || !insn_operand_matches (icode
, 1, x
)
5445 || !insn_operand_matches (icode
, 2, y
))
5451 /* Generate and return an insn body to add Y to X. */
5454 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
5456 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
5458 gcc_assert (insn_operand_matches (icode
, 0, x
));
5459 gcc_assert (insn_operand_matches (icode
, 1, y
));
5460 gcc_assert (insn_operand_matches (icode
, 2, z
));
5462 return GEN_FCN (icode
) (x
, y
, z
);
5465 /* Return true if the target implements an addptr pattern and X, Y,
5466 and Z are valid for the pattern predicates. */
5469 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
5471 enum insn_code icode
;
5473 gcc_assert (GET_MODE (x
) != VOIDmode
);
5475 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
5477 if (icode
== CODE_FOR_nothing
)
5480 if (!insn_operand_matches (icode
, 0, x
)
5481 || !insn_operand_matches (icode
, 1, y
)
5482 || !insn_operand_matches (icode
, 2, z
))
5488 /* Generate and return an insn body to subtract Y from X. */
5491 gen_sub2_insn (rtx x
, rtx y
)
5493 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
5495 gcc_assert (insn_operand_matches (icode
, 0, x
));
5496 gcc_assert (insn_operand_matches (icode
, 1, x
));
5497 gcc_assert (insn_operand_matches (icode
, 2, y
));
5499 return GEN_FCN (icode
) (x
, x
, y
);
5502 /* Generate and return an insn body to subtract r1 and c,
5503 storing the result in r0. */
5506 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
5508 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
5510 if (icode
== CODE_FOR_nothing
5511 || !insn_operand_matches (icode
, 0, r0
)
5512 || !insn_operand_matches (icode
, 1, r1
)
5513 || !insn_operand_matches (icode
, 2, c
))
5516 return GEN_FCN (icode
) (r0
, r1
, c
);
5520 have_sub2_insn (rtx x
, rtx y
)
5522 enum insn_code icode
;
5524 gcc_assert (GET_MODE (x
) != VOIDmode
);
5526 icode
= optab_handler (sub_optab
, GET_MODE (x
));
5528 if (icode
== CODE_FOR_nothing
)
5531 if (!insn_operand_matches (icode
, 0, x
)
5532 || !insn_operand_matches (icode
, 1, x
)
5533 || !insn_operand_matches (icode
, 2, y
))
5539 /* Generate the body of an insn to extend Y (with mode MFROM)
5540 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5543 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
5544 machine_mode mfrom
, int unsignedp
)
5546 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
5547 return GEN_FCN (icode
) (x
, y
);
5550 /* Generate code to convert FROM to floating point
5551 and store in TO. FROM must be fixed point and not VOIDmode.
5552 UNSIGNEDP nonzero means regard FROM as unsigned.
5553 Normally this is done by correcting the final value
5554 if it is negative. */
5557 expand_float (rtx to
, rtx from
, int unsignedp
)
5559 enum insn_code icode
;
5561 scalar_mode from_mode
, to_mode
;
5562 machine_mode fmode
, imode
;
5563 bool can_do_signed
= false;
5565 /* Crash now, because we won't be able to decide which mode to use. */
5566 gcc_assert (GET_MODE (from
) != VOIDmode
);
5568 /* Look for an insn to do the conversion. Do it in the specified
5569 modes if possible; otherwise convert either input, output or both to
5570 wider mode. If the integer mode is wider than the mode of FROM,
5571 we can do the conversion signed even if the input is unsigned. */
5573 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
5574 FOR_EACH_MODE_FROM (imode
, GET_MODE (from
))
5576 int doing_unsigned
= unsignedp
;
5578 if (fmode
!= GET_MODE (to
)
5579 && (significand_size (fmode
)
5580 < GET_MODE_UNIT_PRECISION (GET_MODE (from
))))
5583 icode
= can_float_p (fmode
, imode
, unsignedp
);
5584 if (icode
== CODE_FOR_nothing
&& unsignedp
)
5586 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
5587 if (scode
!= CODE_FOR_nothing
)
5588 can_do_signed
= true;
5589 if (imode
!= GET_MODE (from
))
5590 icode
= scode
, doing_unsigned
= 0;
5593 if (icode
!= CODE_FOR_nothing
)
5595 if (imode
!= GET_MODE (from
))
5596 from
= convert_to_mode (imode
, from
, unsignedp
);
5598 if (fmode
!= GET_MODE (to
))
5599 target
= gen_reg_rtx (fmode
);
5601 emit_unop_insn (icode
, target
, from
,
5602 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
5605 convert_move (to
, target
, 0);
5610 /* Unsigned integer, and no way to convert directly. Convert as signed,
5611 then unconditionally adjust the result. */
5614 && is_a
<scalar_mode
> (GET_MODE (to
), &to_mode
)
5615 && is_a
<scalar_mode
> (GET_MODE (from
), &from_mode
))
5617 opt_scalar_mode fmode_iter
;
5618 rtx_code_label
*label
= gen_label_rtx ();
5620 REAL_VALUE_TYPE offset
;
5622 /* Look for a usable floating mode FMODE wider than the source and at
5623 least as wide as the target. Using FMODE will avoid rounding woes
5624 with unsigned values greater than the signed maximum value. */
5626 FOR_EACH_MODE_FROM (fmode_iter
, to_mode
)
5628 scalar_mode fmode
= fmode_iter
.require ();
5629 if (GET_MODE_PRECISION (from_mode
) < GET_MODE_BITSIZE (fmode
)
5630 && can_float_p (fmode
, from_mode
, 0) != CODE_FOR_nothing
)
5634 if (!fmode_iter
.exists (&fmode
))
5636 /* There is no such mode. Pretend the target is wide enough. */
5639 /* Avoid double-rounding when TO is narrower than FROM. */
5640 if ((significand_size (fmode
) + 1)
5641 < GET_MODE_PRECISION (from_mode
))
5644 rtx_code_label
*neglabel
= gen_label_rtx ();
5646 /* Don't use TARGET if it isn't a register, is a hard register,
5647 or is the wrong mode. */
5649 || REGNO (target
) < FIRST_PSEUDO_REGISTER
5650 || GET_MODE (target
) != fmode
)
5651 target
= gen_reg_rtx (fmode
);
5654 do_pending_stack_adjust ();
5656 /* Test whether the sign bit is set. */
5657 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
5660 /* The sign bit is not set. Convert as signed. */
5661 expand_float (target
, from
, 0);
5662 emit_jump_insn (targetm
.gen_jump (label
));
5665 /* The sign bit is set.
5666 Convert to a usable (positive signed) value by shifting right
5667 one bit, while remembering if a nonzero bit was shifted
5668 out; i.e., compute (from & 1) | (from >> 1). */
5670 emit_label (neglabel
);
5671 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
5672 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
5673 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
5674 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
5676 expand_float (target
, temp
, 0);
5678 /* Multiply by 2 to undo the shift above. */
5679 temp
= expand_binop (fmode
, add_optab
, target
, target
,
5680 target
, 0, OPTAB_LIB_WIDEN
);
5682 emit_move_insn (target
, temp
);
5684 do_pending_stack_adjust ();
5690 /* If we are about to do some arithmetic to correct for an
5691 unsigned operand, do it in a pseudo-register. */
5693 if (to_mode
!= fmode
5694 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
5695 target
= gen_reg_rtx (fmode
);
5697 /* Convert as signed integer to floating. */
5698 expand_float (target
, from
, 0);
5700 /* If FROM is negative (and therefore TO is negative),
5701 correct its value by 2**bitwidth. */
5703 do_pending_stack_adjust ();
5704 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, from_mode
,
5708 real_2expN (&offset
, GET_MODE_PRECISION (from_mode
), fmode
);
5709 temp
= expand_binop (fmode
, add_optab
, target
,
5710 const_double_from_real_value (offset
, fmode
),
5711 target
, 0, OPTAB_LIB_WIDEN
);
5713 emit_move_insn (target
, temp
);
5715 do_pending_stack_adjust ();
5720 /* No hardware instruction available; call a library routine. */
5725 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
5727 if (is_narrower_int_mode (GET_MODE (from
), SImode
))
5728 from
= convert_to_mode (SImode
, from
, unsignedp
);
5730 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5731 gcc_assert (libfunc
);
5735 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5736 GET_MODE (to
), from
, GET_MODE (from
));
5737 insns
= get_insns ();
5740 emit_libcall_block (insns
, target
, value
,
5741 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
5742 GET_MODE (to
), from
));
5747 /* Copy result to requested destination
5748 if we have been computing in a temp location. */
5752 if (GET_MODE (target
) == GET_MODE (to
))
5753 emit_move_insn (to
, target
);
5755 convert_move (to
, target
, 0);
5759 /* Generate code to convert FROM to fixed point and store in TO. FROM
5760 must be floating point. */
5763 expand_fix (rtx to
, rtx from
, int unsignedp
)
5765 enum insn_code icode
;
5767 machine_mode fmode
, imode
;
5768 opt_scalar_mode fmode_iter
;
5769 bool must_trunc
= false;
5771 /* We first try to find a pair of modes, one real and one integer, at
5772 least as wide as FROM and TO, respectively, in which we can open-code
5773 this conversion. If the integer mode is wider than the mode of TO,
5774 we can do the conversion either signed or unsigned. */
5776 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5777 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5779 int doing_unsigned
= unsignedp
;
5781 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
5782 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
5783 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
5785 if (icode
!= CODE_FOR_nothing
)
5787 rtx_insn
*last
= get_last_insn ();
5789 if (fmode
!= GET_MODE (from
))
5791 if (REAL_MODE_FORMAT (GET_MODE (from
))
5792 == &arm_bfloat_half_format
5793 && REAL_MODE_FORMAT (fmode
) == &ieee_single_format
)
5794 /* The BF -> SF conversions can be just a shift, doesn't
5795 need to handle sNANs. */
5797 int save_flag_finite_math_only
= flag_finite_math_only
;
5798 flag_finite_math_only
= true;
5799 from1
= convert_to_mode (fmode
, from
, 0);
5800 flag_finite_math_only
= save_flag_finite_math_only
;
5803 from1
= convert_to_mode (fmode
, from
, 0);
5808 rtx temp
= gen_reg_rtx (GET_MODE (from1
));
5809 from1
= expand_unop (GET_MODE (from1
), ftrunc_optab
, from1
,
5813 if (imode
!= GET_MODE (to
))
5814 target
= gen_reg_rtx (imode
);
5816 if (maybe_emit_unop_insn (icode
, target
, from1
,
5817 doing_unsigned
? UNSIGNED_FIX
: FIX
))
5820 convert_move (to
, target
, unsignedp
);
5823 delete_insns_since (last
);
5827 /* For an unsigned conversion, there is one more way to do it.
5828 If we have a signed conversion, we generate code that compares
5829 the real value to the largest representable positive number. If if
5830 is smaller, the conversion is done normally. Otherwise, subtract
5831 one plus the highest signed number, convert, and add it back.
5833 We only need to check all real modes, since we know we didn't find
5834 anything with a wider integer mode.
5836 This code used to extend FP value into mode wider than the destination.
5837 This is needed for decimal float modes which cannot accurately
5838 represent one plus the highest signed number of the same size, but
5839 not for binary modes. Consider, for instance conversion from SFmode
5842 The hot path through the code is dealing with inputs smaller than 2^63
5843 and doing just the conversion, so there is no bits to lose.
5845 In the other path we know the value is positive in the range 2^63..2^64-1
5846 inclusive. (as for other input overflow happens and result is undefined)
5847 So we know that the most important bit set in mantissa corresponds to
5848 2^63. The subtraction of 2^63 should not generate any rounding as it
5849 simply clears out that bit. The rest is trivial. */
5851 scalar_int_mode to_mode
;
5853 && is_a
<scalar_int_mode
> (GET_MODE (to
), &to_mode
)
5854 && HWI_COMPUTABLE_MODE_P (to_mode
))
5855 FOR_EACH_MODE_FROM (fmode_iter
, as_a
<scalar_mode
> (GET_MODE (from
)))
5857 scalar_mode fmode
= fmode_iter
.require ();
5858 if (CODE_FOR_nothing
!= can_fix_p (to_mode
, fmode
,
5860 && (!DECIMAL_FLOAT_MODE_P (fmode
)
5861 || (GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (to_mode
))))
5864 REAL_VALUE_TYPE offset
;
5866 rtx_code_label
*lab1
, *lab2
;
5869 bitsize
= GET_MODE_PRECISION (to_mode
);
5870 real_2expN (&offset
, bitsize
- 1, fmode
);
5871 limit
= const_double_from_real_value (offset
, fmode
);
5872 lab1
= gen_label_rtx ();
5873 lab2
= gen_label_rtx ();
5875 if (fmode
!= GET_MODE (from
))
5877 if (REAL_MODE_FORMAT (GET_MODE (from
))
5878 == &arm_bfloat_half_format
5879 && REAL_MODE_FORMAT (fmode
) == &ieee_single_format
)
5880 /* The BF -> SF conversions can be just a shift, doesn't
5881 need to handle sNANs. */
5883 int save_flag_finite_math_only
= flag_finite_math_only
;
5884 flag_finite_math_only
= true;
5885 from
= convert_to_mode (fmode
, from
, 0);
5886 flag_finite_math_only
= save_flag_finite_math_only
;
5889 from
= convert_to_mode (fmode
, from
, 0);
5892 /* See if we need to do the subtraction. */
5893 do_pending_stack_adjust ();
5894 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
,
5895 GET_MODE (from
), 0, lab1
);
5897 /* If not, do the signed "fix" and branch around fixup code. */
5898 expand_fix (to
, from
, 0);
5899 emit_jump_insn (targetm
.gen_jump (lab2
));
5902 /* Otherwise, subtract 2**(N-1), convert to signed number,
5903 then add 2**(N-1). Do the addition using XOR since this
5904 will often generate better code. */
5906 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
5907 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
5908 expand_fix (to
, target
, 0);
5909 target
= expand_binop (to_mode
, xor_optab
, to
,
5911 (HOST_WIDE_INT_1
<< (bitsize
- 1),
5913 to
, 1, OPTAB_LIB_WIDEN
);
5916 emit_move_insn (to
, target
);
5920 if (optab_handler (mov_optab
, to_mode
) != CODE_FOR_nothing
)
5922 /* Make a place for a REG_NOTE and add it. */
5923 insn
= emit_move_insn (to
, to
);
5924 set_dst_reg_note (insn
, REG_EQUAL
,
5925 gen_rtx_fmt_e (UNSIGNED_FIX
, to_mode
,
5935 if (REAL_MODE_FORMAT (GET_MODE (from
)) == &arm_bfloat_half_format
5936 && REAL_MODE_FORMAT (SFmode
) == &ieee_single_format
)
5937 /* We don't have BF -> TI library functions, use BF -> SF -> TI
5938 instead but the BF -> SF conversion can be just a shift, doesn't
5939 need to handle sNANs. */
5941 int save_flag_finite_math_only
= flag_finite_math_only
;
5942 flag_finite_math_only
= true;
5943 from
= convert_to_mode (SFmode
, from
, 0);
5944 flag_finite_math_only
= save_flag_finite_math_only
;
5945 expand_fix (to
, from
, unsignedp
);
5950 /* We can't do it with an insn, so use a library call. But first ensure
5951 that the mode of TO is at least as wide as SImode, since those are the
5952 only library calls we know about. */
5954 if (is_narrower_int_mode (GET_MODE (to
), SImode
))
5956 target
= gen_reg_rtx (SImode
);
5958 expand_fix (target
, from
, unsignedp
);
5966 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
5967 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5968 gcc_assert (libfunc
);
5972 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5973 GET_MODE (to
), from
, GET_MODE (from
));
5974 insns
= get_insns ();
5977 emit_libcall_block (insns
, target
, value
,
5978 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5979 GET_MODE (to
), from
));
5984 if (GET_MODE (to
) == GET_MODE (target
))
5985 emit_move_insn (to
, target
);
5987 convert_move (to
, target
, 0);
5992 /* Promote integer arguments for a libcall if necessary.
5993 emit_library_call_value cannot do the promotion because it does not
5994 know if it should do a signed or unsigned promotion. This is because
5995 there are no tree types defined for libcalls. */
5998 prepare_libcall_arg (rtx arg
, int uintp
)
6000 scalar_int_mode mode
;
6001 machine_mode arg_mode
;
6002 if (is_a
<scalar_int_mode
> (GET_MODE (arg
), &mode
))
6004 /* If we need to promote the integer function argument we need to do
6005 it here instead of inside emit_library_call_value because in
6006 emit_library_call_value we don't know if we should do a signed or
6007 unsigned promotion. */
6010 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
6011 &unsigned_p
, NULL_TREE
, 0);
6012 if (arg_mode
!= mode
)
6013 return convert_to_mode (arg_mode
, arg
, uintp
);
6018 /* Generate code to convert FROM or TO a fixed-point.
6019 If UINTP is true, either TO or FROM is an unsigned integer.
6020 If SATP is true, we need to saturate the result. */
6023 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
6025 machine_mode to_mode
= GET_MODE (to
);
6026 machine_mode from_mode
= GET_MODE (from
);
6028 enum rtx_code this_code
;
6029 enum insn_code code
;
6034 if (to_mode
== from_mode
)
6036 emit_move_insn (to
, from
);
6042 tab
= satp
? satfractuns_optab
: fractuns_optab
;
6043 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
6047 tab
= satp
? satfract_optab
: fract_optab
;
6048 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
6050 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
6051 if (code
!= CODE_FOR_nothing
)
6053 emit_unop_insn (code
, to
, from
, this_code
);
6057 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
6058 gcc_assert (libfunc
);
6060 from
= prepare_libcall_arg (from
, uintp
);
6061 from_mode
= GET_MODE (from
);
6064 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
6066 insns
= get_insns ();
6069 emit_libcall_block (insns
, to
, value
,
6070 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
6073 /* Generate code to convert FROM to fixed point and store in TO. FROM
6074 must be floating point, TO must be signed. Use the conversion optab
6075 TAB to do the conversion. */
6078 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
6080 enum insn_code icode
;
6082 machine_mode fmode
, imode
;
6084 /* We first try to find a pair of modes, one real and one integer, at
6085 least as wide as FROM and TO, respectively, in which we can open-code
6086 this conversion. If the integer mode is wider than the mode of TO,
6087 we can do the conversion either signed or unsigned. */
6089 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
6090 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
6092 icode
= convert_optab_handler (tab
, imode
, fmode
,
6093 insn_optimization_type ());
6094 if (icode
!= CODE_FOR_nothing
)
6096 rtx_insn
*last
= get_last_insn ();
6097 if (fmode
!= GET_MODE (from
))
6098 from
= convert_to_mode (fmode
, from
, 0);
6100 if (imode
!= GET_MODE (to
))
6101 target
= gen_reg_rtx (imode
);
6103 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
6105 delete_insns_since (last
);
6109 convert_move (to
, target
, 0);
6117 /* Report whether we have an instruction to perform the operation
6118 specified by CODE on operands of mode MODE. */
6120 have_insn_for (enum rtx_code code
, machine_mode mode
)
6122 return (code_to_optab (code
)
6123 && (optab_handler (code_to_optab (code
), mode
)
6124 != CODE_FOR_nothing
));
6127 /* Print information about the current contents of the optabs on
6131 debug_optab_libfuncs (void)
6135 /* Dump the arithmetic optabs. */
6136 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
6137 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
6139 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
6142 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
6143 fprintf (stderr
, "%s\t%s:\t%s\n",
6144 GET_RTX_NAME (optab_to_code ((optab
) i
)),
6150 /* Dump the conversion optabs. */
6151 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
6152 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
6153 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
6155 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
6159 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
6160 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
6161 GET_RTX_NAME (optab_to_code ((optab
) i
)),
6169 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6170 CODE. Return 0 on failure. */
6173 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
6175 machine_mode mode
= GET_MODE (op1
);
6176 enum insn_code icode
;
6180 if (mode
== VOIDmode
)
6183 icode
= optab_handler (ctrap_optab
, mode
);
6184 if (icode
== CODE_FOR_nothing
)
6187 /* Some targets only accept a zero trap code. */
6188 if (!insn_operand_matches (icode
, 3, tcode
))
6191 do_pending_stack_adjust ();
6193 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
6198 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
6201 /* If that failed, then give up. */
6209 insn
= get_insns ();
6214 /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
6215 or unsigned operation code. */
6218 get_rtx_code_1 (enum tree_code tcode
, bool unsignedp
)
6230 code
= unsignedp
? LTU
: LT
;
6233 code
= unsignedp
? LEU
: LE
;
6236 code
= unsignedp
? GTU
: GT
;
6239 code
= unsignedp
? GEU
: GE
;
6242 case UNORDERED_EXPR
:
6282 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6283 or unsigned operation code. */
6286 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
6288 enum rtx_code code
= get_rtx_code_1 (tcode
, unsignedp
);
6289 gcc_assert (code
!= UNKNOWN
);
6293 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6294 select signed or unsigned operators. OPNO holds the index of the
6295 first comparison operand for insn ICODE. Do not generate the
6296 compare instruction itself. */
6299 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
6300 tree t_op0
, tree t_op1
, bool unsignedp
,
6301 enum insn_code icode
, unsigned int opno
)
6303 class expand_operand ops
[2];
6304 rtx rtx_op0
, rtx_op1
;
6305 machine_mode m0
, m1
;
6306 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
6308 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
6310 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6311 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6312 cases, use the original mode. */
6313 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
6315 m0
= GET_MODE (rtx_op0
);
6317 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
6319 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
6321 m1
= GET_MODE (rtx_op1
);
6323 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
6325 create_input_operand (&ops
[0], rtx_op0
, m0
);
6326 create_input_operand (&ops
[1], rtx_op1
, m1
);
6327 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
6329 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
6332 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
6333 the first vec_perm operand, assuming the second operand (for left shift
6334 first operand) is a constant vector of zeros. Return the shift distance
6335 in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6336 mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6337 shift or vec_shl_optab for left shift. */
6339 shift_amt_for_vec_perm_mask (machine_mode mode
, const vec_perm_indices
&sel
,
6342 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (mode
);
6343 poly_int64 first
= sel
[0];
6344 if (maybe_ge (sel
[0], GET_MODE_NUNITS (mode
)))
6347 if (shift_optab
== vec_shl_optab
)
6350 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
6352 unsigned firstidx
= 0;
6353 for (unsigned int i
= 0; i
< nelt
; i
++)
6355 if (known_eq (sel
[i
], nelt
))
6357 if (i
== 0 || firstidx
)
6362 ? maybe_ne (sel
[i
], nelt
+ i
- firstidx
)
6363 : maybe_ge (sel
[i
], nelt
))
6371 else if (!sel
.series_p (0, 1, first
, 1))
6374 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
6376 for (unsigned int i
= 1; i
< nelt
; i
++)
6378 poly_int64 expected
= i
+ first
;
6379 /* Indices into the second vector are all equivalent. */
6380 if (maybe_lt (sel
[i
], nelt
)
6381 ? maybe_ne (sel
[i
], expected
)
6382 : maybe_lt (expected
, nelt
))
6387 return gen_int_shift_amount (mode
, first
* bitsize
);
6390 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6393 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
6394 rtx v0
, rtx v1
, rtx sel
)
6396 machine_mode tmode
= GET_MODE (target
);
6397 machine_mode smode
= GET_MODE (sel
);
6398 class expand_operand ops
[4];
6400 gcc_assert (GET_MODE_CLASS (smode
) == MODE_VECTOR_INT
6401 || related_int_vector_mode (tmode
).require () == smode
);
6402 create_output_operand (&ops
[0], target
, tmode
);
6403 create_input_operand (&ops
[3], sel
, smode
);
6405 /* Make an effort to preserve v0 == v1. The target expander is able to
6406 rely on this to determine if we're permuting a single input operand. */
6407 if (rtx_equal_p (v0
, v1
))
6409 if (!insn_operand_matches (icode
, 1, v0
))
6410 v0
= force_reg (tmode
, v0
);
6411 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
6412 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
6414 create_fixed_operand (&ops
[1], v0
);
6415 create_fixed_operand (&ops
[2], v0
);
6419 create_input_operand (&ops
[1], v0
, tmode
);
6420 create_input_operand (&ops
[2], v1
, tmode
);
6423 if (maybe_expand_insn (icode
, 4, ops
))
6424 return ops
[0].value
;
6428 /* Implement a permutation of vectors v0 and v1 using the permutation
6429 vector in SEL and return the result. Use TARGET to hold the result
6430 if nonnull and convenient.
6432 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
6433 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
6434 to have a particular mode. */
6437 expand_vec_perm_const (machine_mode mode
, rtx v0
, rtx v1
,
6438 const vec_perm_builder
&sel
, machine_mode sel_mode
,
6441 if (!target
|| !register_operand (target
, mode
))
6442 target
= gen_reg_rtx (mode
);
6444 /* Set QIMODE to a different vector mode with byte elements.
6445 If no such mode, or if MODE already has byte elements, use VOIDmode. */
6446 machine_mode qimode
;
6447 if (!qimode_for_vec_perm (mode
).exists (&qimode
))
6450 rtx_insn
*last
= get_last_insn ();
6452 bool single_arg_p
= rtx_equal_p (v0
, v1
);
6453 /* Always specify two input vectors here and leave the target to handle
6454 cases in which the inputs are equal. Not all backends can cope with
6455 the single-input representation when testing for a double-input
6456 target instruction. */
6457 vec_perm_indices
indices (sel
, 2, GET_MODE_NUNITS (mode
));
6459 /* See if this can be handled with a vec_shr or vec_shl. We only do this
6460 if the second (for vec_shr) or first (for vec_shl) vector is all
6462 insn_code shift_code
= CODE_FOR_nothing
;
6463 insn_code shift_code_qi
= CODE_FOR_nothing
;
6464 optab shift_optab
= unknown_optab
;
6466 if (v1
== CONST0_RTX (GET_MODE (v1
)))
6467 shift_optab
= vec_shr_optab
;
6468 else if (v0
== CONST0_RTX (GET_MODE (v0
)))
6470 shift_optab
= vec_shl_optab
;
6473 if (shift_optab
!= unknown_optab
)
6475 shift_code
= optab_handler (shift_optab
, mode
);
6476 shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
6477 ? optab_handler (shift_optab
, qimode
)
6478 : CODE_FOR_nothing
);
6480 if (shift_code
!= CODE_FOR_nothing
|| shift_code_qi
!= CODE_FOR_nothing
)
6482 rtx shift_amt
= shift_amt_for_vec_perm_mask (mode
, indices
, shift_optab
);
6485 class expand_operand ops
[3];
6486 if (shift_amt
== const0_rtx
)
6488 if (shift_code
!= CODE_FOR_nothing
)
6490 create_output_operand (&ops
[0], target
, mode
);
6491 create_input_operand (&ops
[1], v2
, mode
);
6492 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
6493 if (maybe_expand_insn (shift_code
, 3, ops
))
6494 return ops
[0].value
;
6496 if (shift_code_qi
!= CODE_FOR_nothing
)
6498 rtx tmp
= gen_reg_rtx (qimode
);
6499 create_output_operand (&ops
[0], tmp
, qimode
);
6500 create_input_operand (&ops
[1], gen_lowpart (qimode
, v2
), qimode
);
6501 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
6502 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
6503 return gen_lowpart (mode
, ops
[0].value
);
6508 if (targetm
.vectorize
.vec_perm_const
!= NULL
)
6513 gcc_checking_assert (GET_MODE (v0
) == GET_MODE (v1
));
6514 machine_mode op_mode
= GET_MODE (v0
);
6515 if (targetm
.vectorize
.vec_perm_const (mode
, op_mode
, target
, v0
, v1
,
6520 /* Fall back to a constant byte-based permutation. */
6521 vec_perm_indices qimode_indices
;
6522 rtx target_qi
= NULL_RTX
, v0_qi
= NULL_RTX
, v1_qi
= NULL_RTX
;
6523 if (qimode
!= VOIDmode
)
6525 qimode_indices
.new_expanded_vector (indices
, GET_MODE_UNIT_SIZE (mode
));
6526 target_qi
= gen_reg_rtx (qimode
);
6527 v0_qi
= gen_lowpart (qimode
, v0
);
6528 v1_qi
= gen_lowpart (qimode
, v1
);
6529 if (targetm
.vectorize
.vec_perm_const
!= NULL
6530 && targetm
.vectorize
.vec_perm_const (qimode
, qimode
, target_qi
, v0_qi
,
6531 v1_qi
, qimode_indices
))
6532 return gen_lowpart (mode
, target_qi
);
6535 v0
= force_reg (mode
, v0
);
6538 v1
= force_reg (mode
, v1
);
6540 /* Otherwise expand as a fully variable permuation. */
6542 /* The optabs are only defined for selectors with the same width
6543 as the values being permuted. */
6544 machine_mode required_sel_mode
;
6545 if (!related_int_vector_mode (mode
).exists (&required_sel_mode
))
6547 delete_insns_since (last
);
6551 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
6552 If that isn't the mode we want then we need to prove that using
6553 REQUIRED_SEL_MODE is OK. */
6554 if (sel_mode
!= required_sel_mode
)
6556 if (!selector_fits_mode_p (required_sel_mode
, indices
))
6558 delete_insns_since (last
);
6561 sel_mode
= required_sel_mode
;
6564 insn_code icode
= direct_optab_handler (vec_perm_optab
, mode
);
6565 if (icode
!= CODE_FOR_nothing
)
6567 rtx sel_rtx
= vec_perm_indices_to_rtx (sel_mode
, indices
);
6568 rtx tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel_rtx
);
6573 if (qimode
!= VOIDmode
6574 && selector_fits_mode_p (qimode
, qimode_indices
))
6576 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6577 if (icode
!= CODE_FOR_nothing
)
6579 rtx sel_qi
= vec_perm_indices_to_rtx (qimode
, qimode_indices
);
6580 rtx tmp
= expand_vec_perm_1 (icode
, target_qi
, v0_qi
, v1_qi
, sel_qi
);
6582 return gen_lowpart (mode
, tmp
);
6586 delete_insns_since (last
);
6590 /* Implement a permutation of vectors v0 and v1 using the permutation
6591 vector in SEL and return the result. Use TARGET to hold the result
6592 if nonnull and convenient.
6594 MODE is the mode of the vectors being permuted (V0 and V1).
6595 SEL must have the integer equivalent of MODE and is known to be
6596 unsuitable for permutes with a constant permutation vector. */
6599 expand_vec_perm_var (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
6601 enum insn_code icode
;
6605 u
= GET_MODE_UNIT_SIZE (mode
);
6607 if (!target
|| GET_MODE (target
) != mode
)
6608 target
= gen_reg_rtx (mode
);
6610 icode
= direct_optab_handler (vec_perm_optab
, mode
);
6611 if (icode
!= CODE_FOR_nothing
)
6613 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
6618 /* As a special case to aid several targets, lower the element-based
6619 permutation to a byte-based permutation and try again. */
6620 machine_mode qimode
;
6621 if (!qimode_for_vec_perm (mode
).exists (&qimode
)
6622 || maybe_gt (GET_MODE_NUNITS (qimode
), GET_MODE_MASK (QImode
) + 1))
6624 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6625 if (icode
== CODE_FOR_nothing
)
6628 /* Multiply each element by its byte size. */
6629 machine_mode selmode
= GET_MODE (sel
);
6631 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
6632 NULL
, 0, OPTAB_DIRECT
);
6634 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
6635 gen_int_shift_amount (selmode
, exact_log2 (u
)),
6636 NULL
, 0, OPTAB_DIRECT
);
6637 gcc_assert (sel
!= NULL
);
6639 /* Broadcast the low byte each element into each of its bytes.
6640 The encoding has U interleaved stepped patterns, one for each
6641 byte of an element. */
6642 vec_perm_builder
const_sel (GET_MODE_SIZE (mode
), u
, 3);
6643 unsigned int low_byte_in_u
= BYTES_BIG_ENDIAN
? u
- 1 : 0;
6644 for (i
= 0; i
< 3; ++i
)
6645 for (unsigned int j
= 0; j
< u
; ++j
)
6646 const_sel
.quick_push (i
* u
+ low_byte_in_u
);
6647 sel
= gen_lowpart (qimode
, sel
);
6648 sel
= expand_vec_perm_const (qimode
, sel
, sel
, const_sel
, qimode
, NULL
);
6649 gcc_assert (sel
!= NULL
);
6651 /* Add the byte offset to each byte element. */
6652 /* Note that the definition of the indicies here is memory ordering,
6653 so there should be no difference between big and little endian. */
6654 rtx_vector_builder
byte_indices (qimode
, u
, 1);
6655 for (i
= 0; i
< u
; ++i
)
6656 byte_indices
.quick_push (GEN_INT (i
));
6657 tmp
= byte_indices
.build ();
6658 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
6659 sel
, 0, OPTAB_DIRECT
);
6660 gcc_assert (sel_qi
!= NULL
);
6662 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
6663 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
6664 gen_lowpart (qimode
, v1
), sel_qi
);
6666 tmp
= gen_lowpart (mode
, tmp
);
6670 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
6671 Use TARGET for the result if nonnull and convenient. */
6674 expand_vec_series_expr (machine_mode vmode
, rtx op0
, rtx op1
, rtx target
)
6676 class expand_operand ops
[3];
6677 enum insn_code icode
;
6678 machine_mode emode
= GET_MODE_INNER (vmode
);
6680 icode
= direct_optab_handler (vec_series_optab
, vmode
);
6681 gcc_assert (icode
!= CODE_FOR_nothing
);
6683 create_output_operand (&ops
[0], target
, vmode
);
6684 create_input_operand (&ops
[1], op0
, emode
);
6685 create_input_operand (&ops
[2], op1
, emode
);
6687 expand_insn (icode
, 3, ops
);
6688 return ops
[0].value
;
6691 /* Generate insns for a vector comparison into a mask. */
6694 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
6696 class expand_operand ops
[4];
6697 enum insn_code icode
;
6699 machine_mode mask_mode
= TYPE_MODE (type
);
6703 enum tree_code tcode
;
6705 op0a
= TREE_OPERAND (exp
, 0);
6706 op0b
= TREE_OPERAND (exp
, 1);
6707 tcode
= TREE_CODE (exp
);
6709 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
6710 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
6712 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
6713 if (icode
== CODE_FOR_nothing
)
6715 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
6716 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
6717 if (icode
== CODE_FOR_nothing
)
6721 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
6722 unsignedp
, icode
, 2);
6723 create_output_operand (&ops
[0], target
, mask_mode
);
6724 create_fixed_operand (&ops
[1], comparison
);
6725 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
6726 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
6727 expand_insn (icode
, 4, ops
);
6728 return ops
[0].value
;
6731 /* Expand a highpart multiply. */
6734 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
6735 rtx target
, bool uns_p
)
6737 class expand_operand eops
[3];
6738 enum insn_code icode
;
6744 method
= can_mult_highpart_p (mode
, uns_p
);
6750 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
6751 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
6754 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
6755 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
6758 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
6759 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
6760 if (BYTES_BIG_ENDIAN
)
6761 std::swap (tab1
, tab2
);
6767 icode
= optab_handler (tab1
, mode
);
6768 wmode
= insn_data
[icode
].operand
[0].mode
;
6769 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode
),
6770 GET_MODE_NUNITS (mode
)));
6771 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode
), GET_MODE_SIZE (mode
)));
6773 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
6774 create_input_operand (&eops
[1], op0
, mode
);
6775 create_input_operand (&eops
[2], op1
, mode
);
6776 expand_insn (icode
, 3, eops
);
6777 m1
= gen_lowpart (mode
, eops
[0].value
);
6779 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
6780 create_input_operand (&eops
[1], op0
, mode
);
6781 create_input_operand (&eops
[2], op1
, mode
);
6782 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
6783 m2
= gen_lowpart (mode
, eops
[0].value
);
6785 vec_perm_builder sel
;
6788 /* The encoding has 2 interleaved stepped patterns. */
6789 sel
.new_vector (GET_MODE_NUNITS (mode
), 2, 3);
6790 for (i
= 0; i
< 6; ++i
)
6791 sel
.quick_push (!BYTES_BIG_ENDIAN
+ (i
& ~1)
6792 + ((i
& 1) ? GET_MODE_NUNITS (mode
) : 0));
6796 /* The encoding has a single interleaved stepped pattern. */
6797 sel
.new_vector (GET_MODE_NUNITS (mode
), 1, 3);
6798 for (i
= 0; i
< 3; ++i
)
6799 sel
.quick_push (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
6802 return expand_vec_perm_const (mode
, m1
, m2
, sel
, BLKmode
, target
);
6805 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6809 find_cc_set (rtx x
, const_rtx pat
, void *data
)
6811 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
6812 && GET_CODE (pat
) == SET
)
6814 rtx
*p_cc_reg
= (rtx
*) data
;
6815 gcc_assert (!*p_cc_reg
);
6820 /* This is a helper function for the other atomic operations. This function
6821 emits a loop that contains SEQ that iterates until a compare-and-swap
6822 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6823 a set of instructions that takes a value from OLD_REG as an input and
6824 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6825 set to the current contents of MEM. After SEQ, a compare-and-swap will
6826 attempt to update MEM with NEW_REG. The function returns true when the
6827 loop was generated successfully. */
6830 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
6832 machine_mode mode
= GET_MODE (mem
);
6833 rtx_code_label
*label
;
6834 rtx cmp_reg
, success
, oldval
;
6836 /* The loop we want to generate looks like
6842 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6846 Note that we only do the plain load from memory once. Subsequent
6847 iterations use the value loaded by the compare-and-swap pattern. */
6849 label
= gen_label_rtx ();
6850 cmp_reg
= gen_reg_rtx (mode
);
6852 emit_move_insn (cmp_reg
, mem
);
6854 emit_move_insn (old_reg
, cmp_reg
);
6860 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
6861 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
6865 if (oldval
!= cmp_reg
)
6866 emit_move_insn (cmp_reg
, oldval
);
6868 /* Mark this jump predicted not taken. */
6869 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
6870 GET_MODE (success
), 1, label
,
6871 profile_probability::guessed_never ());
6876 /* This function tries to emit an atomic_exchange intruction. VAL is written
6877 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6878 using TARGET if possible. */
6881 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6883 machine_mode mode
= GET_MODE (mem
);
6884 enum insn_code icode
;
6886 /* If the target supports the exchange directly, great. */
6887 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
6888 if (icode
!= CODE_FOR_nothing
)
6890 class expand_operand ops
[4];
6892 create_output_operand (&ops
[0], target
, mode
);
6893 create_fixed_operand (&ops
[1], mem
);
6894 create_input_operand (&ops
[2], val
, mode
);
6895 create_integer_operand (&ops
[3], model
);
6896 if (maybe_expand_insn (icode
, 4, ops
))
6897 return ops
[0].value
;
6903 /* This function tries to implement an atomic exchange operation using
6904 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6905 The previous contents of *MEM are returned, using TARGET if possible.
6906 Since this instructionn is an acquire barrier only, stronger memory
6907 models may require additional barriers to be emitted. */
6910 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
6911 enum memmodel model
)
6913 machine_mode mode
= GET_MODE (mem
);
6914 enum insn_code icode
;
6915 rtx_insn
*last_insn
= get_last_insn ();
6917 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
6919 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6920 exists, and the memory model is stronger than acquire, add a release
6921 barrier before the instruction. */
6923 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
6924 expand_mem_thread_fence (model
);
6926 if (icode
!= CODE_FOR_nothing
)
6928 class expand_operand ops
[3];
6929 create_output_operand (&ops
[0], target
, mode
);
6930 create_fixed_operand (&ops
[1], mem
);
6931 create_input_operand (&ops
[2], val
, mode
);
6932 if (maybe_expand_insn (icode
, 3, ops
))
6933 return ops
[0].value
;
6936 /* If an external test-and-set libcall is provided, use that instead of
6937 any external compare-and-swap that we might get from the compare-and-
6938 swap-loop expansion later. */
6939 if (!can_compare_and_swap_p (mode
, false))
6941 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
6942 if (libfunc
!= NULL
)
6946 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6947 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6948 mode
, addr
, ptr_mode
,
6953 /* If the test_and_set can't be emitted, eliminate any barrier that might
6954 have been emitted. */
6955 delete_insns_since (last_insn
);
6959 /* This function tries to implement an atomic exchange operation using a
6960 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6961 *MEM are returned, using TARGET if possible. No memory model is required
6962 since a compare_and_swap loop is seq-cst. */
6965 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
6967 machine_mode mode
= GET_MODE (mem
);
6969 if (can_compare_and_swap_p (mode
, true))
6971 if (!target
|| !register_operand (target
, mode
))
6972 target
= gen_reg_rtx (mode
);
6973 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
6980 /* This function tries to implement an atomic test-and-set operation
6981 using the atomic_test_and_set instruction pattern. A boolean value
6982 is returned from the operation, using TARGET if possible. */
6985 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6987 machine_mode pat_bool_mode
;
6988 class expand_operand ops
[3];
6990 if (!targetm
.have_atomic_test_and_set ())
6993 /* While we always get QImode from __atomic_test_and_set, we get
6994 other memory modes from __sync_lock_test_and_set. Note that we
6995 use no endian adjustment here. This matches the 4.6 behavior
6996 in the Sparc backend. */
6997 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
6998 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
6999 if (GET_MODE (mem
) != QImode
)
7000 mem
= adjust_address_nv (mem
, QImode
, 0);
7002 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
7003 create_output_operand (&ops
[0], target
, pat_bool_mode
);
7004 create_fixed_operand (&ops
[1], mem
);
7005 create_integer_operand (&ops
[2], model
);
7007 if (maybe_expand_insn (icode
, 3, ops
))
7008 return ops
[0].value
;
7012 /* This function expands the legacy _sync_lock test_and_set operation which is
7013 generally an atomic exchange. Some limited targets only allow the
7014 constant 1 to be stored. This is an ACQUIRE operation.
7016 TARGET is an optional place to stick the return value.
7017 MEM is where VAL is stored. */
7020 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
7024 /* Try an atomic_exchange first. */
7025 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
7029 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
7030 MEMMODEL_SYNC_ACQUIRE
);
7034 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
7038 /* If there are no other options, try atomic_test_and_set if the value
7039 being stored is 1. */
7040 if (val
== const1_rtx
)
7041 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
7046 /* This function expands the atomic test_and_set operation:
7047 atomically store a boolean TRUE into MEM and return the previous value.
7049 MEMMODEL is the memory model variant to use.
7050 TARGET is an optional place to stick the return value. */
7053 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
7055 machine_mode mode
= GET_MODE (mem
);
7056 rtx ret
, trueval
, subtarget
;
7058 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
7062 /* Be binary compatible with non-default settings of trueval, and different
7063 cpu revisions. E.g. one revision may have atomic-test-and-set, but
7064 another only has atomic-exchange. */
7065 if (targetm
.atomic_test_and_set_trueval
== 1)
7067 trueval
= const1_rtx
;
7068 subtarget
= target
? target
: gen_reg_rtx (mode
);
7072 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
7073 subtarget
= gen_reg_rtx (mode
);
7076 /* Try the atomic-exchange optab... */
7077 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
7079 /* ... then an atomic-compare-and-swap loop ... */
7081 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
7083 /* ... before trying the vaguely defined legacy lock_test_and_set. */
7085 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
7087 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7088 things with the value 1. Thus we try again without trueval. */
7089 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
7091 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
7095 /* Rectify the not-one trueval. */
7096 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
7104 /* This function expands the atomic exchange operation:
7105 atomically store VAL in MEM and return the previous value in MEM.
7107 MEMMODEL is the memory model variant to use.
7108 TARGET is an optional place to stick the return value. */
7111 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
7113 machine_mode mode
= GET_MODE (mem
);
7116 /* If loads are not atomic for the required size and we are not called to
7117 provide a __sync builtin, do not do anything so that we stay consistent
7118 with atomic loads of the same size. */
7119 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
7122 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7124 /* Next try a compare-and-swap loop for the exchange. */
7126 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
7131 /* This function expands the atomic compare exchange operation:
7133 *PTARGET_BOOL is an optional place to store the boolean success/failure.
7134 *PTARGET_OVAL is an optional place to store the old value from memory.
7135 Both target parameters may be NULL or const0_rtx to indicate that we do
7136 not care about that return value. Both target parameters are updated on
7137 success to the actual location of the corresponding result.
7139 MEMMODEL is the memory model variant to use.
7141 The return value of the function is true for success. */
7144 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
7145 rtx mem
, rtx expected
, rtx desired
,
7146 bool is_weak
, enum memmodel succ_model
,
7147 enum memmodel fail_model
)
7149 machine_mode mode
= GET_MODE (mem
);
7150 class expand_operand ops
[8];
7151 enum insn_code icode
;
7152 rtx target_oval
, target_bool
= NULL_RTX
;
7155 /* If loads are not atomic for the required size and we are not called to
7156 provide a __sync builtin, do not do anything so that we stay consistent
7157 with atomic loads of the same size. */
7158 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
7161 /* Load expected into a register for the compare and swap. */
7162 if (MEM_P (expected
))
7163 expected
= copy_to_reg (expected
);
7165 /* Make sure we always have some place to put the return oldval.
7166 Further, make sure that place is distinct from the input expected,
7167 just in case we need that path down below. */
7168 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
7169 ptarget_oval
= NULL
;
7171 if (ptarget_oval
== NULL
7172 || (target_oval
= *ptarget_oval
) == NULL
7173 || reg_overlap_mentioned_p (expected
, target_oval
))
7174 target_oval
= gen_reg_rtx (mode
);
7176 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
7177 if (icode
!= CODE_FOR_nothing
)
7179 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
7181 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
7182 ptarget_bool
= NULL
;
7184 /* Make sure we always have a place for the bool operand. */
7185 if (ptarget_bool
== NULL
7186 || (target_bool
= *ptarget_bool
) == NULL
7187 || GET_MODE (target_bool
) != bool_mode
)
7188 target_bool
= gen_reg_rtx (bool_mode
);
7190 /* Emit the compare_and_swap. */
7191 create_output_operand (&ops
[0], target_bool
, bool_mode
);
7192 create_output_operand (&ops
[1], target_oval
, mode
);
7193 create_fixed_operand (&ops
[2], mem
);
7194 create_input_operand (&ops
[3], expected
, mode
);
7195 create_input_operand (&ops
[4], desired
, mode
);
7196 create_integer_operand (&ops
[5], is_weak
);
7197 create_integer_operand (&ops
[6], succ_model
);
7198 create_integer_operand (&ops
[7], fail_model
);
7199 if (maybe_expand_insn (icode
, 8, ops
))
7201 /* Return success/failure. */
7202 target_bool
= ops
[0].value
;
7203 target_oval
= ops
[1].value
;
7208 /* Otherwise fall back to the original __sync_val_compare_and_swap
7209 which is always seq-cst. */
7210 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
7211 if (icode
!= CODE_FOR_nothing
)
7215 create_output_operand (&ops
[0], target_oval
, mode
);
7216 create_fixed_operand (&ops
[1], mem
);
7217 create_input_operand (&ops
[2], expected
, mode
);
7218 create_input_operand (&ops
[3], desired
, mode
);
7219 if (!maybe_expand_insn (icode
, 4, ops
))
7222 target_oval
= ops
[0].value
;
7224 /* If the caller isn't interested in the boolean return value,
7225 skip the computation of it. */
7226 if (ptarget_bool
== NULL
)
7229 /* Otherwise, work out if the compare-and-swap succeeded. */
7231 if (have_insn_for (COMPARE
, CCmode
))
7232 note_stores (get_last_insn (), find_cc_set
, &cc_reg
);
7235 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
7236 const0_rtx
, VOIDmode
, 0, 1);
7239 goto success_bool_from_val
;
7242 /* Also check for library support for __sync_val_compare_and_swap. */
7243 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
7244 if (libfunc
!= NULL
)
7246 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
7247 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
7248 mode
, addr
, ptr_mode
,
7249 expected
, mode
, desired
, mode
);
7250 emit_move_insn (target_oval
, target
);
7252 /* Compute the boolean return value only if requested. */
7254 goto success_bool_from_val
;
7262 success_bool_from_val
:
7263 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
7264 expected
, VOIDmode
, 1, 1);
7266 /* Make sure that the oval output winds up where the caller asked. */
7268 *ptarget_oval
= target_oval
;
7270 *ptarget_bool
= target_bool
;
7274 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
7277 expand_asm_memory_blockage (void)
7281 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
7282 rtvec_alloc (0), rtvec_alloc (0),
7283 rtvec_alloc (0), UNKNOWN_LOCATION
);
7284 MEM_VOLATILE_P (asm_op
) = 1;
7286 clob
= gen_rtx_SCRATCH (VOIDmode
);
7287 clob
= gen_rtx_MEM (BLKmode
, clob
);
7288 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
7290 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
7293 /* Do not propagate memory accesses across this point. */
7296 expand_memory_blockage (void)
7298 if (targetm
.have_memory_blockage ())
7299 emit_insn (targetm
.gen_memory_blockage ());
7301 expand_asm_memory_blockage ();
7304 /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7305 same time clobbering the register set specified by REGS. */
7308 expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs
)
7310 rtx asm_op
, clob_mem
;
7312 unsigned int num_of_regs
= 0;
7313 for (unsigned int i
= 0; i
< FIRST_PSEUDO_REGISTER
; i
++)
7314 if (TEST_HARD_REG_BIT (regs
, i
))
7317 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
7318 rtvec_alloc (0), rtvec_alloc (0),
7319 rtvec_alloc (0), UNKNOWN_LOCATION
);
7320 MEM_VOLATILE_P (asm_op
) = 1;
7322 rtvec v
= rtvec_alloc (num_of_regs
+ 2);
7324 clob_mem
= gen_rtx_SCRATCH (VOIDmode
);
7325 clob_mem
= gen_rtx_MEM (BLKmode
, clob_mem
);
7326 clob_mem
= gen_rtx_CLOBBER (VOIDmode
, clob_mem
);
7328 RTVEC_ELT (v
, 0) = asm_op
;
7329 RTVEC_ELT (v
, 1) = clob_mem
;
7331 if (num_of_regs
> 0)
7334 for (unsigned int i
= 0; i
< FIRST_PSEUDO_REGISTER
; i
++)
7335 if (TEST_HARD_REG_BIT (regs
, i
))
7337 RTVEC_ELT (v
, j
) = gen_rtx_CLOBBER (VOIDmode
, regno_reg_rtx
[i
]);
7340 gcc_assert (j
== (num_of_regs
+ 2));
7343 emit_insn (gen_rtx_PARALLEL (VOIDmode
, v
));
7346 /* This routine will either emit the mem_thread_fence pattern or issue a
7347 sync_synchronize to generate a fence for memory model MEMMODEL. */
7350 expand_mem_thread_fence (enum memmodel model
)
7352 if (is_mm_relaxed (model
))
7354 if (targetm
.have_mem_thread_fence ())
7356 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
7357 expand_memory_blockage ();
7359 else if (targetm
.have_memory_barrier ())
7360 emit_insn (targetm
.gen_memory_barrier ());
7361 else if (synchronize_libfunc
!= NULL_RTX
)
7362 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
);
7364 expand_memory_blockage ();
7367 /* Emit a signal fence with given memory model. */
7370 expand_mem_signal_fence (enum memmodel model
)
7372 /* No machine barrier is required to implement a signal fence, but
7373 a compiler memory barrier must be issued, except for relaxed MM. */
7374 if (!is_mm_relaxed (model
))
7375 expand_memory_blockage ();
7378 /* This function expands the atomic load operation:
7379 return the atomically loaded value in MEM.
7381 MEMMODEL is the memory model variant to use.
7382 TARGET is an option place to stick the return value. */
7385 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
7387 machine_mode mode
= GET_MODE (mem
);
7388 enum insn_code icode
;
7390 /* If the target supports the load directly, great. */
7391 icode
= direct_optab_handler (atomic_load_optab
, mode
);
7392 if (icode
!= CODE_FOR_nothing
)
7394 class expand_operand ops
[3];
7395 rtx_insn
*last
= get_last_insn ();
7396 if (is_mm_seq_cst (model
))
7397 expand_memory_blockage ();
7399 create_output_operand (&ops
[0], target
, mode
);
7400 create_fixed_operand (&ops
[1], mem
);
7401 create_integer_operand (&ops
[2], model
);
7402 if (maybe_expand_insn (icode
, 3, ops
))
7404 if (!is_mm_relaxed (model
))
7405 expand_memory_blockage ();
7406 return ops
[0].value
;
7408 delete_insns_since (last
);
7411 /* If the size of the object is greater than word size on this target,
7412 then we assume that a load will not be atomic. We could try to
7413 emulate a load with a compare-and-swap operation, but the store that
7414 doing this could result in would be incorrect if this is a volatile
7415 atomic load or targetting read-only-mapped memory. */
7416 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
7417 /* If there is no atomic load, leave the library call. */
7420 /* Otherwise assume loads are atomic, and emit the proper barriers. */
7421 if (!target
|| target
== const0_rtx
)
7422 target
= gen_reg_rtx (mode
);
7424 /* For SEQ_CST, emit a barrier before the load. */
7425 if (is_mm_seq_cst (model
))
7426 expand_mem_thread_fence (model
);
7428 emit_move_insn (target
, mem
);
7430 /* Emit the appropriate barrier after the load. */
7431 expand_mem_thread_fence (model
);
7436 /* This function expands the atomic store operation:
7437 Atomically store VAL in MEM.
7438 MEMMODEL is the memory model variant to use.
7439 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7440 function returns const0_rtx if a pattern was emitted. */
7443 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
7445 machine_mode mode
= GET_MODE (mem
);
7446 enum insn_code icode
;
7447 class expand_operand ops
[3];
7449 /* If the target supports the store directly, great. */
7450 icode
= direct_optab_handler (atomic_store_optab
, mode
);
7451 if (icode
!= CODE_FOR_nothing
)
7453 rtx_insn
*last
= get_last_insn ();
7454 if (!is_mm_relaxed (model
))
7455 expand_memory_blockage ();
7456 create_fixed_operand (&ops
[0], mem
);
7457 create_input_operand (&ops
[1], val
, mode
);
7458 create_integer_operand (&ops
[2], model
);
7459 if (maybe_expand_insn (icode
, 3, ops
))
7461 if (is_mm_seq_cst (model
))
7462 expand_memory_blockage ();
7465 delete_insns_since (last
);
7468 /* If using __sync_lock_release is a viable alternative, try it.
7469 Note that this will not be set to true if we are expanding a generic
7470 __atomic_store_n. */
7473 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
7474 if (icode
!= CODE_FOR_nothing
)
7476 create_fixed_operand (&ops
[0], mem
);
7477 create_input_operand (&ops
[1], const0_rtx
, mode
);
7478 if (maybe_expand_insn (icode
, 2, ops
))
7480 /* lock_release is only a release barrier. */
7481 if (is_mm_seq_cst (model
))
7482 expand_mem_thread_fence (model
);
7488 /* If the size of the object is greater than word size on this target,
7489 a default store will not be atomic. */
7490 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
7492 /* If loads are atomic or we are called to provide a __sync builtin,
7493 we can try a atomic_exchange and throw away the result. Otherwise,
7494 don't do anything so that we do not create an inconsistency between
7495 loads and stores. */
7496 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
7498 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
7500 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
7508 /* Otherwise assume stores are atomic, and emit the proper barriers. */
7509 expand_mem_thread_fence (model
);
7511 emit_move_insn (mem
, val
);
7513 /* For SEQ_CST, also emit a barrier after the store. */
7514 if (is_mm_seq_cst (model
))
7515 expand_mem_thread_fence (model
);
7521 /* Structure containing the pointers and values required to process the
7522 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7524 struct atomic_op_functions
7526 direct_optab mem_fetch_before
;
7527 direct_optab mem_fetch_after
;
7528 direct_optab mem_no_result
;
7531 direct_optab no_result
;
7532 enum rtx_code reverse_code
;
7536 /* Fill in structure pointed to by OP with the various optab entries for an
7537 operation of type CODE. */
7540 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
7542 gcc_assert (op
!= NULL
);
7544 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7545 in the source code during compilation, and the optab entries are not
7546 computable until runtime. Fill in the values at runtime. */
7550 op
->mem_fetch_before
= atomic_fetch_add_optab
;
7551 op
->mem_fetch_after
= atomic_add_fetch_optab
;
7552 op
->mem_no_result
= atomic_add_optab
;
7553 op
->fetch_before
= sync_old_add_optab
;
7554 op
->fetch_after
= sync_new_add_optab
;
7555 op
->no_result
= sync_add_optab
;
7556 op
->reverse_code
= MINUS
;
7559 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
7560 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
7561 op
->mem_no_result
= atomic_sub_optab
;
7562 op
->fetch_before
= sync_old_sub_optab
;
7563 op
->fetch_after
= sync_new_sub_optab
;
7564 op
->no_result
= sync_sub_optab
;
7565 op
->reverse_code
= PLUS
;
7568 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
7569 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
7570 op
->mem_no_result
= atomic_xor_optab
;
7571 op
->fetch_before
= sync_old_xor_optab
;
7572 op
->fetch_after
= sync_new_xor_optab
;
7573 op
->no_result
= sync_xor_optab
;
7574 op
->reverse_code
= XOR
;
7577 op
->mem_fetch_before
= atomic_fetch_and_optab
;
7578 op
->mem_fetch_after
= atomic_and_fetch_optab
;
7579 op
->mem_no_result
= atomic_and_optab
;
7580 op
->fetch_before
= sync_old_and_optab
;
7581 op
->fetch_after
= sync_new_and_optab
;
7582 op
->no_result
= sync_and_optab
;
7583 op
->reverse_code
= UNKNOWN
;
7586 op
->mem_fetch_before
= atomic_fetch_or_optab
;
7587 op
->mem_fetch_after
= atomic_or_fetch_optab
;
7588 op
->mem_no_result
= atomic_or_optab
;
7589 op
->fetch_before
= sync_old_ior_optab
;
7590 op
->fetch_after
= sync_new_ior_optab
;
7591 op
->no_result
= sync_ior_optab
;
7592 op
->reverse_code
= UNKNOWN
;
7595 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
7596 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
7597 op
->mem_no_result
= atomic_nand_optab
;
7598 op
->fetch_before
= sync_old_nand_optab
;
7599 op
->fetch_after
= sync_new_nand_optab
;
7600 op
->no_result
= sync_nand_optab
;
7601 op
->reverse_code
= UNKNOWN
;
7608 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7609 using memory order MODEL. If AFTER is true the operation needs to return
7610 the value of *MEM after the operation, otherwise the previous value.
7611 TARGET is an optional place to place the result. The result is unused if
7613 Return the result if there is a better sequence, otherwise NULL_RTX. */
7616 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7617 enum memmodel model
, bool after
)
7619 /* If the value is prefetched, or not used, it may be possible to replace
7620 the sequence with a native exchange operation. */
7621 if (!after
|| target
== const0_rtx
)
7623 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7624 if (code
== AND
&& val
== const0_rtx
)
7626 if (target
== const0_rtx
)
7627 target
= gen_reg_rtx (GET_MODE (mem
));
7628 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7631 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7632 if (code
== IOR
&& val
== constm1_rtx
)
7634 if (target
== const0_rtx
)
7635 target
= gen_reg_rtx (GET_MODE (mem
));
7636 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7643 /* Try to emit an instruction for a specific operation varaition.
7644 OPTAB contains the OP functions.
7645 TARGET is an optional place to return the result. const0_rtx means unused.
7646 MEM is the memory location to operate on.
7647 VAL is the value to use in the operation.
7648 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7649 MODEL is the memory model, if used.
7650 AFTER is true if the returned result is the value after the operation. */
7653 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
7654 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
7656 machine_mode mode
= GET_MODE (mem
);
7657 class expand_operand ops
[4];
7658 enum insn_code icode
;
7662 /* Check to see if there is a result returned. */
7663 if (target
== const0_rtx
)
7667 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
7668 create_integer_operand (&ops
[2], model
);
7673 icode
= direct_optab_handler (optab
->no_result
, mode
);
7677 /* Otherwise, we need to generate a result. */
7682 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
7683 : optab
->mem_fetch_before
, mode
);
7684 create_integer_operand (&ops
[3], model
);
7689 icode
= optab_handler (after
? optab
->fetch_after
7690 : optab
->fetch_before
, mode
);
7693 create_output_operand (&ops
[op_counter
++], target
, mode
);
7695 if (icode
== CODE_FOR_nothing
)
7698 create_fixed_operand (&ops
[op_counter
++], mem
);
7699 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7700 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
7702 if (maybe_expand_insn (icode
, num_ops
, ops
))
7703 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
7709 /* This function expands an atomic fetch_OP or OP_fetch operation:
7710 TARGET is an option place to stick the return value. const0_rtx indicates
7711 the result is unused.
7712 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7713 CODE is the operation being performed (OP)
7714 MEMMODEL is the memory model variant to use.
7715 AFTER is true to return the result of the operation (OP_fetch).
7716 AFTER is false to return the value before the operation (fetch_OP).
7718 This function will *only* generate instructions if there is a direct
7719 optab. No compare and swap loops or libcalls will be generated. */
7722 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
7723 enum rtx_code code
, enum memmodel model
,
7726 machine_mode mode
= GET_MODE (mem
);
7727 struct atomic_op_functions optab
;
7729 bool unused_result
= (target
== const0_rtx
);
7731 get_atomic_op_for_code (&optab
, code
);
7733 /* Check to see if there are any better instructions. */
7734 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
7738 /* Check for the case where the result isn't used and try those patterns. */
7741 /* Try the memory model variant first. */
7742 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
7746 /* Next try the old style withuot a memory model. */
7747 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
7751 /* There is no no-result pattern, so try patterns with a result. */
7755 /* Try the __atomic version. */
7756 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
7760 /* Try the older __sync version. */
7761 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
7765 /* If the fetch value can be calculated from the other variation of fetch,
7766 try that operation. */
7767 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
7769 /* Try the __atomic version, then the older __sync version. */
7770 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
7772 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
7776 /* If the result isn't used, no need to do compensation code. */
7780 /* Issue compensation code. Fetch_after == fetch_before OP val.
7781 Fetch_before == after REVERSE_OP val. */
7783 code
= optab
.reverse_code
;
7786 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
7787 true, OPTAB_LIB_WIDEN
);
7788 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
7791 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
7792 true, OPTAB_LIB_WIDEN
);
7797 /* No direct opcode can be generated. */
7803 /* This function expands an atomic fetch_OP or OP_fetch operation:
7804 TARGET is an option place to stick the return value. const0_rtx indicates
7805 the result is unused.
7806 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7807 CODE is the operation being performed (OP)
7808 MEMMODEL is the memory model variant to use.
7809 AFTER is true to return the result of the operation (OP_fetch).
7810 AFTER is false to return the value before the operation (fetch_OP). */
7812 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7813 enum memmodel model
, bool after
)
7815 machine_mode mode
= GET_MODE (mem
);
7817 bool unused_result
= (target
== const0_rtx
);
7819 /* If loads are not atomic for the required size and we are not called to
7820 provide a __sync builtin, do not do anything so that we stay consistent
7821 with atomic loads of the same size. */
7822 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
7825 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
7831 /* Add/sub can be implemented by doing the reverse operation with -(val). */
7832 if (code
== PLUS
|| code
== MINUS
)
7835 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
7838 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
7839 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
7843 /* PLUS worked so emit the insns and return. */
7850 /* PLUS did not work, so throw away the negation code and continue. */
7854 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7855 if (!can_compare_and_swap_p (mode
, false))
7859 enum rtx_code orig_code
= code
;
7860 struct atomic_op_functions optab
;
7862 get_atomic_op_for_code (&optab
, code
);
7863 libfunc
= optab_libfunc (after
? optab
.fetch_after
7864 : optab
.fetch_before
, mode
);
7866 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
7870 code
= optab
.reverse_code
;
7871 libfunc
= optab_libfunc (after
? optab
.fetch_before
7872 : optab
.fetch_after
, mode
);
7874 if (libfunc
!= NULL
)
7876 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
7877 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
7878 addr
, ptr_mode
, val
, mode
);
7880 if (!unused_result
&& fixup
)
7881 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
7882 true, OPTAB_LIB_WIDEN
);
7886 /* We need the original code for any further attempts. */
7890 /* If nothing else has succeeded, default to a compare and swap loop. */
7891 if (can_compare_and_swap_p (mode
, true))
7894 rtx t0
= gen_reg_rtx (mode
), t1
;
7898 /* If the result is used, get a register for it. */
7901 if (!target
|| !register_operand (target
, mode
))
7902 target
= gen_reg_rtx (mode
);
7903 /* If fetch_before, copy the value now. */
7905 emit_move_insn (target
, t0
);
7908 target
= const0_rtx
;
7913 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
7914 true, OPTAB_LIB_WIDEN
);
7915 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
7918 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
7921 /* For after, copy the value now. */
7922 if (!unused_result
&& after
)
7923 emit_move_insn (target
, t1
);
7924 insn
= get_insns ();
7927 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
7934 /* Return true if OPERAND is suitable for operand number OPNO of
7935 instruction ICODE. */
7938 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
7940 return (!insn_data
[(int) icode
].operand
[opno
].predicate
7941 || (insn_data
[(int) icode
].operand
[opno
].predicate
7942 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
7945 /* TARGET is a target of a multiword operation that we are going to
7946 implement as a series of word-mode operations. Return true if
7947 TARGET is suitable for this purpose. */
7950 valid_multiword_target_p (rtx target
)
7955 mode
= GET_MODE (target
);
7956 if (!GET_MODE_SIZE (mode
).is_constant (&size
))
7958 for (i
= 0; i
< size
; i
+= UNITS_PER_WORD
)
7959 if (!validate_subreg (word_mode
, mode
, target
, i
))
7964 /* Make OP describe an input operand that has value INTVAL and that has
7965 no inherent mode. This function should only be used for operands that
7966 are always expand-time constants. The backend may request that INTVAL
7967 be copied into a different kind of rtx, but it must specify the mode
7968 of that rtx if so. */
7971 create_integer_operand (class expand_operand
*op
, poly_int64 intval
)
7973 create_expand_operand (op
, EXPAND_INTEGER
,
7974 gen_int_mode (intval
, MAX_MODE_INT
),
7975 VOIDmode
, false, intval
);
7978 /* Like maybe_legitimize_operand, but do not change the code of the
7979 current rtx value. */
7982 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
7983 class expand_operand
*op
)
7985 /* See if the operand matches in its current form. */
7986 if (insn_operand_matches (icode
, opno
, op
->value
))
7989 /* If the operand is a memory whose address has no side effects,
7990 try forcing the address into a non-virtual pseudo register.
7991 The check for side effects is important because copy_to_mode_reg
7992 cannot handle things like auto-modified addresses. */
7993 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
7998 addr
= XEXP (mem
, 0);
7999 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
8000 && !side_effects_p (addr
))
8005 last
= get_last_insn ();
8006 mode
= get_address_mode (mem
);
8007 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
8008 if (insn_operand_matches (icode
, opno
, mem
))
8013 delete_insns_since (last
);
8020 /* Try to make OP match operand OPNO of instruction ICODE. Return true
8021 on success, storing the new operand value back in OP. */
8024 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
8025 class expand_operand
*op
)
8027 machine_mode mode
, imode
, tmode
;
8034 temporary_volatile_ok
v (true);
8035 return maybe_legitimize_operand_same_code (icode
, opno
, op
);
8039 gcc_assert (mode
!= VOIDmode
);
8041 && op
->value
!= const0_rtx
8042 && GET_MODE (op
->value
) == mode
8043 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
8046 op
->value
= gen_reg_rtx (mode
);
8052 gcc_assert (mode
!= VOIDmode
);
8053 gcc_assert (GET_MODE (op
->value
) == VOIDmode
8054 || GET_MODE (op
->value
) == mode
);
8055 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
8058 op
->value
= copy_to_mode_reg (mode
, op
->value
);
8061 case EXPAND_CONVERT_TO
:
8062 gcc_assert (mode
!= VOIDmode
);
8063 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
8066 case EXPAND_CONVERT_FROM
:
8067 if (GET_MODE (op
->value
) != VOIDmode
)
8068 mode
= GET_MODE (op
->value
);
8070 /* The caller must tell us what mode this value has. */
8071 gcc_assert (mode
!= VOIDmode
);
8073 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
8074 tmode
= (VECTOR_MODE_P (imode
) && !VECTOR_MODE_P (mode
)
8075 ? GET_MODE_INNER (imode
) : imode
);
8076 if (tmode
!= VOIDmode
&& tmode
!= mode
)
8078 op
->value
= convert_modes (tmode
, mode
, op
->value
, op
->unsigned_p
);
8081 if (imode
!= VOIDmode
&& imode
!= mode
)
8083 gcc_assert (VECTOR_MODE_P (imode
) && !VECTOR_MODE_P (mode
));
8084 op
->value
= expand_vector_broadcast (imode
, op
->value
);
8089 case EXPAND_ADDRESS
:
8090 op
->value
= convert_memory_address (as_a
<scalar_int_mode
> (mode
),
8094 case EXPAND_INTEGER
:
8095 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
8096 if (mode
!= VOIDmode
8097 && known_eq (trunc_int_for_mode (op
->int_value
, mode
),
8100 op
->value
= gen_int_mode (op
->int_value
, mode
);
8105 case EXPAND_UNDEFINED_INPUT
:
8106 /* See if the predicate accepts a SCRATCH rtx, which in this context
8107 indicates an undefined value. Use an uninitialized register if not. */
8108 if (!insn_operand_matches (icode
, opno
, op
->value
))
8110 op
->value
= gen_reg_rtx (op
->mode
);
8115 return insn_operand_matches (icode
, opno
, op
->value
);
8118 /* Make OP describe an input operand that should have the same value
8119 as VALUE, after any mode conversion that the target might request.
8120 TYPE is the type of VALUE. */
8123 create_convert_operand_from_type (class expand_operand
*op
,
8124 rtx value
, tree type
)
8126 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
8127 TYPE_UNSIGNED (type
));
8130 /* Return true if the requirements on operands OP1 and OP2 of instruction
8131 ICODE are similar enough for the result of legitimizing OP1 to be
8132 reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
8133 with OP1 and OP2 respectively. */
8136 can_reuse_operands_p (enum insn_code icode
,
8137 unsigned int opno1
, unsigned int opno2
,
8138 const class expand_operand
*op1
,
8139 const class expand_operand
*op2
)
8141 /* Check requirements that are common to all types. */
8142 if (op1
->type
!= op2
->type
8143 || op1
->mode
!= op2
->mode
8144 || (insn_data
[(int) icode
].operand
[opno1
].mode
8145 != insn_data
[(int) icode
].operand
[opno2
].mode
))
8148 /* Check the requirements for specific types. */
8152 case EXPAND_UNDEFINED_INPUT
:
8153 /* Outputs and undefined intputs must remain distinct. */
8158 case EXPAND_ADDRESS
:
8159 case EXPAND_INTEGER
:
8162 case EXPAND_CONVERT_TO
:
8163 case EXPAND_CONVERT_FROM
:
8164 return op1
->unsigned_p
== op2
->unsigned_p
;
8169 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8170 of instruction ICODE. Return true on success, leaving the new operand
8171 values in the OPS themselves. Emit no code on failure. */
8174 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
8175 unsigned int nops
, class expand_operand
*ops
)
8177 rtx_insn
*last
= get_last_insn ();
8178 rtx
*orig_values
= XALLOCAVEC (rtx
, nops
);
8179 for (unsigned int i
= 0; i
< nops
; i
++)
8181 orig_values
[i
] = ops
[i
].value
;
8183 /* First try reusing the result of an earlier legitimization.
8184 This avoids duplicate rtl and ensures that tied operands
8187 This search is linear, but NOPS is bounded at compile time
8188 to a small number (current a single digit). */
8191 if (can_reuse_operands_p (icode
, opno
+ j
, opno
+ i
, &ops
[j
], &ops
[i
])
8192 && rtx_equal_p (orig_values
[j
], orig_values
[i
])
8194 && insn_operand_matches (icode
, opno
+ i
, ops
[j
].value
))
8196 ops
[i
].value
= copy_rtx (ops
[j
].value
);
8200 /* Otherwise try legitimizing the operand on its own. */
8201 if (j
== i
&& !maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
8203 delete_insns_since (last
);
8210 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8211 as its operands. Return the instruction pattern on success,
8212 and emit any necessary set-up code. Return null and emit no
8216 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
8217 class expand_operand
*ops
)
8219 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
8220 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
8226 return GEN_FCN (icode
) ();
8228 return GEN_FCN (icode
) (ops
[0].value
);
8230 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
8232 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
8234 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8237 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8238 ops
[3].value
, ops
[4].value
);
8240 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8241 ops
[3].value
, ops
[4].value
, ops
[5].value
);
8243 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8244 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8247 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8248 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8249 ops
[6].value
, ops
[7].value
);
8251 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8252 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8253 ops
[6].value
, ops
[7].value
, ops
[8].value
);
8255 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8256 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8257 ops
[6].value
, ops
[7].value
, ops
[8].value
,
8260 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8261 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8262 ops
[6].value
, ops
[7].value
, ops
[8].value
,
8263 ops
[9].value
, ops
[10].value
);
8268 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8269 as its operands. Return true on success and emit no code on failure. */
8272 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
8273 class expand_operand
*ops
)
8275 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
8284 /* Like maybe_expand_insn, but for jumps. */
8287 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
8288 class expand_operand
*ops
)
8290 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
8293 emit_jump_insn (pat
);
8299 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8303 expand_insn (enum insn_code icode
, unsigned int nops
,
8304 class expand_operand
*ops
)
8306 if (!maybe_expand_insn (icode
, nops
, ops
))
8310 /* Like expand_insn, but for jumps. */
8313 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
8314 class expand_operand
*ops
)
8316 if (!maybe_expand_jump_insn (icode
, nops
, ops
))