1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2023 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
45 #include "optabs-tree.h"
47 #include "internal-fn.h"
48 #include "langhooks.h"
52 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
54 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
55 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
57 static rtx
emit_conditional_move_1 (rtx
, rtx
, rtx
, rtx
, machine_mode
);
59 /* Debug facility for use in GDB. */
60 void debug_optab_libfuncs (void);
62 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
63 the result of operation CODE applied to OP0 (and OP1 if it is a binary
64 operation). OP0_MODE is OP0's mode.
66 If the last insn does not set TARGET, don't do anything, but return true.
68 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
69 or OP1, don't add the REG_EQUAL note but return false. Our caller can then
70 try again, ensuring that TARGET is not one of the operands. */
73 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
,
74 rtx op1
, machine_mode op0_mode
)
80 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
82 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
83 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
84 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
85 && GET_RTX_CLASS (code
) != RTX_COMPARE
86 && GET_RTX_CLASS (code
) != RTX_UNARY
)
89 if (GET_CODE (target
) == ZERO_EXTRACT
)
92 for (last_insn
= insns
;
93 NEXT_INSN (last_insn
) != NULL_RTX
;
94 last_insn
= NEXT_INSN (last_insn
))
97 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
98 a value changing in the insn, so the note would be invalid for CSE. */
99 if (reg_overlap_mentioned_p (target
, op0
)
100 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
103 && (rtx_equal_p (target
, op0
)
104 || (op1
&& rtx_equal_p (target
, op1
))))
106 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
107 over expanding it as temp = MEM op X, MEM = temp. If the target
108 supports MEM = MEM op X instructions, it is sometimes too hard
109 to reconstruct that form later, especially if X is also a memory,
110 and due to multiple occurrences of addresses the address might
111 be forced into register unnecessarily.
112 Note that not emitting the REG_EQUIV note might inhibit
113 CSE in some cases. */
114 set
= single_set (last_insn
);
116 && GET_CODE (SET_SRC (set
)) == code
117 && MEM_P (SET_DEST (set
))
118 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
119 || (op1
&& rtx_equal_p (SET_DEST (set
),
120 XEXP (SET_SRC (set
), 1)))))
126 set
= set_for_reg_notes (last_insn
);
130 if (! rtx_equal_p (SET_DEST (set
), target
)
131 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
132 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
133 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
136 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
146 if (op0_mode
!= VOIDmode
&& GET_MODE (target
) != op0_mode
)
148 note
= gen_rtx_fmt_e (code
, op0_mode
, copy_rtx (op0
));
149 if (GET_MODE_UNIT_SIZE (op0_mode
)
150 > GET_MODE_UNIT_SIZE (GET_MODE (target
)))
151 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
154 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
160 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
164 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
166 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
171 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
172 for a widening operation would be. In most cases this would be OP0, but if
173 that's a constant it'll be VOIDmode, which isn't useful. */
176 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
178 machine_mode m0
= GET_MODE (op0
);
179 machine_mode m1
= GET_MODE (op1
);
182 if (m0
== VOIDmode
&& m1
== VOIDmode
)
184 else if (m0
== VOIDmode
|| GET_MODE_UNIT_SIZE (m0
) < GET_MODE_UNIT_SIZE (m1
))
189 if (GET_MODE_UNIT_SIZE (result
) > GET_MODE_UNIT_SIZE (to_mode
))
195 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
196 says whether OP is signed or unsigned. NO_EXTEND is true if we need
197 not actually do a sign-extend or zero-extend, but can leave the
198 higher-order bits of the result rtx undefined, for example, in the case
199 of logical operations, but not right shifts. */
202 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
203 int unsignedp
, bool no_extend
)
206 scalar_int_mode int_mode
;
208 /* If we don't have to extend and this is a constant, return it. */
209 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
212 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
213 extend since it will be more efficient to do so unless the signedness of
214 a promoted object differs from our extension. */
216 || !is_a
<scalar_int_mode
> (mode
, &int_mode
)
217 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
218 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
219 return convert_modes (mode
, oldmode
, op
, unsignedp
);
221 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
223 if (GET_MODE_SIZE (int_mode
) <= UNITS_PER_WORD
)
224 return gen_lowpart (int_mode
, force_reg (GET_MODE (op
), op
));
226 /* Otherwise, get an object of MODE, clobber it, and set the low-order
229 result
= gen_reg_rtx (int_mode
);
230 emit_clobber (result
);
231 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
235 /* Expand vector widening operations.
237 There are two different classes of operations handled here:
238 1) Operations whose result is wider than all the arguments to the operation.
239 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
240 In this case OP0 and optionally OP1 would be initialized,
241 but WIDE_OP wouldn't (not relevant for this case).
242 2) Operations whose result is of the same size as the last argument to the
243 operation, but wider than all the other arguments to the operation.
244 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
245 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
247 E.g, when called to expand the following operations, this is how
248 the arguments will be initialized:
250 widening-sum 2 oprnd0 - oprnd1
251 widening-dot-product 3 oprnd0 oprnd1 oprnd2
252 widening-mult 2 oprnd0 oprnd1 -
253 type-promotion (vec-unpack) 1 oprnd0 - - */
256 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
257 rtx target
, int unsignedp
)
259 class expand_operand eops
[4];
260 tree oprnd0
, oprnd1
, oprnd2
;
261 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
262 optab widen_pattern_optab
;
263 enum insn_code icode
;
264 int nops
= TREE_CODE_LENGTH (ops
->code
);
269 oprnd1
= nops
>= 2 ? ops
->op1
: NULL_TREE
;
270 oprnd2
= nops
>= 3 ? ops
->op2
: NULL_TREE
;
272 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
273 if (ops
->code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
274 || ops
->code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
)
275 /* The sign is from the result type rather than operand's type
278 = optab_for_tree_code (ops
->code
, ops
->type
, optab_default
);
279 else if ((ops
->code
== VEC_UNPACK_HI_EXPR
280 || ops
->code
== VEC_UNPACK_LO_EXPR
)
281 && VECTOR_BOOLEAN_TYPE_P (ops
->type
)
282 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0
))
283 && TYPE_MODE (ops
->type
) == TYPE_MODE (TREE_TYPE (oprnd0
))
284 && SCALAR_INT_MODE_P (TYPE_MODE (ops
->type
)))
286 /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
287 the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
288 vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
289 the pattern number of elements in the wider vector. */
291 = (ops
->code
== VEC_UNPACK_HI_EXPR
292 ? vec_unpacks_sbool_hi_optab
: vec_unpacks_sbool_lo_optab
);
295 else if (ops
->code
== DOT_PROD_EXPR
)
297 enum optab_subtype subtype
= optab_default
;
298 signop sign1
= TYPE_SIGN (TREE_TYPE (oprnd0
));
299 signop sign2
= TYPE_SIGN (TREE_TYPE (oprnd1
));
302 else if (sign1
== SIGNED
&& sign2
== UNSIGNED
)
304 subtype
= optab_vector_mixed_sign
;
305 /* Same as optab_vector_mixed_sign but flip the operands. */
306 std::swap (op0
, op1
);
308 else if (sign1
== UNSIGNED
&& sign2
== SIGNED
)
309 subtype
= optab_vector_mixed_sign
;
314 = optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), subtype
);
318 = optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
319 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
320 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
321 icode
= find_widening_optab_handler (widen_pattern_optab
,
322 TYPE_MODE (TREE_TYPE (ops
->op2
)),
325 icode
= optab_handler (widen_pattern_optab
, tmode0
);
326 gcc_assert (icode
!= CODE_FOR_nothing
);
329 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
333 op1
= GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0
)).to_constant ());
337 /* The last operand is of a wider mode than the rest of the operands. */
342 gcc_assert (tmode1
== tmode0
);
344 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
348 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
349 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
351 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
353 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
354 expand_insn (icode
, op
, eops
);
355 return eops
[0].value
;
358 /* Generate code to perform an operation specified by TERNARY_OPTAB
359 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
361 UNSIGNEDP is for the case where we have to widen the operands
362 to perform the operation. It says to use zero-extension.
364 If TARGET is nonzero, the value
365 is generated there, if it is convenient to do so.
366 In all cases an rtx is returned for the locus of the value;
367 this may or may not be TARGET. */
370 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
371 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
373 class expand_operand ops
[4];
374 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
376 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
378 create_output_operand (&ops
[0], target
, mode
);
379 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
380 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
381 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
382 expand_insn (icode
, 4, ops
);
387 /* Like expand_binop, but return a constant rtx if the result can be
388 calculated at compile time. The arguments and return value are
389 otherwise the same as for expand_binop. */
392 simplify_expand_binop (machine_mode mode
, optab binoptab
,
393 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
394 enum optab_methods methods
)
396 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
398 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
404 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
407 /* Like simplify_expand_binop, but always put the result in TARGET.
408 Return true if the expansion succeeded. */
411 force_expand_binop (machine_mode mode
, optab binoptab
,
412 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
413 enum optab_methods methods
)
415 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
416 target
, unsignedp
, methods
);
420 emit_move_insn (target
, x
);
424 /* Create a new vector value in VMODE with all elements set to OP. The
425 mode of OP must be the element mode of VMODE. If OP is a constant,
426 then the return value will be a constant. */
429 expand_vector_broadcast (machine_mode vmode
, rtx op
)
434 gcc_checking_assert (VECTOR_MODE_P (vmode
));
436 if (valid_for_const_vector_p (vmode
, op
))
437 return gen_const_vec_duplicate (vmode
, op
);
439 insn_code icode
= optab_handler (vec_duplicate_optab
, vmode
);
440 if (icode
!= CODE_FOR_nothing
)
442 class expand_operand ops
[2];
443 create_output_operand (&ops
[0], NULL_RTX
, vmode
);
444 create_input_operand (&ops
[1], op
, GET_MODE (op
));
445 expand_insn (icode
, 2, ops
);
449 if (!GET_MODE_NUNITS (vmode
).is_constant (&n
))
452 /* ??? If the target doesn't have a vec_init, then we have no easy way
453 of performing this operation. Most of this sort of generic support
454 is hidden away in the vector lowering support in gimple. */
455 icode
= convert_optab_handler (vec_init_optab
, vmode
,
456 GET_MODE_INNER (vmode
));
457 if (icode
== CODE_FOR_nothing
)
460 vec
= rtvec_alloc (n
);
461 for (int i
= 0; i
< n
; ++i
)
462 RTVEC_ELT (vec
, i
) = op
;
463 rtx ret
= gen_reg_rtx (vmode
);
464 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
469 /* This subroutine of expand_doubleword_shift handles the cases in which
470 the effective shift value is >= BITS_PER_WORD. The arguments and return
471 value are the same as for the parent routine, except that SUPERWORD_OP1
472 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
473 INTO_TARGET may be null if the caller has decided to calculate it. */
476 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
477 rtx outof_target
, rtx into_target
,
478 int unsignedp
, enum optab_methods methods
)
480 if (into_target
!= 0)
481 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
482 into_target
, unsignedp
, methods
))
485 if (outof_target
!= 0)
487 /* For a signed right shift, we must fill OUTOF_TARGET with copies
488 of the sign bit, otherwise we must fill it with zeros. */
489 if (binoptab
!= ashr_optab
)
490 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
492 if (!force_expand_binop (word_mode
, binoptab
, outof_input
,
493 gen_int_shift_amount (word_mode
,
495 outof_target
, unsignedp
, methods
))
501 /* This subroutine of expand_doubleword_shift handles the cases in which
502 the effective shift value is < BITS_PER_WORD. The arguments and return
503 value are the same as for the parent routine. */
506 expand_subword_shift (scalar_int_mode op1_mode
, optab binoptab
,
507 rtx outof_input
, rtx into_input
, rtx op1
,
508 rtx outof_target
, rtx into_target
,
509 int unsignedp
, enum optab_methods methods
,
510 unsigned HOST_WIDE_INT shift_mask
)
512 optab reverse_unsigned_shift
, unsigned_shift
;
515 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
516 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
518 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
519 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
520 the opposite direction to BINOPTAB. */
521 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
523 carries
= outof_input
;
524 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
525 op1_mode
), op1_mode
);
526 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
531 /* We must avoid shifting by BITS_PER_WORD bits since that is either
532 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
533 has unknown behavior. Do a single shift first, then shift by the
534 remainder. It's OK to use ~OP1 as the remainder if shift counts
535 are truncated to the mode size. */
536 carries
= simplify_expand_binop (word_mode
, reverse_unsigned_shift
,
537 outof_input
, const1_rtx
, 0,
539 if (carries
== const0_rtx
)
541 else if (shift_mask
== BITS_PER_WORD
- 1)
542 tmp
= expand_unop (op1_mode
, one_cmpl_optab
, op1
, 0, true);
545 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
546 op1_mode
), op1_mode
);
547 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
551 if (tmp
== 0 || carries
== 0)
553 if (carries
!= const0_rtx
&& tmp
!= const0_rtx
)
554 carries
= simplify_expand_binop (word_mode
, reverse_unsigned_shift
,
555 carries
, tmp
, 0, unsignedp
, methods
);
559 if (into_input
!= const0_rtx
)
561 /* Shift INTO_INPUT logically by OP1. This is the last use of
562 INTO_INPUT so the result can go directly into INTO_TARGET if
564 tmp
= simplify_expand_binop (word_mode
, unsigned_shift
, into_input
,
565 op1
, into_target
, unsignedp
, methods
);
569 /* Now OR in the bits carried over from OUTOF_INPUT. */
570 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
571 into_target
, unsignedp
, methods
))
575 emit_move_insn (into_target
, carries
);
577 /* Use a standard word_mode shift for the out-of half. */
578 if (outof_target
!= 0)
579 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
580 outof_target
, unsignedp
, methods
))
587 /* Try implementing expand_doubleword_shift using conditional moves.
588 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
589 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
590 are the shift counts to use in the former and latter case. All other
591 arguments are the same as the parent routine. */
594 expand_doubleword_shift_condmove (scalar_int_mode op1_mode
, optab binoptab
,
595 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
596 rtx outof_input
, rtx into_input
,
597 rtx subword_op1
, rtx superword_op1
,
598 rtx outof_target
, rtx into_target
,
599 int unsignedp
, enum optab_methods methods
,
600 unsigned HOST_WIDE_INT shift_mask
)
602 rtx outof_superword
, into_superword
;
604 /* Put the superword version of the output into OUTOF_SUPERWORD and
606 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
607 if (outof_target
!= 0 && subword_op1
== superword_op1
)
609 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
610 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
611 into_superword
= outof_target
;
612 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
613 outof_superword
, 0, unsignedp
, methods
))
618 into_superword
= gen_reg_rtx (word_mode
);
619 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
620 outof_superword
, into_superword
,
625 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
626 if (!expand_subword_shift (op1_mode
, binoptab
,
627 outof_input
, into_input
, subword_op1
,
628 outof_target
, into_target
,
629 unsignedp
, methods
, shift_mask
))
632 /* Select between them. Do the INTO half first because INTO_SUPERWORD
633 might be the current value of OUTOF_TARGET. */
634 if (!emit_conditional_move (into_target
, { cmp_code
, cmp1
, cmp2
, op1_mode
},
635 into_target
, into_superword
, word_mode
, false))
638 if (outof_target
!= 0)
639 if (!emit_conditional_move (outof_target
,
640 { cmp_code
, cmp1
, cmp2
, op1_mode
},
641 outof_target
, outof_superword
,
648 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
649 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
650 input operand; the shift moves bits in the direction OUTOF_INPUT->
651 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
652 of the target. OP1 is the shift count and OP1_MODE is its mode.
653 If OP1 is constant, it will have been truncated as appropriate
654 and is known to be nonzero.
656 If SHIFT_MASK is zero, the result of word shifts is undefined when the
657 shift count is outside the range [0, BITS_PER_WORD). This routine must
658 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
660 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
661 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
662 fill with zeros or sign bits as appropriate.
664 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
665 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
666 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
667 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
670 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
671 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
672 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
673 function wants to calculate it itself.
675 Return true if the shift could be successfully synthesized. */
678 expand_doubleword_shift (scalar_int_mode op1_mode
, optab binoptab
,
679 rtx outof_input
, rtx into_input
, rtx op1
,
680 rtx outof_target
, rtx into_target
,
681 int unsignedp
, enum optab_methods methods
,
682 unsigned HOST_WIDE_INT shift_mask
)
684 rtx superword_op1
, tmp
, cmp1
, cmp2
;
685 enum rtx_code cmp_code
;
687 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
688 fill the result with sign or zero bits as appropriate. If so, the value
689 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
690 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
691 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
693 This isn't worthwhile for constant shifts since the optimizers will
694 cope better with in-range shift counts. */
695 if (shift_mask
>= BITS_PER_WORD
697 && !CONSTANT_P (op1
))
699 if (!expand_doubleword_shift (op1_mode
, binoptab
,
700 outof_input
, into_input
, op1
,
702 unsignedp
, methods
, shift_mask
))
704 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
705 outof_target
, unsignedp
, methods
))
710 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
711 is true when the effective shift value is less than BITS_PER_WORD.
712 Set SUPERWORD_OP1 to the shift count that should be used to shift
713 OUTOF_INPUT into INTO_TARGET when the condition is false. */
714 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
715 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
717 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
718 is a subword shift count. */
719 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
721 cmp2
= CONST0_RTX (op1_mode
);
727 /* Set CMP1 to OP1 - BITS_PER_WORD. */
728 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
730 cmp2
= CONST0_RTX (op1_mode
);
732 superword_op1
= cmp1
;
737 /* If we can compute the condition at compile time, pick the
738 appropriate subroutine. */
739 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
740 if (tmp
!= 0 && CONST_INT_P (tmp
))
742 if (tmp
== const0_rtx
)
743 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
744 outof_target
, into_target
,
747 return expand_subword_shift (op1_mode
, binoptab
,
748 outof_input
, into_input
, op1
,
749 outof_target
, into_target
,
750 unsignedp
, methods
, shift_mask
);
753 /* Try using conditional moves to generate straight-line code. */
754 if (HAVE_conditional_move
)
756 rtx_insn
*start
= get_last_insn ();
757 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
758 cmp_code
, cmp1
, cmp2
,
759 outof_input
, into_input
,
761 outof_target
, into_target
,
762 unsignedp
, methods
, shift_mask
))
764 delete_insns_since (start
);
767 /* As a last resort, use branches to select the correct alternative. */
768 rtx_code_label
*subword_label
= gen_label_rtx ();
769 rtx_code_label
*done_label
= gen_label_rtx ();
772 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
774 profile_probability::uninitialized ());
777 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
778 outof_target
, into_target
,
782 emit_jump_insn (targetm
.gen_jump (done_label
));
784 emit_label (subword_label
);
786 if (!expand_subword_shift (op1_mode
, binoptab
,
787 outof_input
, into_input
, op1
,
788 outof_target
, into_target
,
789 unsignedp
, methods
, shift_mask
))
792 emit_label (done_label
);
796 /* Subroutine of expand_binop. Perform a double word multiplication of
797 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
798 as the target's word_mode. This function return NULL_RTX if anything
799 goes wrong, in which case it may have already emitted instructions
800 which need to be deleted.
802 If we want to multiply two two-word values and have normal and widening
803 multiplies of single-word values, we can do this with three smaller
806 The multiplication proceeds as follows:
807 _______________________
808 [__op0_high_|__op0_low__]
809 _______________________
810 * [__op1_high_|__op1_low__]
811 _______________________________________________
812 _______________________
813 (1) [__op0_low__*__op1_low__]
814 _______________________
815 (2a) [__op0_low__*__op1_high_]
816 _______________________
817 (2b) [__op0_high_*__op1_low__]
818 _______________________
819 (3) [__op0_high_*__op1_high_]
822 This gives a 4-word result. Since we are only interested in the
823 lower 2 words, partial result (3) and the upper words of (2a) and
824 (2b) don't need to be calculated. Hence (2a) and (2b) can be
825 calculated using non-widening multiplication.
827 (1), however, needs to be calculated with an unsigned widening
828 multiplication. If this operation is not directly supported we
829 try using a signed widening multiplication and adjust the result.
830 This adjustment works as follows:
832 If both operands are positive then no adjustment is needed.
834 If the operands have different signs, for example op0_low < 0 and
835 op1_low >= 0, the instruction treats the most significant bit of
836 op0_low as a sign bit instead of a bit with significance
837 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
838 with 2**BITS_PER_WORD - op0_low, and two's complements the
839 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
842 Similarly, if both operands are negative, we need to add
843 (op0_low + op1_low) * 2**BITS_PER_WORD.
845 We use a trick to adjust quickly. We logically shift op0_low right
846 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
847 op0_high (op1_high) before it is used to calculate 2b (2a). If no
848 logical shift exists, we do an arithmetic right shift and subtract
852 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
853 bool umulp
, enum optab_methods methods
)
855 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
856 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
857 rtx wordm1
= (umulp
? NULL_RTX
858 : gen_int_shift_amount (word_mode
, BITS_PER_WORD
- 1));
859 rtx product
, adjust
, product_high
, temp
;
861 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
862 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
863 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
864 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
866 /* If we're using an unsigned multiply to directly compute the product
867 of the low-order words of the operands and perform any required
868 adjustments of the operands, we begin by trying two more multiplications
869 and then computing the appropriate sum.
871 We have checked above that the required addition is provided.
872 Full-word addition will normally always succeed, especially if
873 it is provided at all, so we don't worry about its failure. The
874 multiplication may well fail, however, so we do handle that. */
878 /* ??? This could be done with emit_store_flag where available. */
879 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
880 NULL_RTX
, 1, methods
);
882 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
883 NULL_RTX
, 0, OPTAB_DIRECT
);
886 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
887 NULL_RTX
, 0, methods
);
890 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
891 NULL_RTX
, 0, OPTAB_DIRECT
);
898 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
899 NULL_RTX
, 0, OPTAB_DIRECT
);
903 /* OP0_HIGH should now be dead. */
907 /* ??? This could be done with emit_store_flag where available. */
908 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
909 NULL_RTX
, 1, methods
);
911 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
912 NULL_RTX
, 0, OPTAB_DIRECT
);
915 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
916 NULL_RTX
, 0, methods
);
919 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
920 NULL_RTX
, 0, OPTAB_DIRECT
);
927 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
928 NULL_RTX
, 0, OPTAB_DIRECT
);
932 /* OP1_HIGH should now be dead. */
934 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
935 NULL_RTX
, 0, OPTAB_DIRECT
);
937 if (target
&& !REG_P (target
))
940 /* *_widen_optab needs to determine operand mode, make sure at least
941 one operand has non-VOID mode. */
942 if (GET_MODE (op0_low
) == VOIDmode
&& GET_MODE (op1_low
) == VOIDmode
)
943 op0_low
= force_reg (word_mode
, op0_low
);
946 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
947 target
, 1, OPTAB_DIRECT
);
949 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
950 target
, 1, OPTAB_DIRECT
);
955 product_high
= operand_subword (product
, high
, 1, mode
);
956 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
957 NULL_RTX
, 0, OPTAB_DIRECT
);
958 emit_move_insn (product_high
, adjust
);
962 /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
963 constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
964 (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
965 computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
966 + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
967 depends on the bit value, if 2, then carry from the addition needs to be
968 added too, i.e. like:
969 sum += __builtin_add_overflow (low, high, &sum)
971 Optimize signed double-word OP0 % OP1 similarly, just apply some correction
972 factor to the sum before doing unsigned remainder, in the form of
973 sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
974 then perform unsigned
975 remainder = sum % OP1;
977 remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
980 expand_doubleword_mod (machine_mode mode
, rtx op0
, rtx op1
, bool unsignedp
)
982 if (INTVAL (op1
) <= 1 || (INTVAL (op1
) & 1) == 0)
985 rtx_insn
*last
= get_last_insn ();
986 for (int bit
= BITS_PER_WORD
; bit
>= BITS_PER_WORD
/ 2; bit
--)
988 wide_int w
= wi::shifted_mask (bit
, 1, false, 2 * BITS_PER_WORD
);
989 if (wi::ne_p (wi::umod_trunc (w
, INTVAL (op1
)), 1))
991 rtx sum
= NULL_RTX
, mask
= NULL_RTX
;
992 if (bit
== BITS_PER_WORD
)
994 /* For signed modulo we need to add correction to the sum
995 and that might again overflow. */
998 if (optab_handler (uaddv4_optab
, word_mode
) == CODE_FOR_nothing
)
1000 tree wtype
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
1001 if (wtype
== NULL_TREE
)
1003 tree ctype
= build_complex_type (wtype
);
1004 if (TYPE_MODE (ctype
) != GET_MODE_COMPLEX_MODE (word_mode
))
1006 machine_mode cmode
= TYPE_MODE (ctype
);
1007 rtx op00
= operand_subword_force (op0
, 0, mode
);
1008 rtx op01
= operand_subword_force (op0
, 1, mode
);
1009 rtx cres
= gen_rtx_CONCAT (cmode
, gen_reg_rtx (word_mode
),
1010 gen_reg_rtx (word_mode
));
1011 tree lhs
= make_tree (ctype
, cres
);
1012 tree arg0
= make_tree (wtype
, op00
);
1013 tree arg1
= make_tree (wtype
, op01
);
1014 expand_addsub_overflow (UNKNOWN_LOCATION
, PLUS_EXPR
, lhs
, arg0
,
1015 arg1
, true, true, true, false, NULL
);
1016 sum
= expand_simple_binop (word_mode
, PLUS
, XEXP (cres
, 0),
1017 XEXP (cres
, 1), NULL_RTX
, 1,
1019 if (sum
== NULL_RTX
)
1024 /* Code below uses GEN_INT, so we need the masks to be representable
1025 in HOST_WIDE_INTs. */
1026 if (bit
>= HOST_BITS_PER_WIDE_INT
)
1028 /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1029 overflow. Consider 64-bit -1ULL for word size 32, if we add
1030 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1031 if (bit
== BITS_PER_WORD
- 1)
1034 int count
= (2 * BITS_PER_WORD
+ bit
- 1) / bit
;
1035 rtx sum_corr
= NULL_RTX
;
1039 /* For signed modulo, compute it as unsigned modulo of
1040 sum with a correction added to it if OP0 is negative,
1041 such that the result can be computed as unsigned
1042 remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1043 w
= wi::min_value (2 * BITS_PER_WORD
, SIGNED
);
1044 wide_int wmod1
= wi::umod_trunc (w
, INTVAL (op1
));
1045 wide_int wmod2
= wi::smod_trunc (w
, INTVAL (op1
));
1046 /* wmod2 == -wmod1. */
1047 wmod2
= wmod2
+ (INTVAL (op1
) - 1);
1048 if (wi::ne_p (wmod1
, wmod2
))
1050 wide_int wcorr
= wmod2
- wmod1
;
1052 wcorr
= wcorr
+ INTVAL (op1
);
1053 /* Now verify if the count sums can't overflow, and punt
1055 w
= wi::mask (bit
, false, 2 * BITS_PER_WORD
);
1056 w
= w
* (count
- 1);
1057 w
= w
+ wi::mask (2 * BITS_PER_WORD
- (count
- 1) * bit
,
1058 false, 2 * BITS_PER_WORD
);
1060 w
= wi::lrshift (w
, BITS_PER_WORD
);
1061 if (wi::ne_p (w
, 0))
1064 mask
= operand_subword_force (op0
, WORDS_BIG_ENDIAN
? 0 : 1,
1066 mask
= expand_simple_binop (word_mode
, ASHIFTRT
, mask
,
1067 GEN_INT (BITS_PER_WORD
- 1),
1068 NULL_RTX
, 0, OPTAB_DIRECT
);
1069 if (mask
== NULL_RTX
)
1071 sum_corr
= immed_wide_int_const (wcorr
, word_mode
);
1072 sum_corr
= expand_simple_binop (word_mode
, AND
, mask
,
1073 sum_corr
, NULL_RTX
, 1,
1075 if (sum_corr
== NULL_RTX
)
1080 for (int i
= 0; i
< count
; i
++)
1084 v
= expand_simple_binop (mode
, LSHIFTRT
, v
, GEN_INT (i
* bit
),
1085 NULL_RTX
, 1, OPTAB_DIRECT
);
1088 v
= lowpart_subreg (word_mode
, v
, mode
);
1092 v
= expand_simple_binop (word_mode
, AND
, v
,
1093 GEN_INT ((HOST_WIDE_INT_1U
<< bit
)
1098 if (sum
== NULL_RTX
)
1101 sum
= expand_simple_binop (word_mode
, PLUS
, sum
, v
, NULL_RTX
,
1103 if (sum
== NULL_RTX
)
1108 sum
= expand_simple_binop (word_mode
, PLUS
, sum
, sum_corr
,
1109 NULL_RTX
, 1, OPTAB_DIRECT
);
1110 if (sum
== NULL_RTX
)
1114 rtx remainder
= expand_divmod (1, TRUNC_MOD_EXPR
, word_mode
, sum
,
1115 gen_int_mode (INTVAL (op1
), word_mode
),
1116 NULL_RTX
, 1, OPTAB_DIRECT
);
1117 if (remainder
== NULL_RTX
)
1122 if (mask
== NULL_RTX
)
1124 mask
= operand_subword_force (op0
, WORDS_BIG_ENDIAN
? 0 : 1,
1126 mask
= expand_simple_binop (word_mode
, ASHIFTRT
, mask
,
1127 GEN_INT (BITS_PER_WORD
- 1),
1128 NULL_RTX
, 0, OPTAB_DIRECT
);
1129 if (mask
== NULL_RTX
)
1132 mask
= expand_simple_binop (word_mode
, AND
, mask
,
1133 gen_int_mode (1 - INTVAL (op1
),
1135 NULL_RTX
, 1, OPTAB_DIRECT
);
1136 if (mask
== NULL_RTX
)
1138 remainder
= expand_simple_binop (word_mode
, PLUS
, remainder
,
1139 mask
, NULL_RTX
, 1, OPTAB_DIRECT
);
1140 if (remainder
== NULL_RTX
)
1144 remainder
= convert_modes (mode
, word_mode
, remainder
, unsignedp
);
1145 /* Punt if we need any library calls. */
1147 last
= NEXT_INSN (last
);
1149 last
= get_insns ();
1150 for (; last
; last
= NEXT_INSN (last
))
1158 /* Similarly to the above function, but compute both quotient and remainder.
1159 Quotient can be computed from the remainder as:
1160 rem = op0 % op1; // Handled using expand_doubleword_mod
1161 quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1162 // 2 * BITS_PER_WORD
1164 We can also handle cases where op1 is a multiple of power of two constant
1165 and constant handled by expand_doubleword_mod.
1166 op11 = 1 << __builtin_ctz (op1);
1168 rem1 = op0 % op12; // Handled using expand_doubleword_mod
1169 quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1170 // 2 * BITS_PER_WORD
1171 rem = (quot1 % op11) * op12 + rem1;
1172 quot = quot1 / op11; */
1175 expand_doubleword_divmod (machine_mode mode
, rtx op0
, rtx op1
, rtx
*rem
,
1180 /* Negative dividend should have been optimized into positive,
1181 similarly modulo by 1 and modulo by power of two is optimized
1183 if (INTVAL (op1
) <= 1 || pow2p_hwi (INTVAL (op1
)))
1186 rtx op11
= const1_rtx
;
1188 if ((INTVAL (op1
) & 1) == 0)
1190 int bit
= ctz_hwi (INTVAL (op1
));
1191 op11
= GEN_INT (HOST_WIDE_INT_1
<< bit
);
1192 op12
= GEN_INT (INTVAL (op1
) >> bit
);
1195 rtx rem1
= expand_doubleword_mod (mode
, op0
, op12
, unsignedp
);
1196 if (rem1
== NULL_RTX
)
1199 int prec
= 2 * BITS_PER_WORD
;
1200 wide_int a
= wide_int::from (INTVAL (op12
), prec
+ 1, UNSIGNED
);
1201 wide_int b
= wi::shifted_mask (prec
, 1, false, prec
+ 1);
1202 wide_int m
= wide_int::from (wi::mod_inv (a
, b
), prec
, UNSIGNED
);
1203 rtx inv
= immed_wide_int_const (m
, mode
);
1205 rtx_insn
*last
= get_last_insn ();
1206 rtx quot1
= expand_simple_binop (mode
, MINUS
, op0
, rem1
,
1207 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1208 if (quot1
== NULL_RTX
)
1211 quot1
= expand_simple_binop (mode
, MULT
, quot1
, inv
,
1212 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1213 if (quot1
== NULL_RTX
)
1216 if (op11
!= const1_rtx
)
1218 rtx rem2
= expand_divmod (1, TRUNC_MOD_EXPR
, mode
, quot1
, op11
,
1219 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1220 if (rem2
== NULL_RTX
)
1223 rem2
= expand_simple_binop (mode
, MULT
, rem2
, op12
, NULL_RTX
,
1224 unsignedp
, OPTAB_DIRECT
);
1225 if (rem2
== NULL_RTX
)
1228 rem2
= expand_simple_binop (mode
, PLUS
, rem2
, rem1
, NULL_RTX
,
1229 unsignedp
, OPTAB_DIRECT
);
1230 if (rem2
== NULL_RTX
)
1233 rtx quot2
= expand_divmod (0, TRUNC_DIV_EXPR
, mode
, quot1
, op11
,
1234 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1235 if (quot2
== NULL_RTX
)
1242 /* Punt if we need any library calls. */
1244 last
= NEXT_INSN (last
);
1246 last
= get_insns ();
1247 for (; last
; last
= NEXT_INSN (last
))
1255 /* Wrapper around expand_binop which takes an rtx code to specify
1256 the operation to perform, not an optab pointer. All other
1257 arguments are the same. */
1259 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
1260 rtx op1
, rtx target
, int unsignedp
,
1261 enum optab_methods methods
)
1263 optab binop
= code_to_optab (code
);
1266 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
1269 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1270 binop. Order them according to commutative_operand_precedence and, if
1271 possible, try to put TARGET or a pseudo first. */
1273 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
1275 int op0_prec
= commutative_operand_precedence (op0
);
1276 int op1_prec
= commutative_operand_precedence (op1
);
1278 if (op0_prec
< op1_prec
)
1281 if (op0_prec
> op1_prec
)
1284 /* With equal precedence, both orders are ok, but it is better if the
1285 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1286 if (target
== 0 || REG_P (target
))
1287 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
1289 return rtx_equal_p (op1
, target
);
1292 /* Return true if BINOPTAB implements a shift operation. */
1295 shift_optab_p (optab binoptab
)
1297 switch (optab_to_code (binoptab
))
1313 /* Return true if BINOPTAB implements a commutative binary operation. */
1316 commutative_optab_p (optab binoptab
)
1318 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
1319 || binoptab
== smul_widen_optab
1320 || binoptab
== umul_widen_optab
1321 || binoptab
== smul_highpart_optab
1322 || binoptab
== umul_highpart_optab
1323 || binoptab
== vec_widen_sadd_optab
1324 || binoptab
== vec_widen_uadd_optab
1325 || binoptab
== vec_widen_sadd_hi_optab
1326 || binoptab
== vec_widen_sadd_lo_optab
1327 || binoptab
== vec_widen_uadd_hi_optab
1328 || binoptab
== vec_widen_uadd_lo_optab
1329 || binoptab
== vec_widen_sadd_even_optab
1330 || binoptab
== vec_widen_sadd_odd_optab
1331 || binoptab
== vec_widen_uadd_even_optab
1332 || binoptab
== vec_widen_uadd_odd_optab
);
1335 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1336 optimizing, and if the operand is a constant that costs more than
1337 1 instruction, force the constant into a register and return that
1338 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1341 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
1342 int opn
, rtx x
, bool unsignedp
)
1344 bool speed
= optimize_insn_for_speed_p ();
1346 if (mode
!= VOIDmode
1349 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
1350 > set_src_cost (x
, mode
, speed
)))
1352 if (CONST_INT_P (x
))
1354 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
1355 if (intval
!= INTVAL (x
))
1356 x
= GEN_INT (intval
);
1359 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
1360 x
= force_reg (mode
, x
);
1365 /* Helper function for expand_binop: handle the case where there
1366 is an insn ICODE that directly implements the indicated operation.
1367 Returns null if this is not possible. */
1369 expand_binop_directly (enum insn_code icode
, machine_mode mode
, optab binoptab
,
1371 rtx target
, int unsignedp
, enum optab_methods methods
,
1374 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
1375 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
1376 machine_mode mode0
, mode1
, tmp_mode
;
1377 class expand_operand ops
[3];
1380 rtx xop0
= op0
, xop1
= op1
;
1381 bool canonicalize_op1
= false;
1383 /* If it is a commutative operator and the modes would match
1384 if we would swap the operands, we can save the conversions. */
1385 commutative_p
= commutative_optab_p (binoptab
);
1387 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1388 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode0
)
1389 std::swap (xop0
, xop1
);
1391 /* If we are optimizing, force expensive constants into a register. */
1392 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1393 if (!shift_optab_p (binoptab
))
1394 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1396 /* Shifts and rotates often use a different mode for op1 from op0;
1397 for VOIDmode constants we don't know the mode, so force it
1398 to be canonicalized using convert_modes. */
1399 canonicalize_op1
= true;
1401 /* In case the insn wants input operands in modes different from
1402 those of the actual operands, convert the operands. It would
1403 seem that we don't need to convert CONST_INTs, but we do, so
1404 that they're properly zero-extended, sign-extended or truncated
1407 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1408 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1410 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1414 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1415 ? GET_MODE (xop1
) : mode
);
1416 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1418 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1422 /* If operation is commutative,
1423 try to make the first operand a register.
1424 Even better, try to make it the same as the target.
1425 Also try to make the last operand a constant. */
1427 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1428 std::swap (xop0
, xop1
);
1430 /* Now, if insn's predicates don't allow our operands, put them into
1433 if (binoptab
== vec_pack_trunc_optab
1434 || binoptab
== vec_pack_usat_optab
1435 || binoptab
== vec_pack_ssat_optab
1436 || binoptab
== vec_pack_ufix_trunc_optab
1437 || binoptab
== vec_pack_sfix_trunc_optab
1438 || binoptab
== vec_packu_float_optab
1439 || binoptab
== vec_packs_float_optab
)
1441 /* The mode of the result is different then the mode of the
1443 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1444 if (VECTOR_MODE_P (mode
)
1445 && maybe_ne (GET_MODE_NUNITS (tmp_mode
), 2 * GET_MODE_NUNITS (mode
)))
1447 delete_insns_since (last
);
1454 create_output_operand (&ops
[0], target
, tmp_mode
);
1455 create_input_operand (&ops
[1], xop0
, mode0
);
1456 create_input_operand (&ops
[2], xop1
, mode1
);
1457 pat
= maybe_gen_insn (icode
, 3, ops
);
1460 /* If PAT is composed of more than one insn, try to add an appropriate
1461 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1462 operand, call expand_binop again, this time without a target. */
1463 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1464 && ! add_equal_note (pat
, ops
[0].value
,
1465 optab_to_code (binoptab
),
1466 ops
[1].value
, ops
[2].value
, mode0
))
1468 delete_insns_since (last
);
1469 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1470 unsignedp
, methods
);
1474 return ops
[0].value
;
1476 delete_insns_since (last
);
1480 /* Generate code to perform an operation specified by BINOPTAB
1481 on operands OP0 and OP1, with result having machine-mode MODE.
1483 UNSIGNEDP is for the case where we have to widen the operands
1484 to perform the operation. It says to use zero-extension.
1486 If TARGET is nonzero, the value
1487 is generated there, if it is convenient to do so.
1488 In all cases an rtx is returned for the locus of the value;
1489 this may or may not be TARGET. */
1492 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1493 rtx target
, int unsignedp
, enum optab_methods methods
)
1495 enum optab_methods next_methods
1496 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1497 ? OPTAB_WIDEN
: methods
);
1498 enum mode_class mclass
;
1499 enum insn_code icode
;
1500 machine_mode wider_mode
;
1501 scalar_int_mode int_mode
;
1504 rtx_insn
*entry_last
= get_last_insn ();
1507 mclass
= GET_MODE_CLASS (mode
);
1509 /* If subtracting an integer constant, convert this into an addition of
1510 the negated constant. */
1512 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1514 op1
= negate_rtx (mode
, op1
);
1515 binoptab
= add_optab
;
1517 /* For shifts, constant invalid op1 might be expanded from different
1518 mode than MODE. As those are invalid, force them to a register
1519 to avoid further problems during expansion. */
1520 else if (CONST_INT_P (op1
)
1521 && shift_optab_p (binoptab
)
1522 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1524 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1525 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1528 /* Record where to delete back to if we backtrack. */
1529 last
= get_last_insn ();
1531 /* If we can do it with a three-operand insn, do so. */
1533 if (methods
!= OPTAB_MUST_WIDEN
)
1535 if (convert_optab_p (binoptab
))
1537 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
1538 icode
= find_widening_optab_handler (binoptab
, mode
, from_mode
);
1541 icode
= optab_handler (binoptab
, mode
);
1542 if (icode
!= CODE_FOR_nothing
)
1544 temp
= expand_binop_directly (icode
, mode
, binoptab
, op0
, op1
,
1545 target
, unsignedp
, methods
, last
);
1551 /* If we were trying to rotate, and that didn't work, try rotating
1552 the other direction before falling back to shifts and bitwise-or. */
1553 if (((binoptab
== rotl_optab
1554 && (icode
= optab_handler (rotr_optab
, mode
)) != CODE_FOR_nothing
)
1555 || (binoptab
== rotr_optab
1556 && (icode
= optab_handler (rotl_optab
, mode
)) != CODE_FOR_nothing
))
1557 && is_int_mode (mode
, &int_mode
))
1559 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1561 unsigned int bits
= GET_MODE_PRECISION (int_mode
);
1563 if (CONST_INT_P (op1
))
1564 newop1
= gen_int_shift_amount (int_mode
, bits
- INTVAL (op1
));
1565 else if (targetm
.shift_truncation_mask (int_mode
) == bits
- 1)
1566 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1568 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1569 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1570 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1572 temp
= expand_binop_directly (icode
, int_mode
, otheroptab
, op0
, newop1
,
1573 target
, unsignedp
, methods
, last
);
1578 /* If this is a multiply, see if we can do a widening operation that
1579 takes operands of this mode and makes a wider mode. */
1581 if (binoptab
== smul_optab
1582 && GET_MODE_2XWIDER_MODE (mode
).exists (&wider_mode
)
1583 && (convert_optab_handler ((unsignedp
1585 : smul_widen_optab
),
1586 wider_mode
, mode
) != CODE_FOR_nothing
))
1588 /* *_widen_optab needs to determine operand mode, make sure at least
1589 one operand has non-VOID mode. */
1590 if (GET_MODE (op0
) == VOIDmode
&& GET_MODE (op1
) == VOIDmode
)
1591 op0
= force_reg (mode
, op0
);
1592 temp
= expand_binop (wider_mode
,
1593 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1594 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1598 if (GET_MODE_CLASS (mode
) == MODE_INT
1599 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1600 return gen_lowpart (mode
, temp
);
1602 return convert_to_mode (mode
, temp
, unsignedp
);
1606 /* If this is a vector shift by a scalar, see if we can do a vector
1607 shift by a vector. If so, broadcast the scalar into a vector. */
1608 if (mclass
== MODE_VECTOR_INT
)
1610 optab otheroptab
= unknown_optab
;
1612 if (binoptab
== ashl_optab
)
1613 otheroptab
= vashl_optab
;
1614 else if (binoptab
== ashr_optab
)
1615 otheroptab
= vashr_optab
;
1616 else if (binoptab
== lshr_optab
)
1617 otheroptab
= vlshr_optab
;
1618 else if (binoptab
== rotl_optab
)
1619 otheroptab
= vrotl_optab
;
1620 else if (binoptab
== rotr_optab
)
1621 otheroptab
= vrotr_optab
;
1624 && (icode
= optab_handler (otheroptab
, mode
)) != CODE_FOR_nothing
)
1626 /* The scalar may have been extended to be too wide. Truncate
1627 it back to the proper size to fit in the broadcast vector. */
1628 scalar_mode inner_mode
= GET_MODE_INNER (mode
);
1629 if (!CONST_INT_P (op1
)
1630 && (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (op1
)))
1631 > GET_MODE_BITSIZE (inner_mode
)))
1632 op1
= force_reg (inner_mode
,
1633 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1635 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1638 temp
= expand_binop_directly (icode
, mode
, otheroptab
, op0
, vop1
,
1639 target
, unsignedp
, methods
, last
);
1646 /* Look for a wider mode of the same class for which we think we
1647 can open-code the operation. Check for a widening multiply at the
1648 wider mode as well. */
1650 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1651 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1652 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1654 machine_mode next_mode
;
1655 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1656 || (binoptab
== smul_optab
1657 && GET_MODE_WIDER_MODE (wider_mode
).exists (&next_mode
)
1658 && (find_widening_optab_handler ((unsignedp
1660 : smul_widen_optab
),
1662 != CODE_FOR_nothing
)))
1664 rtx xop0
= op0
, xop1
= op1
;
1665 bool no_extend
= false;
1667 /* For certain integer operations, we need not actually extend
1668 the narrow operands, as long as we will truncate
1669 the results to the same narrowness. */
1671 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1672 || binoptab
== xor_optab
1673 || binoptab
== add_optab
|| binoptab
== sub_optab
1674 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1675 && mclass
== MODE_INT
)
1678 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1680 if (binoptab
!= ashl_optab
)
1681 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1685 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1687 /* The second operand of a shift must always be extended. */
1688 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1689 no_extend
&& binoptab
!= ashl_optab
);
1691 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1692 unsignedp
, OPTAB_DIRECT
);
1695 if (mclass
!= MODE_INT
1696 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1699 target
= gen_reg_rtx (mode
);
1700 convert_move (target
, temp
, 0);
1704 return gen_lowpart (mode
, temp
);
1707 delete_insns_since (last
);
1711 /* If operation is commutative,
1712 try to make the first operand a register.
1713 Even better, try to make it the same as the target.
1714 Also try to make the last operand a constant. */
1715 if (commutative_optab_p (binoptab
)
1716 && swap_commutative_operands_with_target (target
, op0
, op1
))
1717 std::swap (op0
, op1
);
1719 /* These can be done a word at a time. */
1720 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1721 && is_int_mode (mode
, &int_mode
)
1722 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
1723 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1728 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1729 won't be accurate, so use a new target. */
1733 || reg_overlap_mentioned_p (target
, op0
)
1734 || reg_overlap_mentioned_p (target
, op1
)
1735 || !valid_multiword_target_p (target
))
1736 target
= gen_reg_rtx (int_mode
);
1740 /* Do the actual arithmetic. */
1741 machine_mode op0_mode
= GET_MODE (op0
);
1742 machine_mode op1_mode
= GET_MODE (op1
);
1743 if (op0_mode
== VOIDmode
)
1744 op0_mode
= int_mode
;
1745 if (op1_mode
== VOIDmode
)
1746 op1_mode
= int_mode
;
1747 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
1749 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
1750 rtx x
= expand_binop (word_mode
, binoptab
,
1751 operand_subword_force (op0
, i
, op0_mode
),
1752 operand_subword_force (op1
, i
, op1_mode
),
1753 target_piece
, unsignedp
, next_methods
);
1758 if (target_piece
!= x
)
1759 emit_move_insn (target_piece
, x
);
1762 insns
= get_insns ();
1765 if (i
== GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
)
1772 /* Synthesize double word shifts from single word shifts. */
1773 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1774 || binoptab
== ashr_optab
)
1775 && is_int_mode (mode
, &int_mode
)
1776 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1777 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1778 && GET_MODE_PRECISION (int_mode
) == GET_MODE_BITSIZE (int_mode
)
1779 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1780 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1781 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1783 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1784 scalar_int_mode op1_mode
;
1786 double_shift_mask
= targetm
.shift_truncation_mask (int_mode
);
1787 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1788 op1_mode
= (GET_MODE (op1
) != VOIDmode
1789 ? as_a
<scalar_int_mode
> (GET_MODE (op1
))
1792 /* Apply the truncation to constant shifts. */
1793 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1794 op1
= gen_int_mode (INTVAL (op1
) & double_shift_mask
, op1_mode
);
1796 if (op1
== CONST0_RTX (op1_mode
))
1799 /* Make sure that this is a combination that expand_doubleword_shift
1800 can handle. See the comments there for details. */
1801 if (double_shift_mask
== 0
1802 || (shift_mask
== BITS_PER_WORD
- 1
1803 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1806 rtx into_target
, outof_target
;
1807 rtx into_input
, outof_input
;
1808 int left_shift
, outof_word
;
1810 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1811 won't be accurate, so use a new target. */
1815 || reg_overlap_mentioned_p (target
, op0
)
1816 || reg_overlap_mentioned_p (target
, op1
)
1817 || !valid_multiword_target_p (target
))
1818 target
= gen_reg_rtx (int_mode
);
1822 /* OUTOF_* is the word we are shifting bits away from, and
1823 INTO_* is the word that we are shifting bits towards, thus
1824 they differ depending on the direction of the shift and
1825 WORDS_BIG_ENDIAN. */
1827 left_shift
= binoptab
== ashl_optab
;
1828 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1830 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1831 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1833 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1834 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1836 if (expand_doubleword_shift (op1_mode
, binoptab
,
1837 outof_input
, into_input
, op1
,
1838 outof_target
, into_target
,
1839 unsignedp
, next_methods
, shift_mask
))
1841 insns
= get_insns ();
1851 /* Synthesize double word rotates from single word shifts. */
1852 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1853 && is_int_mode (mode
, &int_mode
)
1854 && CONST_INT_P (op1
)
1855 && GET_MODE_PRECISION (int_mode
) == 2 * BITS_PER_WORD
1856 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1857 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1860 rtx into_target
, outof_target
;
1861 rtx into_input
, outof_input
;
1863 int shift_count
, left_shift
, outof_word
;
1865 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1866 won't be accurate, so use a new target. Do this also if target is not
1867 a REG, first because having a register instead may open optimization
1868 opportunities, and second because if target and op0 happen to be MEMs
1869 designating the same location, we would risk clobbering it too early
1870 in the code sequence we generate below. */
1875 || reg_overlap_mentioned_p (target
, op0
)
1876 || reg_overlap_mentioned_p (target
, op1
)
1877 || !valid_multiword_target_p (target
))
1878 target
= gen_reg_rtx (int_mode
);
1882 shift_count
= INTVAL (op1
);
1884 /* OUTOF_* is the word we are shifting bits away from, and
1885 INTO_* is the word that we are shifting bits towards, thus
1886 they differ depending on the direction of the shift and
1887 WORDS_BIG_ENDIAN. */
1889 left_shift
= (binoptab
== rotl_optab
);
1890 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1892 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1893 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1895 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1896 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1898 if (shift_count
== BITS_PER_WORD
)
1900 /* This is just a word swap. */
1901 emit_move_insn (outof_target
, into_input
);
1902 emit_move_insn (into_target
, outof_input
);
1907 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1908 HOST_WIDE_INT first_shift_count
, second_shift_count
;
1909 optab reverse_unsigned_shift
, unsigned_shift
;
1911 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1912 ? lshr_optab
: ashl_optab
);
1914 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1915 ? ashl_optab
: lshr_optab
);
1917 if (shift_count
> BITS_PER_WORD
)
1919 first_shift_count
= shift_count
- BITS_PER_WORD
;
1920 second_shift_count
= 2 * BITS_PER_WORD
- shift_count
;
1924 first_shift_count
= BITS_PER_WORD
- shift_count
;
1925 second_shift_count
= shift_count
;
1927 rtx first_shift_count_rtx
1928 = gen_int_shift_amount (word_mode
, first_shift_count
);
1929 rtx second_shift_count_rtx
1930 = gen_int_shift_amount (word_mode
, second_shift_count
);
1932 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1933 outof_input
, first_shift_count_rtx
,
1934 NULL_RTX
, unsignedp
, next_methods
);
1935 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1936 into_input
, second_shift_count_rtx
,
1937 NULL_RTX
, unsignedp
, next_methods
);
1939 if (into_temp1
!= 0 && into_temp2
!= 0)
1940 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1941 into_target
, unsignedp
, next_methods
);
1945 if (inter
!= 0 && inter
!= into_target
)
1946 emit_move_insn (into_target
, inter
);
1948 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1949 into_input
, first_shift_count_rtx
,
1950 NULL_RTX
, unsignedp
, next_methods
);
1951 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1952 outof_input
, second_shift_count_rtx
,
1953 NULL_RTX
, unsignedp
, next_methods
);
1955 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1956 inter
= expand_binop (word_mode
, ior_optab
,
1957 outof_temp1
, outof_temp2
,
1958 outof_target
, unsignedp
, next_methods
);
1960 if (inter
!= 0 && inter
!= outof_target
)
1961 emit_move_insn (outof_target
, inter
);
1964 insns
= get_insns ();
1974 /* These can be done a word at a time by propagating carries. */
1975 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1976 && is_int_mode (mode
, &int_mode
)
1977 && GET_MODE_SIZE (int_mode
) >= 2 * UNITS_PER_WORD
1978 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1981 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1982 const unsigned int nwords
= GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
;
1983 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1984 rtx xop0
, xop1
, xtarget
;
1986 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1987 value is one of those, use it. Otherwise, use 1 since it is the
1988 one easiest to get. */
1989 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1990 int normalizep
= STORE_FLAG_VALUE
;
1995 /* Prepare the operands. */
1996 xop0
= force_reg (int_mode
, op0
);
1997 xop1
= force_reg (int_mode
, op1
);
1999 xtarget
= gen_reg_rtx (int_mode
);
2001 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
2004 /* Indicate for flow that the entire target reg is being set. */
2006 emit_clobber (xtarget
);
2008 /* Do the actual arithmetic. */
2009 for (i
= 0; i
< nwords
; i
++)
2011 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
2012 rtx target_piece
= operand_subword (xtarget
, index
, 1, int_mode
);
2013 rtx op0_piece
= operand_subword_force (xop0
, index
, int_mode
);
2014 rtx op1_piece
= operand_subword_force (xop1
, index
, int_mode
);
2017 /* Main add/subtract of the input operands. */
2018 x
= expand_binop (word_mode
, binoptab
,
2019 op0_piece
, op1_piece
,
2020 target_piece
, unsignedp
, next_methods
);
2026 /* Store carry from main add/subtract. */
2027 carry_out
= gen_reg_rtx (word_mode
);
2028 carry_out
= emit_store_flag_force (carry_out
,
2029 (binoptab
== add_optab
2032 word_mode
, 1, normalizep
);
2039 /* Add/subtract previous carry to main result. */
2040 newx
= expand_binop (word_mode
,
2041 normalizep
== 1 ? binoptab
: otheroptab
,
2043 NULL_RTX
, 1, next_methods
);
2047 /* Get out carry from adding/subtracting carry in. */
2048 rtx carry_tmp
= gen_reg_rtx (word_mode
);
2049 carry_tmp
= emit_store_flag_force (carry_tmp
,
2050 (binoptab
== add_optab
2053 word_mode
, 1, normalizep
);
2055 /* Logical-ior the two poss. carry together. */
2056 carry_out
= expand_binop (word_mode
, ior_optab
,
2057 carry_out
, carry_tmp
,
2058 carry_out
, 0, next_methods
);
2062 emit_move_insn (target_piece
, newx
);
2066 if (x
!= target_piece
)
2067 emit_move_insn (target_piece
, x
);
2070 carry_in
= carry_out
;
2073 if (i
== GET_MODE_BITSIZE (int_mode
) / (unsigned) BITS_PER_WORD
)
2075 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
2076 || ! rtx_equal_p (target
, xtarget
))
2078 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
2080 set_dst_reg_note (temp
, REG_EQUAL
,
2081 gen_rtx_fmt_ee (optab_to_code (binoptab
),
2082 int_mode
, copy_rtx (xop0
),
2093 delete_insns_since (last
);
2096 /* Attempt to synthesize double word multiplies using a sequence of word
2097 mode multiplications. We first attempt to generate a sequence using a
2098 more efficient unsigned widening multiply, and if that fails we then
2099 try using a signed widening multiply. */
2101 if (binoptab
== smul_optab
2102 && is_int_mode (mode
, &int_mode
)
2103 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2104 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
2105 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
2107 rtx product
= NULL_RTX
;
2108 if (convert_optab_handler (umul_widen_optab
, int_mode
, word_mode
)
2109 != CODE_FOR_nothing
)
2111 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
2114 delete_insns_since (last
);
2117 if (product
== NULL_RTX
2118 && (convert_optab_handler (smul_widen_optab
, int_mode
, word_mode
)
2119 != CODE_FOR_nothing
))
2121 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
2124 delete_insns_since (last
);
2127 if (product
!= NULL_RTX
)
2129 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
2131 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
2133 set_dst_reg_note (move
,
2135 gen_rtx_fmt_ee (MULT
, int_mode
,
2138 target
? target
: product
);
2144 /* Attempt to synthetize double word modulo by constant divisor. */
2145 if ((binoptab
== umod_optab
2146 || binoptab
== smod_optab
2147 || binoptab
== udiv_optab
2148 || binoptab
== sdiv_optab
)
2150 && CONST_INT_P (op1
)
2151 && is_int_mode (mode
, &int_mode
)
2152 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2153 && optab_handler ((binoptab
== umod_optab
|| binoptab
== udiv_optab
)
2154 ? udivmod_optab
: sdivmod_optab
,
2155 int_mode
) == CODE_FOR_nothing
2156 && optab_handler (and_optab
, word_mode
) != CODE_FOR_nothing
2157 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
2158 && optimize_insn_for_speed_p ())
2161 if ((binoptab
== umod_optab
|| binoptab
== smod_optab
)
2162 && (INTVAL (op1
) & 1) == 0)
2163 res
= expand_doubleword_mod (int_mode
, op0
, op1
,
2164 binoptab
== umod_optab
);
2167 rtx quot
= expand_doubleword_divmod (int_mode
, op0
, op1
, &res
,
2168 binoptab
== umod_optab
2169 || binoptab
== udiv_optab
);
2170 if (quot
== NULL_RTX
)
2172 else if (binoptab
== udiv_optab
|| binoptab
== sdiv_optab
)
2175 if (res
!= NULL_RTX
)
2177 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
2179 rtx_insn
*move
= emit_move_insn (target
? target
: res
,
2181 set_dst_reg_note (move
, REG_EQUAL
,
2182 gen_rtx_fmt_ee (optab_to_code (binoptab
),
2183 int_mode
, copy_rtx (op0
), op1
),
2184 target
? target
: res
);
2189 delete_insns_since (last
);
2192 /* It can't be open-coded in this mode.
2193 Use a library call if one is available and caller says that's ok. */
2195 libfunc
= optab_libfunc (binoptab
, mode
);
2197 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
2201 machine_mode op1_mode
= mode
;
2206 if (shift_optab_p (binoptab
))
2208 op1_mode
= targetm
.libgcc_shift_count_mode ();
2209 /* Specify unsigned here,
2210 since negative shift counts are meaningless. */
2211 op1x
= convert_to_mode (op1_mode
, op1
, 1);
2214 if (GET_MODE (op0
) != VOIDmode
2215 && GET_MODE (op0
) != mode
)
2216 op0
= convert_to_mode (mode
, op0
, unsignedp
);
2218 /* Pass 1 for NO_QUEUE so we don't lose any increments
2219 if the libcall is cse'd or moved. */
2220 value
= emit_library_call_value (libfunc
,
2221 NULL_RTX
, LCT_CONST
, mode
,
2222 op0
, mode
, op1x
, op1_mode
);
2224 insns
= get_insns ();
2227 bool trapv
= trapv_binoptab_p (binoptab
);
2228 target
= gen_reg_rtx (mode
);
2229 emit_libcall_block_1 (insns
, target
, value
,
2231 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
2232 mode
, op0
, op1
), trapv
);
2237 delete_insns_since (last
);
2239 /* It can't be done in this mode. Can we do it in a wider mode? */
2241 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
2242 || methods
== OPTAB_MUST_WIDEN
))
2244 /* Caller says, don't even try. */
2245 delete_insns_since (entry_last
);
2249 /* Compute the value of METHODS to pass to recursive calls.
2250 Don't allow widening to be tried recursively. */
2252 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
2254 /* Look for a wider mode of the same class for which it appears we can do
2257 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2259 /* This code doesn't make sense for conversion optabs, since we
2260 wouldn't then want to extend the operands to be the same size
2262 gcc_assert (!convert_optab_p (binoptab
));
2263 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2265 if (optab_handler (binoptab
, wider_mode
)
2266 || (methods
== OPTAB_LIB
2267 && optab_libfunc (binoptab
, wider_mode
)))
2269 rtx xop0
= op0
, xop1
= op1
;
2270 bool no_extend
= false;
2272 /* For certain integer operations, we need not actually extend
2273 the narrow operands, as long as we will truncate
2274 the results to the same narrowness. */
2276 if ((binoptab
== ior_optab
|| binoptab
== and_optab
2277 || binoptab
== xor_optab
2278 || binoptab
== add_optab
|| binoptab
== sub_optab
2279 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
2280 && mclass
== MODE_INT
)
2283 xop0
= widen_operand (xop0
, wider_mode
, mode
,
2284 unsignedp
, no_extend
);
2286 /* The second operand of a shift must always be extended. */
2287 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
2288 no_extend
&& binoptab
!= ashl_optab
);
2290 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
2291 unsignedp
, methods
);
2294 if (mclass
!= MODE_INT
2295 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2298 target
= gen_reg_rtx (mode
);
2299 convert_move (target
, temp
, 0);
2303 return gen_lowpart (mode
, temp
);
2306 delete_insns_since (last
);
2311 delete_insns_since (entry_last
);
2315 /* Expand a binary operator which has both signed and unsigned forms.
2316 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2319 If we widen unsigned operands, we may use a signed wider operation instead
2320 of an unsigned wider operation, since the result would be the same. */
2323 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
2324 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
2325 enum optab_methods methods
)
2328 optab direct_optab
= unsignedp
? uoptab
: soptab
;
2331 /* Do it without widening, if possible. */
2332 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2333 unsignedp
, OPTAB_DIRECT
);
2334 if (temp
|| methods
== OPTAB_DIRECT
)
2337 /* Try widening to a signed int. Disable any direct use of any
2338 signed insn in the current mode. */
2339 save_enable
= swap_optab_enable (soptab
, mode
, false);
2341 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2342 unsignedp
, OPTAB_WIDEN
);
2344 /* For unsigned operands, try widening to an unsigned int. */
2345 if (!temp
&& unsignedp
)
2346 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2347 unsignedp
, OPTAB_WIDEN
);
2348 if (temp
|| methods
== OPTAB_WIDEN
)
2351 /* Use the right width libcall if that exists. */
2352 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2353 unsignedp
, OPTAB_LIB
);
2354 if (temp
|| methods
== OPTAB_LIB
)
2357 /* Must widen and use a libcall, use either signed or unsigned. */
2358 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2359 unsignedp
, methods
);
2360 if (!temp
&& unsignedp
)
2361 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2362 unsignedp
, methods
);
2365 /* Undo the fiddling above. */
2367 swap_optab_enable (soptab
, mode
, true);
2371 /* Generate code to perform an operation specified by UNOPPTAB
2372 on operand OP0, with two results to TARG0 and TARG1.
2373 We assume that the order of the operands for the instruction
2374 is TARG0, TARG1, OP0.
2376 Either TARG0 or TARG1 may be zero, but what that means is that
2377 the result is not actually wanted. We will generate it into
2378 a dummy pseudo-reg and discard it. They may not both be zero.
2380 Returns true if this operation can be performed; false if not. */
2383 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
2386 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2387 enum mode_class mclass
;
2388 machine_mode wider_mode
;
2389 rtx_insn
*entry_last
= get_last_insn ();
2392 mclass
= GET_MODE_CLASS (mode
);
2395 targ0
= gen_reg_rtx (mode
);
2397 targ1
= gen_reg_rtx (mode
);
2399 /* Record where to go back to if we fail. */
2400 last
= get_last_insn ();
2402 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2404 class expand_operand ops
[3];
2405 enum insn_code icode
= optab_handler (unoptab
, mode
);
2407 create_fixed_operand (&ops
[0], targ0
);
2408 create_fixed_operand (&ops
[1], targ1
);
2409 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
2410 if (maybe_expand_insn (icode
, 3, ops
))
2414 /* It can't be done in this mode. Can we do it in a wider mode? */
2416 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2418 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2420 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2422 rtx t0
= gen_reg_rtx (wider_mode
);
2423 rtx t1
= gen_reg_rtx (wider_mode
);
2424 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2426 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
2428 convert_move (targ0
, t0
, unsignedp
);
2429 convert_move (targ1
, t1
, unsignedp
);
2433 delete_insns_since (last
);
2438 delete_insns_since (entry_last
);
2442 /* Generate code to perform an operation specified by BINOPTAB
2443 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2444 We assume that the order of the operands for the instruction
2445 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2446 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2448 Either TARG0 or TARG1 may be zero, but what that means is that
2449 the result is not actually wanted. We will generate it into
2450 a dummy pseudo-reg and discard it. They may not both be zero.
2452 Returns true if this operation can be performed; false if not. */
2455 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
2458 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2459 enum mode_class mclass
;
2460 machine_mode wider_mode
;
2461 rtx_insn
*entry_last
= get_last_insn ();
2464 mclass
= GET_MODE_CLASS (mode
);
2467 targ0
= gen_reg_rtx (mode
);
2469 targ1
= gen_reg_rtx (mode
);
2471 /* Record where to go back to if we fail. */
2472 last
= get_last_insn ();
2474 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2476 class expand_operand ops
[4];
2477 enum insn_code icode
= optab_handler (binoptab
, mode
);
2478 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2479 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2480 rtx xop0
= op0
, xop1
= op1
;
2482 /* If we are optimizing, force expensive constants into a register. */
2483 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2484 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2486 create_fixed_operand (&ops
[0], targ0
);
2487 create_convert_operand_from (&ops
[1], xop0
, mode
, unsignedp
);
2488 create_convert_operand_from (&ops
[2], xop1
, mode
, unsignedp
);
2489 create_fixed_operand (&ops
[3], targ1
);
2490 if (maybe_expand_insn (icode
, 4, ops
))
2492 delete_insns_since (last
);
2495 /* It can't be done in this mode. Can we do it in a wider mode? */
2497 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2499 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2501 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2503 rtx t0
= gen_reg_rtx (wider_mode
);
2504 rtx t1
= gen_reg_rtx (wider_mode
);
2505 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2506 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2508 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2511 convert_move (targ0
, t0
, unsignedp
);
2512 convert_move (targ1
, t1
, unsignedp
);
2516 delete_insns_since (last
);
2521 delete_insns_since (entry_last
);
2525 /* Expand the two-valued library call indicated by BINOPTAB, but
2526 preserve only one of the values. If TARG0 is non-NULL, the first
2527 value is placed into TARG0; otherwise the second value is placed
2528 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2529 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2530 This routine assumes that the value returned by the library call is
2531 as if the return value was of an integral mode twice as wide as the
2532 mode of OP0. Returns 1 if the call was successful. */
2535 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2536 rtx targ0
, rtx targ1
, enum rtx_code code
)
2539 machine_mode libval_mode
;
2544 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2545 gcc_assert (!targ0
!= !targ1
);
2547 mode
= GET_MODE (op0
);
2548 libfunc
= optab_libfunc (binoptab
, mode
);
2552 /* The value returned by the library function will have twice as
2553 many bits as the nominal MODE. */
2554 libval_mode
= smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode
));
2556 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2560 /* Get the part of VAL containing the value that we want. */
2561 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2562 targ0
? 0 : GET_MODE_SIZE (mode
));
2563 insns
= get_insns ();
2565 /* Move the into the desired location. */
2566 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2567 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2573 /* Wrapper around expand_unop which takes an rtx code to specify
2574 the operation to perform, not an optab pointer. All other
2575 arguments are the same. */
2577 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2578 rtx target
, int unsignedp
)
2580 optab unop
= code_to_optab (code
);
2583 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2589 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2591 A similar operation can be used for clrsb. UNOPTAB says which operation
2592 we are trying to expand. */
2594 widen_leading (scalar_int_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2596 opt_scalar_int_mode wider_mode_iter
;
2597 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2599 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2600 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2605 last
= get_last_insn ();
2608 target
= gen_reg_rtx (mode
);
2609 xop0
= widen_operand (op0
, wider_mode
, mode
,
2610 unoptab
!= clrsb_optab
, false);
2611 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2612 unoptab
!= clrsb_optab
);
2615 (wider_mode
, sub_optab
, temp
,
2616 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2617 - GET_MODE_PRECISION (mode
),
2619 target
, true, OPTAB_DIRECT
);
2621 delete_insns_since (last
);
2629 /* Attempt to emit (clrsb:mode op0) as
2630 (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2632 if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2634 (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2635 (ashr:mode op0 (const_int prec-1)))
2640 expand_clrsb_using_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2642 if (optimize_insn_for_size_p ()
2643 || optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2647 HOST_WIDE_INT val
= 0;
2648 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) != 2
2649 || val
!= GET_MODE_PRECISION (mode
))
2657 temp2
= expand_binop (mode
, ashl_optab
, op0
, const1_rtx
,
2658 NULL_RTX
, 0, OPTAB_DIRECT
);
2667 rtx temp
= expand_binop (mode
, ashr_optab
, op0
,
2668 GEN_INT (GET_MODE_PRECISION (mode
) - 1),
2669 NULL_RTX
, 0, OPTAB_DIRECT
);
2673 temp
= expand_binop (mode
, xor_optab
, temp2
, temp
, NULL_RTX
, 0,
2680 temp
= expand_binop (mode
, ior_optab
, temp
, const1_rtx
,
2681 NULL_RTX
, 0, OPTAB_DIRECT
);
2685 temp
= expand_unop_direct (mode
, clz_optab
, temp
, val
? NULL_RTX
: target
,
2691 temp
= expand_binop (mode
, add_optab
, temp
, constm1_rtx
,
2692 target
, 0, OPTAB_DIRECT
);
2697 rtx_insn
*seq
= get_insns ();
2700 add_equal_note (seq
, temp
, CLRSB
, op0
, NULL_RTX
, mode
);
2705 static rtx
expand_ffs (scalar_int_mode
, rtx
, rtx
);
2707 /* Try calculating clz, ctz or ffs of a double-word quantity as two clz, ctz or
2708 ffs operations on word-sized quantities, choosing which based on whether the
2709 high (for clz) or low (for ctz and ffs) word is nonzero. */
2711 expand_doubleword_clz_ctz_ffs (scalar_int_mode mode
, rtx op0
, rtx target
,
2714 rtx xop0
= force_reg (mode
, op0
);
2715 rtx subhi
= gen_highpart (word_mode
, xop0
);
2716 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2717 rtx_code_label
*hi0_label
= gen_label_rtx ();
2718 rtx_code_label
*after_label
= gen_label_rtx ();
2723 /* If we were not given a target, use a word_mode register, not a
2724 'mode' register. The result will fit, and nobody is expecting
2725 anything bigger (the return type of __builtin_clz* is int). */
2727 target
= gen_reg_rtx (word_mode
);
2729 /* In any case, write to a word_mode scratch in both branches of the
2730 conditional, so we can ensure there is a single move insn setting
2731 'target' to tag a REG_EQUAL note on. */
2732 result
= gen_reg_rtx (word_mode
);
2734 if (unoptab
!= clz_optab
)
2735 std::swap (subhi
, sublo
);
2739 /* If the high word is not equal to zero,
2740 then clz of the full value is clz of the high word. */
2741 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2742 word_mode
, true, hi0_label
);
2744 if (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2745 temp
= expand_unop_direct (word_mode
, unoptab
, subhi
, result
, true);
2748 gcc_assert (unoptab
== ffs_optab
);
2749 temp
= expand_ffs (word_mode
, subhi
, result
);
2755 convert_move (result
, temp
, true);
2757 emit_jump_insn (targetm
.gen_jump (after_label
));
2760 /* Else clz of the full value is clz of the low word plus the number
2761 of bits in the high word. Similarly for ctz/ffs of the high word,
2762 except that ffs should be 0 when both words are zero. */
2763 emit_label (hi0_label
);
2765 if (unoptab
== ffs_optab
)
2767 convert_move (result
, const0_rtx
, true);
2768 emit_cmp_and_jump_insns (sublo
, CONST0_RTX (word_mode
), EQ
, 0,
2769 word_mode
, true, after_label
);
2772 if (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2773 temp
= expand_unop_direct (word_mode
, unoptab
, sublo
, NULL_RTX
, true);
2776 gcc_assert (unoptab
== ffs_optab
);
2777 temp
= expand_unop_direct (word_mode
, ctz_optab
, sublo
, NULL_RTX
, true);
2784 temp
= expand_binop (word_mode
, add_optab
, temp
,
2785 gen_int_mode (GET_MODE_BITSIZE (word_mode
) + addend
,
2787 result
, true, OPTAB_DIRECT
);
2791 convert_move (result
, temp
, true);
2793 emit_label (after_label
);
2794 convert_move (target
, result
, true);
2799 add_equal_note (seq
, target
, optab_to_code (unoptab
), xop0
, NULL_RTX
, mode
);
2808 /* Try calculating popcount of a double-word quantity as two popcount's of
2809 word-sized quantities and summing up the results. */
2811 expand_doubleword_popcount (scalar_int_mode mode
, rtx op0
, rtx target
)
2818 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2819 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2821 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2822 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2830 /* If we were not given a target, use a word_mode register, not a
2831 'mode' register. The result will fit, and nobody is expecting
2832 anything bigger (the return type of __builtin_popcount* is int). */
2834 target
= gen_reg_rtx (word_mode
);
2836 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2841 add_equal_note (seq
, t
, POPCOUNT
, op0
, NULL_RTX
, mode
);
2849 (parity:narrow (low (x) ^ high (x))) */
2851 expand_doubleword_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2853 rtx t
= expand_binop (word_mode
, xor_optab
,
2854 operand_subword_force (op0
, 0, mode
),
2855 operand_subword_force (op0
, 1, mode
),
2856 NULL_RTX
, 0, OPTAB_DIRECT
);
2857 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2863 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2865 widen_bswap (scalar_int_mode mode
, rtx op0
, rtx target
)
2869 opt_scalar_int_mode wider_mode_iter
;
2871 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2872 if (optab_handler (bswap_optab
, wider_mode_iter
.require ())
2873 != CODE_FOR_nothing
)
2876 if (!wider_mode_iter
.exists ())
2879 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2880 last
= get_last_insn ();
2882 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2883 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2885 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2886 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2888 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2889 GET_MODE_BITSIZE (wider_mode
)
2890 - GET_MODE_BITSIZE (mode
),
2896 target
= gen_reg_rtx (mode
);
2897 emit_move_insn (target
, gen_lowpart (mode
, x
));
2900 delete_insns_since (last
);
2905 /* Try calculating bswap as two bswaps of two word-sized operands. */
2908 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2912 t1
= expand_unop (word_mode
, bswap_optab
,
2913 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2914 t0
= expand_unop (word_mode
, bswap_optab
,
2915 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2917 if (target
== 0 || !valid_multiword_target_p (target
))
2918 target
= gen_reg_rtx (mode
);
2920 emit_clobber (target
);
2921 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2922 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2927 /* Try calculating (parity x) as (and (popcount x) 1), where
2928 popcount can also be done in a wider mode. */
2930 expand_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2932 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2933 opt_scalar_int_mode wider_mode_iter
;
2934 FOR_EACH_MODE_FROM (wider_mode_iter
, mode
)
2936 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2937 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2942 last
= get_last_insn ();
2944 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2945 target
= gen_reg_rtx (wider_mode
);
2947 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2948 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2951 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2952 target
, true, OPTAB_DIRECT
);
2956 if (mclass
!= MODE_INT
2957 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2958 return convert_to_mode (mode
, temp
, 0);
2960 return gen_lowpart (mode
, temp
);
2963 delete_insns_since (last
);
2969 /* Try calculating ctz(x) as K - clz(x & -x) ,
2970 where K is GET_MODE_PRECISION(mode) - 1.
2972 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2973 don't have to worry about what the hardware does in that case. (If
2974 the clz instruction produces the usual value at 0, which is K, the
2975 result of this code sequence will be -1; expand_ffs, below, relies
2976 on this. It might be nice to have it be K instead, for consistency
2977 with the (very few) processors that provide a ctz with a defined
2978 value, but that would take one more instruction, and it would be
2979 less convenient for expand_ffs anyway. */
2982 expand_ctz (scalar_int_mode mode
, rtx op0
, rtx target
)
2987 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2992 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2994 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2995 true, OPTAB_DIRECT
);
2997 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2999 temp
= expand_binop (mode
, sub_optab
,
3000 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
3002 true, OPTAB_DIRECT
);
3012 add_equal_note (seq
, temp
, CTZ
, op0
, NULL_RTX
, mode
);
3018 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
3019 else with the sequence used by expand_clz.
3021 The ffs builtin promises to return zero for a zero value and ctz/clz
3022 may have an undefined value in that case. If they do not give us a
3023 convenient value, we have to generate a test and branch. */
3025 expand_ffs (scalar_int_mode mode
, rtx op0
, rtx target
)
3027 HOST_WIDE_INT val
= 0;
3028 bool defined_at_zero
= false;
3032 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
3036 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
3040 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
3042 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
3045 temp
= expand_ctz (mode
, op0
, 0);
3049 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
3051 defined_at_zero
= true;
3052 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
3058 if (defined_at_zero
&& val
== -1)
3059 /* No correction needed at zero. */;
3062 /* We don't try to do anything clever with the situation found
3063 on some processors (eg Alpha) where ctz(0:mode) ==
3064 bitsize(mode). If someone can think of a way to send N to -1
3065 and leave alone all values in the range 0..N-1 (where N is a
3066 power of two), cheaper than this test-and-branch, please add it.
3068 The test-and-branch is done after the operation itself, in case
3069 the operation sets condition codes that can be recycled for this.
3070 (This is true on i386, for instance.) */
3072 rtx_code_label
*nonzero_label
= gen_label_rtx ();
3073 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
3074 mode
, true, nonzero_label
);
3076 convert_move (temp
, GEN_INT (-1), false);
3077 emit_label (nonzero_label
);
3080 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
3081 to produce a value in the range 0..bitsize. */
3082 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
3083 target
, false, OPTAB_DIRECT
);
3090 add_equal_note (seq
, temp
, FFS
, op0
, NULL_RTX
, mode
);
3099 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
3100 conditions, VAL may already be a SUBREG against which we cannot generate
3101 a further SUBREG. In this case, we expect forcing the value into a
3102 register will work around the situation. */
3105 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
3109 ret
= lowpart_subreg (omode
, val
, imode
);
3112 val
= force_reg (imode
, val
);
3113 ret
= lowpart_subreg (omode
, val
, imode
);
3114 gcc_assert (ret
!= NULL
);
3119 /* Expand a floating point absolute value or negation operation via a
3120 logical operation on the sign bit. */
3123 expand_absneg_bit (enum rtx_code code
, scalar_float_mode mode
,
3124 rtx op0
, rtx target
)
3126 const struct real_format
*fmt
;
3127 int bitpos
, word
, nwords
, i
;
3128 scalar_int_mode imode
;
3132 /* The format has to have a simple sign bit. */
3133 fmt
= REAL_MODE_FORMAT (mode
);
3137 bitpos
= fmt
->signbit_rw
;
3141 /* Don't create negative zeros if the format doesn't support them. */
3142 if (code
== NEG
&& !fmt
->has_signed_zero
)
3145 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3147 if (!int_mode_for_mode (mode
).exists (&imode
))
3156 if (FLOAT_WORDS_BIG_ENDIAN
)
3157 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3159 word
= bitpos
/ BITS_PER_WORD
;
3160 bitpos
= bitpos
% BITS_PER_WORD
;
3161 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3164 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3170 || reg_overlap_mentioned_p (target
, op0
)
3171 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3172 target
= gen_reg_rtx (mode
);
3178 for (i
= 0; i
< nwords
; ++i
)
3180 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3181 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3185 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
3187 immed_wide_int_const (mask
, imode
),
3188 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3189 if (temp
!= targ_piece
)
3190 emit_move_insn (targ_piece
, temp
);
3193 emit_move_insn (targ_piece
, op0_piece
);
3196 insns
= get_insns ();
3203 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
3204 gen_lowpart (imode
, op0
),
3205 immed_wide_int_const (mask
, imode
),
3206 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3207 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3209 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
3210 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
3217 /* As expand_unop, but will fail rather than attempt the operation in a
3218 different mode or with a libcall. */
3220 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
3223 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
3225 class expand_operand ops
[2];
3226 enum insn_code icode
= optab_handler (unoptab
, mode
);
3227 rtx_insn
*last
= get_last_insn ();
3230 create_output_operand (&ops
[0], target
, mode
);
3231 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
3232 pat
= maybe_gen_insn (icode
, 2, ops
);
3235 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3236 && ! add_equal_note (pat
, ops
[0].value
,
3237 optab_to_code (unoptab
),
3238 ops
[1].value
, NULL_RTX
, mode
))
3240 delete_insns_since (last
);
3241 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
3246 return ops
[0].value
;
3252 /* Generate code to perform an operation specified by UNOPTAB
3253 on operand OP0, with result having machine-mode MODE.
3255 UNSIGNEDP is for the case where we have to widen the operands
3256 to perform the operation. It says to use zero-extension.
3258 If TARGET is nonzero, the value
3259 is generated there, if it is convenient to do so.
3260 In all cases an rtx is returned for the locus of the value;
3261 this may or may not be TARGET. */
3264 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
3267 enum mode_class mclass
= GET_MODE_CLASS (mode
);
3268 machine_mode wider_mode
;
3269 scalar_int_mode int_mode
;
3270 scalar_float_mode float_mode
;
3274 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
3278 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3280 /* Widening (or narrowing) clz needs special treatment. */
3281 if (unoptab
== clz_optab
)
3283 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3285 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
3289 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3290 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3292 temp
= expand_doubleword_clz_ctz_ffs (int_mode
, op0
, target
,
3302 if (unoptab
== clrsb_optab
)
3304 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3306 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
3309 temp
= expand_clrsb_using_clz (int_mode
, op0
, target
);
3316 if (unoptab
== popcount_optab
3317 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3318 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3319 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3320 && optimize_insn_for_speed_p ())
3322 temp
= expand_doubleword_popcount (int_mode
, op0
, target
);
3327 if (unoptab
== parity_optab
3328 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3329 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3330 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3331 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
3332 && optimize_insn_for_speed_p ())
3334 temp
= expand_doubleword_parity (int_mode
, op0
, target
);
3339 /* Widening (or narrowing) bswap needs special treatment. */
3340 if (unoptab
== bswap_optab
)
3342 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3343 or ROTATERT. First try these directly; if this fails, then try the
3344 obvious pair of shifts with allowed widening, as this will probably
3345 be always more efficient than the other fallback methods. */
3351 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
3353 temp
= expand_binop (mode
, rotl_optab
, op0
,
3354 gen_int_shift_amount (mode
, 8),
3355 target
, unsignedp
, OPTAB_DIRECT
);
3360 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
3362 temp
= expand_binop (mode
, rotr_optab
, op0
,
3363 gen_int_shift_amount (mode
, 8),
3364 target
, unsignedp
, OPTAB_DIRECT
);
3369 last
= get_last_insn ();
3371 temp1
= expand_binop (mode
, ashl_optab
, op0
,
3372 gen_int_shift_amount (mode
, 8), NULL_RTX
,
3373 unsignedp
, OPTAB_WIDEN
);
3374 temp2
= expand_binop (mode
, lshr_optab
, op0
,
3375 gen_int_shift_amount (mode
, 8), NULL_RTX
,
3376 unsignedp
, OPTAB_WIDEN
);
3379 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
3380 unsignedp
, OPTAB_WIDEN
);
3385 delete_insns_since (last
);
3388 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
3390 temp
= widen_bswap (int_mode
, op0
, target
);
3394 /* We do not provide a 128-bit bswap in libgcc so force the use of
3395 a double bswap for 64-bit targets. */
3396 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3397 && (UNITS_PER_WORD
== 8
3398 || optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
))
3400 temp
= expand_doubleword_bswap (mode
, op0
, target
);
3409 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3410 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3412 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
3415 rtx_insn
*last
= get_last_insn ();
3417 /* For certain operations, we need not actually extend
3418 the narrow operand, as long as we will truncate the
3419 results to the same narrowness. */
3421 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3422 (unoptab
== neg_optab
3423 || unoptab
== one_cmpl_optab
)
3424 && mclass
== MODE_INT
);
3426 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3431 if (mclass
!= MODE_INT
3432 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
3435 target
= gen_reg_rtx (mode
);
3436 convert_move (target
, temp
, 0);
3440 return gen_lowpart (mode
, temp
);
3443 delete_insns_since (last
);
3447 /* These can be done a word at a time. */
3448 if (unoptab
== one_cmpl_optab
3449 && is_int_mode (mode
, &int_mode
)
3450 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
3451 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3458 || reg_overlap_mentioned_p (target
, op0
)
3459 || !valid_multiword_target_p (target
))
3460 target
= gen_reg_rtx (int_mode
);
3464 /* Do the actual arithmetic. */
3465 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
3467 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
3468 rtx x
= expand_unop (word_mode
, unoptab
,
3469 operand_subword_force (op0
, i
, int_mode
),
3470 target_piece
, unsignedp
);
3472 if (target_piece
!= x
)
3473 emit_move_insn (target_piece
, x
);
3476 insns
= get_insns ();
3483 /* Emit ~op0 as op0 ^ -1. */
3484 if (unoptab
== one_cmpl_optab
3485 && (SCALAR_INT_MODE_P (mode
) || GET_MODE_CLASS (mode
) == MODE_VECTOR_INT
)
3486 && optab_handler (xor_optab
, mode
) != CODE_FOR_nothing
)
3488 temp
= expand_binop (mode
, xor_optab
, op0
, CONSTM1_RTX (mode
),
3489 target
, unsignedp
, OPTAB_DIRECT
);
3494 if (optab_to_code (unoptab
) == NEG
)
3496 /* Try negating floating point values by flipping the sign bit. */
3497 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3499 temp
= expand_absneg_bit (NEG
, float_mode
, op0
, target
);
3504 /* If there is no negation pattern, and we have no negative zero,
3505 try subtracting from zero. */
3506 if (!HONOR_SIGNED_ZEROS (mode
))
3508 temp
= expand_binop (mode
, (unoptab
== negv_optab
3509 ? subv_optab
: sub_optab
),
3510 CONST0_RTX (mode
), op0
, target
,
3511 unsignedp
, OPTAB_DIRECT
);
3517 /* Try calculating parity (x) as popcount (x) % 2. */
3518 if (unoptab
== parity_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3520 temp
= expand_parity (int_mode
, op0
, target
);
3525 /* Try implementing ffs (x) in terms of clz (x). */
3526 if (unoptab
== ffs_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3528 temp
= expand_ffs (int_mode
, op0
, target
);
3533 /* Try implementing ctz (x) in terms of clz (x). */
3534 if (unoptab
== ctz_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
3536 temp
= expand_ctz (int_mode
, op0
, target
);
3541 if ((unoptab
== ctz_optab
|| unoptab
== ffs_optab
)
3542 && optimize_insn_for_speed_p ()
3543 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
3544 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
3545 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
3546 || optab_handler (ctz_optab
, word_mode
) != CODE_FOR_nothing
))
3548 temp
= expand_doubleword_clz_ctz_ffs (int_mode
, op0
, target
, unoptab
);
3554 /* Now try a library call in this mode. */
3555 libfunc
= optab_libfunc (unoptab
, mode
);
3561 machine_mode outmode
= mode
;
3563 /* All of these functions return small values. Thus we choose to
3564 have them return something that isn't a double-word. */
3565 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
3566 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
3567 || unoptab
== parity_optab
)
3569 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
3570 optab_libfunc (unoptab
, mode
)));
3574 /* Pass 1 for NO_QUEUE so we don't lose any increments
3575 if the libcall is cse'd or moved. */
3576 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
3578 insns
= get_insns ();
3581 target
= gen_reg_rtx (outmode
);
3582 bool trapv
= trapv_unoptab_p (unoptab
);
3584 eq_value
= NULL_RTX
;
3587 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
3588 if (GET_MODE_UNIT_SIZE (outmode
) < GET_MODE_UNIT_SIZE (mode
))
3589 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
3590 else if (GET_MODE_UNIT_SIZE (outmode
) > GET_MODE_UNIT_SIZE (mode
))
3591 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
3592 outmode
, eq_value
, mode
);
3594 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
3599 /* It can't be done in this mode. Can we do it in a wider mode? */
3601 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3603 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3605 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
3606 || optab_libfunc (unoptab
, wider_mode
))
3609 rtx_insn
*last
= get_last_insn ();
3611 /* For certain operations, we need not actually extend
3612 the narrow operand, as long as we will truncate the
3613 results to the same narrowness. */
3614 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3615 (unoptab
== neg_optab
3616 || unoptab
== one_cmpl_optab
3617 || unoptab
== bswap_optab
)
3618 && mclass
== MODE_INT
);
3620 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3623 /* If we are generating clz using wider mode, adjust the
3624 result. Similarly for clrsb. */
3625 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3628 scalar_int_mode wider_int_mode
3629 = as_a
<scalar_int_mode
> (wider_mode
);
3630 int_mode
= as_a
<scalar_int_mode
> (mode
);
3632 (wider_mode
, sub_optab
, temp
,
3633 gen_int_mode (GET_MODE_PRECISION (wider_int_mode
)
3634 - GET_MODE_PRECISION (int_mode
),
3636 target
, true, OPTAB_DIRECT
);
3639 /* Likewise for bswap. */
3640 if (unoptab
== bswap_optab
&& temp
!= 0)
3642 scalar_int_mode wider_int_mode
3643 = as_a
<scalar_int_mode
> (wider_mode
);
3644 int_mode
= as_a
<scalar_int_mode
> (mode
);
3645 gcc_assert (GET_MODE_PRECISION (wider_int_mode
)
3646 == GET_MODE_BITSIZE (wider_int_mode
)
3647 && GET_MODE_PRECISION (int_mode
)
3648 == GET_MODE_BITSIZE (int_mode
));
3650 temp
= expand_shift (RSHIFT_EXPR
, wider_int_mode
, temp
,
3651 GET_MODE_BITSIZE (wider_int_mode
)
3652 - GET_MODE_BITSIZE (int_mode
),
3658 if (mclass
!= MODE_INT
)
3661 target
= gen_reg_rtx (mode
);
3662 convert_move (target
, temp
, 0);
3666 return gen_lowpart (mode
, temp
);
3669 delete_insns_since (last
);
3674 /* One final attempt at implementing negation via subtraction,
3675 this time allowing widening of the operand. */
3676 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3679 temp
= expand_binop (mode
,
3680 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3681 CONST0_RTX (mode
), op0
,
3682 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3690 /* Emit code to compute the absolute value of OP0, with result to
3691 TARGET if convenient. (TARGET may be 0.) The return value says
3692 where the result actually is to be found.
3694 MODE is the mode of the operand; the mode of the result is
3695 different but can be deduced from MODE.
3700 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3701 int result_unsignedp
)
3705 if (GET_MODE_CLASS (mode
) != MODE_INT
3707 result_unsignedp
= 1;
3709 /* First try to do it with a special abs instruction. */
3710 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3715 /* For floating point modes, try clearing the sign bit. */
3716 scalar_float_mode float_mode
;
3717 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3719 temp
= expand_absneg_bit (ABS
, float_mode
, op0
, target
);
3724 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3725 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3726 && !HONOR_SIGNED_ZEROS (mode
))
3728 rtx_insn
*last
= get_last_insn ();
3730 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3733 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3739 delete_insns_since (last
);
3742 /* If this machine has expensive jumps, we can do integer absolute
3743 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3744 where W is the width of MODE. */
3746 scalar_int_mode int_mode
;
3747 if (is_int_mode (mode
, &int_mode
)
3748 && BRANCH_COST (optimize_insn_for_speed_p (),
3751 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3752 GET_MODE_PRECISION (int_mode
) - 1,
3755 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3758 temp
= expand_binop (int_mode
,
3759 result_unsignedp
? sub_optab
: subv_optab
,
3760 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3770 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3771 int result_unsignedp
, int safe
)
3774 rtx_code_label
*op1
;
3776 if (GET_MODE_CLASS (mode
) != MODE_INT
3778 result_unsignedp
= 1;
3780 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3784 /* If that does not win, use conditional jump and negate. */
3786 /* It is safe to use the target if it is the same
3787 as the source if this is also a pseudo register */
3788 if (op0
== target
&& REG_P (op0
)
3789 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3792 op1
= gen_label_rtx ();
3793 if (target
== 0 || ! safe
3794 || GET_MODE (target
) != mode
3795 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3797 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3798 target
= gen_reg_rtx (mode
);
3800 emit_move_insn (target
, op0
);
3803 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3804 NULL_RTX
, NULL
, op1
,
3805 profile_probability::uninitialized ());
3807 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3810 emit_move_insn (target
, op0
);
3816 /* Emit code to compute the one's complement absolute value of OP0
3817 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3818 (TARGET may be NULL_RTX.) The return value says where the result
3819 actually is to be found.
3821 MODE is the mode of the operand; the mode of the result is
3822 different but can be deduced from MODE. */
3825 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3829 /* Not applicable for floating point modes. */
3830 if (FLOAT_MODE_P (mode
))
3833 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3834 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3836 rtx_insn
*last
= get_last_insn ();
3838 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3840 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3846 delete_insns_since (last
);
3849 /* If this machine has expensive jumps, we can do one's complement
3850 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3852 scalar_int_mode int_mode
;
3853 if (is_int_mode (mode
, &int_mode
)
3854 && BRANCH_COST (optimize_insn_for_speed_p (),
3857 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3858 GET_MODE_PRECISION (int_mode
) - 1,
3861 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3871 /* A subroutine of expand_copysign, perform the copysign operation using the
3872 abs and neg primitives advertised to exist on the target. The assumption
3873 is that we have a split register file, and leaving op0 in fp registers,
3874 and not playing with subregs so much, will help the register allocator. */
3877 expand_copysign_absneg (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3878 int bitpos
, bool op0_is_abs
)
3880 scalar_int_mode imode
;
3881 enum insn_code icode
;
3883 rtx_code_label
*label
;
3888 /* Check if the back end provides an insn that handles signbit for the
3890 icode
= optab_handler (signbit_optab
, mode
);
3891 if (icode
!= CODE_FOR_nothing
)
3893 imode
= as_a
<scalar_int_mode
> (insn_data
[(int) icode
].operand
[0].mode
);
3894 sign
= gen_reg_rtx (imode
);
3895 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3899 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3901 if (!int_mode_for_mode (mode
).exists (&imode
))
3903 op1
= gen_lowpart (imode
, op1
);
3910 if (FLOAT_WORDS_BIG_ENDIAN
)
3911 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3913 word
= bitpos
/ BITS_PER_WORD
;
3914 bitpos
= bitpos
% BITS_PER_WORD
;
3915 op1
= operand_subword_force (op1
, word
, mode
);
3918 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3919 sign
= expand_binop (imode
, and_optab
, op1
,
3920 immed_wide_int_const (mask
, imode
),
3921 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3926 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3933 if (target
== NULL_RTX
)
3934 target
= copy_to_reg (op0
);
3936 emit_move_insn (target
, op0
);
3939 label
= gen_label_rtx ();
3940 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3942 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3943 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3945 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3947 emit_move_insn (target
, op0
);
3955 /* A subroutine of expand_copysign, perform the entire copysign operation
3956 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3957 is true if op0 is known to have its sign bit clear. */
3960 expand_copysign_bit (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3961 int bitpos
, bool op0_is_abs
)
3963 scalar_int_mode imode
;
3964 int word
, nwords
, i
;
3968 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3970 if (!int_mode_for_mode (mode
).exists (&imode
))
3979 if (FLOAT_WORDS_BIG_ENDIAN
)
3980 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3982 word
= bitpos
/ BITS_PER_WORD
;
3983 bitpos
= bitpos
% BITS_PER_WORD
;
3984 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3987 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3992 || reg_overlap_mentioned_p (target
, op0
)
3993 || reg_overlap_mentioned_p (target
, op1
)
3994 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3995 target
= gen_reg_rtx (mode
);
4001 for (i
= 0; i
< nwords
; ++i
)
4003 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
4004 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
4010 = expand_binop (imode
, and_optab
, op0_piece
,
4011 immed_wide_int_const (~mask
, imode
),
4012 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4013 op1
= expand_binop (imode
, and_optab
,
4014 operand_subword_force (op1
, i
, mode
),
4015 immed_wide_int_const (mask
, imode
),
4016 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4018 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
4019 targ_piece
, 1, OPTAB_LIB_WIDEN
);
4020 if (temp
!= targ_piece
)
4021 emit_move_insn (targ_piece
, temp
);
4024 emit_move_insn (targ_piece
, op0_piece
);
4027 insns
= get_insns ();
4034 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
4035 immed_wide_int_const (mask
, imode
),
4036 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4038 op0
= gen_lowpart (imode
, op0
);
4040 op0
= expand_binop (imode
, and_optab
, op0
,
4041 immed_wide_int_const (~mask
, imode
),
4042 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4044 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
4045 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
4046 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
4052 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
4053 scalar floating point mode. Return NULL if we do not know how to
4054 expand the operation inline. */
4057 expand_copysign (rtx op0
, rtx op1
, rtx target
)
4059 scalar_float_mode mode
;
4060 const struct real_format
*fmt
;
4064 mode
= as_a
<scalar_float_mode
> (GET_MODE (op0
));
4065 gcc_assert (GET_MODE (op1
) == mode
);
4067 /* First try to do it with a special instruction. */
4068 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
4069 target
, 0, OPTAB_DIRECT
);
4073 fmt
= REAL_MODE_FORMAT (mode
);
4074 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
4078 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
4080 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
4081 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
4085 if (fmt
->signbit_ro
>= 0
4086 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
4087 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
4088 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
4090 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
4091 fmt
->signbit_ro
, op0_is_abs
);
4096 if (fmt
->signbit_rw
< 0)
4098 return expand_copysign_bit (mode
, op0
, op1
, target
,
4099 fmt
->signbit_rw
, op0_is_abs
);
4102 /* Generate an instruction whose insn-code is INSN_CODE,
4103 with two operands: an output TARGET and an input OP0.
4104 TARGET *must* be nonzero, and the output is always stored there.
4105 CODE is an rtx code such that (CODE OP0) is an rtx that describes
4106 the value that is stored into TARGET.
4108 Return false if expansion failed. */
4111 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
4114 class expand_operand ops
[2];
4117 create_output_operand (&ops
[0], target
, GET_MODE (target
));
4118 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
4119 pat
= maybe_gen_insn (icode
, 2, ops
);
4123 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
4125 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
,
4130 if (ops
[0].value
!= target
)
4131 emit_move_insn (target
, ops
[0].value
);
4134 /* Generate an instruction whose insn-code is INSN_CODE,
4135 with two operands: an output TARGET and an input OP0.
4136 TARGET *must* be nonzero, and the output is always stored there.
4137 CODE is an rtx code such that (CODE OP0) is an rtx that describes
4138 the value that is stored into TARGET. */
4141 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
4143 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
4147 struct no_conflict_data
4150 rtx_insn
*first
, *insn
;
4154 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
4155 the currently examined clobber / store has to stay in the list of
4156 insns that constitute the actual libcall block. */
4158 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
4160 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
4162 /* If this inns directly contributes to setting the target, it must stay. */
4163 if (reg_overlap_mentioned_p (p
->target
, dest
))
4164 p
->must_stay
= true;
4165 /* If we haven't committed to keeping any other insns in the list yet,
4166 there is nothing more to check. */
4167 else if (p
->insn
== p
->first
)
4169 /* If this insn sets / clobbers a register that feeds one of the insns
4170 already in the list, this insn has to stay too. */
4171 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
4172 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
4173 || reg_used_between_p (dest
, p
->first
, p
->insn
)
4174 /* Likewise if this insn depends on a register set by a previous
4175 insn in the list, or if it sets a result (presumably a hard
4176 register) that is set or clobbered by a previous insn.
4177 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4178 SET_DEST perform the former check on the address, and the latter
4179 check on the MEM. */
4180 || (GET_CODE (set
) == SET
4181 && (modified_in_p (SET_SRC (set
), p
->first
)
4182 || modified_in_p (SET_DEST (set
), p
->first
)
4183 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
4184 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
4185 p
->must_stay
= true;
4189 /* Emit code to make a call to a constant function or a library call.
4191 INSNS is a list containing all insns emitted in the call.
4192 These insns leave the result in RESULT. Our block is to copy RESULT
4193 to TARGET, which is logically equivalent to EQUIV.
4195 We first emit any insns that set a pseudo on the assumption that these are
4196 loading constants into registers; doing so allows them to be safely cse'ed
4197 between blocks. Then we emit all the other insns in the block, followed by
4198 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4199 note with an operand of EQUIV. */
4202 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
4203 bool equiv_may_trap
)
4205 rtx final_dest
= target
;
4206 rtx_insn
*next
, *last
, *insn
;
4208 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4209 into a MEM later. Protect the libcall block from this change. */
4210 if (! REG_P (target
) || REG_USERVAR_P (target
))
4211 target
= gen_reg_rtx (GET_MODE (target
));
4213 /* If we're using non-call exceptions, a libcall corresponding to an
4214 operation that may trap may also trap. */
4215 /* ??? See the comment in front of make_reg_eh_region_note. */
4216 if (cfun
->can_throw_non_call_exceptions
4217 && (equiv_may_trap
|| may_trap_p (equiv
)))
4219 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
4222 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
4225 int lp_nr
= INTVAL (XEXP (note
, 0));
4226 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
4227 remove_note (insn
, note
);
4233 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4234 reg note to indicate that this call cannot throw or execute a nonlocal
4235 goto (unless there is already a REG_EH_REGION note, in which case
4237 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
4239 make_reg_eh_region_note_nothrow_nononlocal (insn
);
4242 /* First emit all insns that set pseudos. Remove them from the list as
4243 we go. Avoid insns that set pseudos which were referenced in previous
4244 insns. These can be generated by move_by_pieces, for example,
4245 to update an address. Similarly, avoid insns that reference things
4246 set in previous insns. */
4248 for (insn
= insns
; insn
; insn
= next
)
4250 rtx set
= single_set (insn
);
4252 next
= NEXT_INSN (insn
);
4254 if (set
!= 0 && REG_P (SET_DEST (set
))
4255 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
4257 struct no_conflict_data data
;
4259 data
.target
= const0_rtx
;
4263 note_stores (insn
, no_conflict_move_test
, &data
);
4264 if (! data
.must_stay
)
4266 if (PREV_INSN (insn
))
4267 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
4272 SET_PREV_INSN (next
) = PREV_INSN (insn
);
4278 /* Some ports use a loop to copy large arguments onto the stack.
4279 Don't move anything outside such a loop. */
4284 /* Write the remaining insns followed by the final copy. */
4285 for (insn
= insns
; insn
; insn
= next
)
4287 next
= NEXT_INSN (insn
);
4292 last
= emit_move_insn (target
, result
);
4294 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
4296 if (final_dest
!= target
)
4297 emit_move_insn (final_dest
, target
);
4301 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
4303 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
4306 /* True if we can perform a comparison of mode MODE straightforwardly.
4307 PURPOSE describes how this comparison will be used. CODE is the rtx
4308 comparison code we will be using.
4310 ??? Actually, CODE is slightly weaker than that. A target is still
4311 required to implement all of the normal bcc operations, but not
4312 required to implement all (or any) of the unordered bcc operations. */
4315 can_compare_p (enum rtx_code code
, machine_mode mode
,
4316 enum can_compare_purpose purpose
)
4319 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
4322 enum insn_code icode
;
4324 if (purpose
== ccp_jump
4325 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
4326 && insn_operand_matches (icode
, 0, test
))
4328 if (purpose
== ccp_store_flag
4329 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
4330 && insn_operand_matches (icode
, 1, test
))
4332 if (purpose
== ccp_cmov
4333 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
4336 mode
= GET_MODE_WIDER_MODE (mode
).else_void ();
4337 PUT_MODE (test
, mode
);
4339 while (mode
!= VOIDmode
);
4344 /* Return whether RTL code CODE corresponds to an unsigned optab. */
4347 unsigned_optab_p (enum rtx_code code
)
4349 return code
== LTU
|| code
== LEU
|| code
== GTU
|| code
== GEU
;
4352 /* Return whether the backend-emitted comparison for code CODE, comparing
4353 operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4354 operand OPNO of pattern ICODE. */
4357 insn_predicate_matches_p (enum insn_code icode
, unsigned int opno
,
4358 enum rtx_code code
, machine_mode mask_mode
,
4359 machine_mode value_mode
)
4361 rtx reg1
= alloca_raw_REG (value_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4362 rtx reg2
= alloca_raw_REG (value_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4363 rtx test
= alloca_rtx_fmt_ee (code
, mask_mode
, reg1
, reg2
);
4364 return insn_operand_matches (icode
, opno
, test
);
4367 /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4368 for code CODE, comparing operands of mode VALUE_MODE and producing a result
4372 can_vec_cmp_compare_p (enum rtx_code code
, machine_mode value_mode
,
4373 machine_mode mask_mode
)
4375 enum insn_code icode
4376 = get_vec_cmp_icode (value_mode
, mask_mode
, unsigned_optab_p (code
));
4377 if (icode
== CODE_FOR_nothing
)
4380 return insn_predicate_matches_p (icode
, 1, code
, mask_mode
, value_mode
);
4383 /* Return whether the backend can emit a vector comparison (vcond/vcondu) for
4384 code CODE, comparing operands of mode CMP_OP_MODE and producing a result
4388 can_vcond_compare_p (enum rtx_code code
, machine_mode value_mode
,
4389 machine_mode cmp_op_mode
)
4391 enum insn_code icode
4392 = get_vcond_icode (value_mode
, cmp_op_mode
, unsigned_optab_p (code
));
4393 if (icode
== CODE_FOR_nothing
)
4396 return insn_predicate_matches_p (icode
, 3, code
, value_mode
, cmp_op_mode
);
4399 /* Return whether the backend can emit vector set instructions for inserting
4400 element into vector at variable index position. */
4403 can_vec_set_var_idx_p (machine_mode vec_mode
)
4405 if (!VECTOR_MODE_P (vec_mode
))
4408 machine_mode inner_mode
= GET_MODE_INNER (vec_mode
);
4410 rtx reg1
= alloca_raw_REG (vec_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4411 rtx reg2
= alloca_raw_REG (inner_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4413 enum insn_code icode
= optab_handler (vec_set_optab
, vec_mode
);
4415 const struct insn_data_d
*data
= &insn_data
[icode
];
4416 machine_mode idx_mode
= data
->operand
[2].mode
;
4418 rtx reg3
= alloca_raw_REG (idx_mode
, LAST_VIRTUAL_REGISTER
+ 3);
4420 return icode
!= CODE_FOR_nothing
&& insn_operand_matches (icode
, 0, reg1
)
4421 && insn_operand_matches (icode
, 1, reg2
)
4422 && insn_operand_matches (icode
, 2, reg3
);
4425 /* Return whether the backend can emit a vec_extract instruction with
4426 a non-constant index. */
4428 can_vec_extract_var_idx_p (machine_mode vec_mode
, machine_mode extr_mode
)
4430 if (!VECTOR_MODE_P (vec_mode
))
4433 rtx reg1
= alloca_raw_REG (extr_mode
, LAST_VIRTUAL_REGISTER
+ 1);
4434 rtx reg2
= alloca_raw_REG (vec_mode
, LAST_VIRTUAL_REGISTER
+ 2);
4436 enum insn_code icode
= convert_optab_handler (vec_extract_optab
,
4437 vec_mode
, extr_mode
);
4439 const struct insn_data_d
*data
= &insn_data
[icode
];
4440 machine_mode idx_mode
= data
->operand
[2].mode
;
4442 rtx reg3
= alloca_raw_REG (idx_mode
, LAST_VIRTUAL_REGISTER
+ 3);
4444 return icode
!= CODE_FOR_nothing
&& insn_operand_matches (icode
, 0, reg1
)
4445 && insn_operand_matches (icode
, 1, reg2
)
4446 && insn_operand_matches (icode
, 2, reg3
);
4449 /* This function is called when we are going to emit a compare instruction that
4450 compares the values found in X and Y, using the rtl operator COMPARISON.
4452 If they have mode BLKmode, then SIZE specifies the size of both operands.
4454 UNSIGNEDP nonzero says that the operands are unsigned;
4455 this matters if they need to be widened (as given by METHODS).
4457 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4458 if we failed to produce one.
4460 *PMODE is the mode of the inputs (in case they are const_int).
4462 This function performs all the setup necessary so that the caller only has
4463 to emit a single comparison insn. This setup can involve doing a BLKmode
4464 comparison or emitting a library call to perform the comparison if no insn
4465 is available to handle it.
4466 The values which are passed in through pointers can be modified; the caller
4467 should perform the comparison on the modified values. Constant
4468 comparisons must have already been folded. */
4471 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4472 int unsignedp
, enum optab_methods methods
,
4473 rtx
*ptest
, machine_mode
*pmode
)
4475 machine_mode mode
= *pmode
;
4477 machine_mode cmp_mode
;
4479 /* The other methods are not needed. */
4480 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
4481 || methods
== OPTAB_LIB_WIDEN
);
4483 if (CONST_SCALAR_INT_P (y
))
4484 canonicalize_comparison (mode
, &comparison
, &y
);
4486 /* If we are optimizing, force expensive constants into a register. */
4487 if (CONSTANT_P (x
) && optimize
4488 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
4489 > COSTS_N_INSNS (1))
4490 && can_create_pseudo_p ())
4491 x
= force_reg (mode
, x
);
4493 if (CONSTANT_P (y
) && optimize
4494 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
4495 > COSTS_N_INSNS (1))
4496 && can_create_pseudo_p ())
4497 y
= force_reg (mode
, y
);
4499 /* Don't let both operands fail to indicate the mode. */
4500 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
4501 x
= force_reg (mode
, x
);
4502 if (mode
== VOIDmode
)
4503 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
4505 /* Handle all BLKmode compares. */
4507 if (mode
== BLKmode
)
4509 machine_mode result_mode
;
4510 enum insn_code cmp_code
;
4513 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
4517 /* Try to use a memory block compare insn - either cmpstr
4518 or cmpmem will do. */
4519 opt_scalar_int_mode cmp_mode_iter
;
4520 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
4522 scalar_int_mode cmp_mode
= cmp_mode_iter
.require ();
4523 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
4524 if (cmp_code
== CODE_FOR_nothing
)
4525 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
4526 if (cmp_code
== CODE_FOR_nothing
)
4527 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
4528 if (cmp_code
== CODE_FOR_nothing
)
4531 /* Must make sure the size fits the insn's mode. */
4532 if (CONST_INT_P (size
)
4533 ? UINTVAL (size
) > GET_MODE_MASK (cmp_mode
)
4534 : (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (size
)))
4535 > GET_MODE_BITSIZE (cmp_mode
)))
4538 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
4539 result
= gen_reg_rtx (result_mode
);
4540 size
= convert_to_mode (cmp_mode
, size
, 1);
4541 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
4543 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
4544 *pmode
= result_mode
;
4548 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
4551 /* Otherwise call a library function. */
4552 result
= emit_block_comp_via_libcall (x
, y
, size
);
4556 mode
= TYPE_MODE (integer_type_node
);
4557 methods
= OPTAB_LIB_WIDEN
;
4561 /* Don't allow operands to the compare to trap, as that can put the
4562 compare and branch in different basic blocks. */
4563 if (cfun
->can_throw_non_call_exceptions
)
4565 if (!can_create_pseudo_p () && (may_trap_p (x
) || may_trap_p (y
)))
4568 x
= copy_to_reg (x
);
4570 y
= copy_to_reg (y
);
4573 if (GET_MODE_CLASS (mode
) == MODE_CC
)
4575 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
4576 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4577 if (icode
!= CODE_FOR_nothing
4578 && insn_operand_matches (icode
, 0, test
))
4587 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4588 FOR_EACH_WIDER_MODE_FROM (cmp_mode
, mode
)
4590 enum insn_code icode
;
4591 icode
= optab_handler (cbranch_optab
, cmp_mode
);
4592 if (icode
!= CODE_FOR_nothing
4593 && insn_operand_matches (icode
, 0, test
))
4595 rtx_insn
*last
= get_last_insn ();
4596 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
4597 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
4599 && insn_operand_matches (icode
, 1, op0
)
4600 && insn_operand_matches (icode
, 2, op1
))
4602 XEXP (test
, 0) = op0
;
4603 XEXP (test
, 1) = op1
;
4608 delete_insns_since (last
);
4611 if (methods
== OPTAB_DIRECT
)
4615 if (methods
!= OPTAB_LIB_WIDEN
)
4618 if (SCALAR_FLOAT_MODE_P (mode
))
4620 /* Small trick if UNORDERED isn't implemented by the hardware. */
4621 if (comparison
== UNORDERED
&& rtx_equal_p (x
, y
))
4623 prepare_cmp_insn (x
, y
, UNLT
, NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4629 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
4634 machine_mode ret_mode
;
4636 /* Handle a libcall just for the mode we are using. */
4637 libfunc
= optab_libfunc (cmp_optab
, mode
);
4638 gcc_assert (libfunc
);
4640 /* If we want unsigned, and this mode has a distinct unsigned
4641 comparison routine, use that. */
4644 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
4649 ret_mode
= targetm
.libgcc_cmp_return_mode ();
4650 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4651 ret_mode
, x
, mode
, y
, mode
);
4653 /* There are two kinds of comparison routines. Biased routines
4654 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4655 of gcc expect that the comparison operation is equivalent
4656 to the modified comparison. For signed comparisons compare the
4657 result against 1 in the biased case, and zero in the unbiased
4658 case. For unsigned comparisons always compare against 1 after
4659 biasing the unbiased result by adding 1. This gives us a way to
4661 The comparisons in the fixed-point helper library are always
4666 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
4669 x
= plus_constant (ret_mode
, result
, 1);
4675 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
4685 /* Before emitting an insn with code ICODE, make sure that X, which is going
4686 to be used for operand OPNUM of the insn, is converted from mode MODE to
4687 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4688 that it is accepted by the operand predicate. Return the new value. */
4691 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
4692 machine_mode wider_mode
, int unsignedp
)
4694 if (mode
!= wider_mode
)
4695 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
4697 if (!insn_operand_matches (icode
, opnum
, x
))
4699 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
4700 if (reload_completed
)
4702 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
4704 x
= copy_to_mode_reg (op_mode
, x
);
4710 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4711 we can do the branch. */
4714 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
4715 direct_optab cmp_optab
, profile_probability prob
,
4718 machine_mode optab_mode
;
4719 enum mode_class mclass
;
4720 enum insn_code icode
;
4723 mclass
= GET_MODE_CLASS (mode
);
4724 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
4725 icode
= optab_handler (cmp_optab
, optab_mode
);
4727 gcc_assert (icode
!= CODE_FOR_nothing
);
4728 gcc_assert (test_branch
|| insn_operand_matches (icode
, 0, test
));
4730 insn
= emit_jump_insn (GEN_FCN (icode
) (XEXP (test
, 0),
4731 XEXP (test
, 1), label
));
4733 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
4734 XEXP (test
, 1), label
));
4736 if (prob
.initialized_p ()
4737 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4740 && any_condjump_p (insn
)
4741 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4742 add_reg_br_prob_note (insn
, prob
);
4745 /* PTEST points to a comparison that compares its first operand with zero.
4746 Check to see if it can be performed as a bit-test-and-branch instead.
4747 On success, return the instruction that performs the bit-test-and-branch
4748 and replace the second operand of *PTEST with the bit number to test.
4749 On failure, return CODE_FOR_nothing and leave *PTEST unchanged.
4751 Note that the comparison described by *PTEST should not be taken
4752 literally after a successful return. *PTEST is just a convenient
4753 place to store the two operands of the bit-and-test.
4755 VAL must contain the original tree expression for the first operand
4758 static enum insn_code
4759 validate_test_and_branch (tree val
, rtx
*ptest
, machine_mode
*pmode
, optab
*res
)
4761 if (!val
|| TREE_CODE (val
) != SSA_NAME
)
4762 return CODE_FOR_nothing
;
4764 machine_mode mode
= TYPE_MODE (TREE_TYPE (val
));
4768 if (GET_CODE (test
) == EQ
)
4769 optab
= tbranch_eq_optab
;
4770 else if (GET_CODE (test
) == NE
)
4771 optab
= tbranch_ne_optab
;
4773 return CODE_FOR_nothing
;
4777 /* If the target supports the testbit comparison directly, great. */
4778 auto icode
= direct_optab_handler (optab
, mode
);
4779 if (icode
== CODE_FOR_nothing
)
4782 if (tree_zero_one_valued_p (val
))
4784 auto pos
= BITS_BIG_ENDIAN
? GET_MODE_BITSIZE (mode
) - 1 : 0;
4785 XEXP (test
, 1) = gen_int_mode (pos
, mode
);
4791 wide_int wcst
= get_nonzero_bits (val
);
4793 return CODE_FOR_nothing
;
4797 if ((bitpos
= wi::exact_log2 (wcst
)) == -1)
4798 return CODE_FOR_nothing
;
4800 auto pos
= BITS_BIG_ENDIAN
? GET_MODE_BITSIZE (mode
) - 1 - bitpos
: bitpos
;
4801 XEXP (test
, 1) = gen_int_mode (pos
, mode
);
4807 /* Generate code to compare X with Y so that the condition codes are
4808 set and to jump to LABEL if the condition is true. If X is a
4809 constant and Y is not a constant, then the comparison is swapped to
4810 ensure that the comparison RTL has the canonical form.
4812 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4813 need to be widened. UNSIGNEDP is also used to select the proper
4814 branch condition code.
4816 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4818 MODE is the mode of the inputs (in case they are const_int).
4820 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4821 It will be potentially converted into an unsigned variant based on
4822 UNSIGNEDP to select a proper jump instruction.
4824 PROB is the probability of jumping to LABEL. If the comparison is against
4825 zero then VAL contains the expression from which the non-zero RTL is
4829 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4830 machine_mode mode
, int unsignedp
, tree val
, rtx label
,
4831 profile_probability prob
)
4833 rtx op0
= x
, op1
= y
;
4836 /* Swap operands and condition to ensure canonical RTL. */
4837 if (swap_commutative_operands_p (x
, y
)
4838 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4841 comparison
= swap_condition (comparison
);
4844 /* If OP0 is still a constant, then both X and Y must be constants
4845 or the opposite comparison is not supported. Force X into a register
4846 to create canonical RTL. */
4847 if (CONSTANT_P (op0
))
4848 op0
= force_reg (mode
, op0
);
4851 comparison
= unsigned_condition (comparison
);
4853 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4856 /* Check if we're comparing a truth type with 0, and if so check if
4857 the target supports tbranch. */
4858 machine_mode tmode
= mode
;
4860 if (op1
== CONST0_RTX (GET_MODE (op1
))
4861 && validate_test_and_branch (val
, &test
, &tmode
,
4862 &optab
) != CODE_FOR_nothing
)
4864 emit_cmp_and_jump_insn_1 (test
, tmode
, label
, optab
, prob
, true);
4868 emit_cmp_and_jump_insn_1 (test
, mode
, label
, cbranch_optab
, prob
, false);
4871 /* Overloaded version of emit_cmp_and_jump_insns in which VAL is unknown. */
4874 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4875 machine_mode mode
, int unsignedp
, rtx label
,
4876 profile_probability prob
)
4878 emit_cmp_and_jump_insns (x
, y
, comparison
, size
, mode
, unsignedp
, NULL
,
4883 /* Emit a library call comparison between floating point X and Y.
4884 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4887 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4888 rtx
*ptest
, machine_mode
*pmode
)
4890 enum rtx_code swapped
= swap_condition (comparison
);
4891 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4892 machine_mode orig_mode
= GET_MODE (x
);
4894 rtx true_rtx
, false_rtx
;
4895 rtx value
, target
, equiv
;
4898 bool reversed_p
= false;
4899 scalar_int_mode cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4901 FOR_EACH_WIDER_MODE_FROM (mode
, orig_mode
)
4903 if (code_to_optab (comparison
)
4904 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4907 if (code_to_optab (swapped
)
4908 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4911 comparison
= swapped
;
4915 if (code_to_optab (reversed
)
4916 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4918 comparison
= reversed
;
4924 gcc_assert (mode
!= VOIDmode
);
4926 if (mode
!= orig_mode
)
4928 x
= convert_to_mode (mode
, x
, 0);
4929 y
= convert_to_mode (mode
, y
, 0);
4932 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4933 the RTL. The allows the RTL optimizers to delete the libcall if the
4934 condition can be determined at compile-time. */
4935 if (comparison
== UNORDERED
4936 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4938 true_rtx
= const_true_rtx
;
4939 false_rtx
= const0_rtx
;
4946 true_rtx
= const0_rtx
;
4947 false_rtx
= const_true_rtx
;
4951 true_rtx
= const_true_rtx
;
4952 false_rtx
= const0_rtx
;
4956 true_rtx
= const1_rtx
;
4957 false_rtx
= const0_rtx
;
4961 true_rtx
= const0_rtx
;
4962 false_rtx
= constm1_rtx
;
4966 true_rtx
= constm1_rtx
;
4967 false_rtx
= const0_rtx
;
4971 true_rtx
= const0_rtx
;
4972 false_rtx
= const1_rtx
;
4980 if (comparison
== UNORDERED
)
4982 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4983 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4984 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4985 temp
, const_true_rtx
, equiv
);
4989 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4990 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4991 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4992 equiv
, true_rtx
, false_rtx
);
4996 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4997 cmp_mode
, x
, mode
, y
, mode
);
4998 insns
= get_insns ();
5001 target
= gen_reg_rtx (cmp_mode
);
5002 emit_libcall_block (insns
, target
, value
, equiv
);
5004 if (comparison
== UNORDERED
5005 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
5007 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
5009 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
5014 /* Generate code to indirectly jump to a location given in the rtx LOC. */
5017 emit_indirect_jump (rtx loc
)
5019 if (!targetm
.have_indirect_jump ())
5020 sorry ("indirect jumps are not available on this target");
5023 class expand_operand ops
[1];
5024 create_address_operand (&ops
[0], loc
);
5025 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
5031 /* Emit a conditional move instruction if the machine supports one for that
5032 condition and machine mode.
5034 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5035 the mode to use should they be constants. If it is VOIDmode, they cannot
5038 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
5039 should be stored there. MODE is the mode to use should they be constants.
5040 If it is VOIDmode, they cannot both be constants.
5042 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5043 is not supported. */
5046 emit_conditional_move (rtx target
, struct rtx_comparison comp
,
5048 machine_mode mode
, int unsignedp
)
5052 enum insn_code icode
;
5053 enum rtx_code reversed
;
5055 /* If the two source operands are identical, that's just a move. */
5057 if (rtx_equal_p (op2
, op3
))
5060 target
= gen_reg_rtx (mode
);
5062 emit_move_insn (target
, op3
);
5066 /* If one operand is constant, make it the second one. Only do this
5067 if the other operand is not constant as well. */
5069 if (swap_commutative_operands_p (comp
.op0
, comp
.op1
))
5071 std::swap (comp
.op0
, comp
.op1
);
5072 comp
.code
= swap_condition (comp
.code
);
5075 /* get_condition will prefer to generate LT and GT even if the old
5076 comparison was against zero, so undo that canonicalization here since
5077 comparisons against zero are cheaper. */
5079 if (comp
.code
== LT
&& comp
.op1
== const1_rtx
)
5080 comp
.code
= LE
, comp
.op1
= const0_rtx
;
5081 else if (comp
.code
== GT
&& comp
.op1
== constm1_rtx
)
5082 comp
.code
= GE
, comp
.op1
= const0_rtx
;
5084 if (comp
.mode
== VOIDmode
)
5085 comp
.mode
= GET_MODE (comp
.op0
);
5087 enum rtx_code orig_code
= comp
.code
;
5088 bool swapped
= false;
5089 if (swap_commutative_operands_p (op2
, op3
)
5091 reversed_comparison_code_parts (comp
.code
, comp
.op0
, comp
.op1
, NULL
))
5094 std::swap (op2
, op3
);
5095 comp
.code
= reversed
;
5099 if (mode
== VOIDmode
)
5100 mode
= GET_MODE (op2
);
5102 icode
= direct_optab_handler (movcc_optab
, mode
);
5104 if (icode
== CODE_FOR_nothing
)
5108 target
= gen_reg_rtx (mode
);
5110 for (int pass
= 0; ; pass
++)
5112 comp
.code
= unsignedp
? unsigned_condition (comp
.code
) : comp
.code
;
5114 simplify_gen_relational (comp
.code
, VOIDmode
,
5115 comp
.mode
, comp
.op0
, comp
.op1
);
5117 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5118 punt and let the caller figure out how best to deal with this
5120 if (COMPARISON_P (comparison
))
5122 saved_pending_stack_adjust save
;
5123 save_pending_stack_adjust (&save
);
5124 last
= get_last_insn ();
5125 do_pending_stack_adjust ();
5126 machine_mode cmpmode
= comp
.mode
;
5127 rtx orig_op0
= XEXP (comparison
, 0);
5128 rtx orig_op1
= XEXP (comparison
, 1);
5131 /* If we are optimizing, force expensive constants into a register
5132 but preserve an eventual equality with op2/op3. */
5133 if (CONSTANT_P (orig_op0
) && optimize
5134 && (rtx_cost (orig_op0
, mode
, COMPARE
, 0,
5135 optimize_insn_for_speed_p ())
5136 > COSTS_N_INSNS (1))
5137 && can_create_pseudo_p ())
5139 if (rtx_equal_p (orig_op0
, op2
))
5140 op2p
= XEXP (comparison
, 0) = force_reg (cmpmode
, orig_op0
);
5141 else if (rtx_equal_p (orig_op0
, op3
))
5142 op3p
= XEXP (comparison
, 0) = force_reg (cmpmode
, orig_op0
);
5144 if (CONSTANT_P (orig_op1
) && optimize
5145 && (rtx_cost (orig_op1
, mode
, COMPARE
, 0,
5146 optimize_insn_for_speed_p ())
5147 > COSTS_N_INSNS (1))
5148 && can_create_pseudo_p ())
5150 if (rtx_equal_p (orig_op1
, op2
))
5151 op2p
= XEXP (comparison
, 1) = force_reg (cmpmode
, orig_op1
);
5152 else if (rtx_equal_p (orig_op1
, op3
))
5153 op3p
= XEXP (comparison
, 1) = force_reg (cmpmode
, orig_op1
);
5155 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
5156 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
5157 OPTAB_WIDEN
, &comparison
, &cmpmode
);
5160 rtx res
= emit_conditional_move_1 (target
, comparison
,
5162 if (res
!= NULL_RTX
)
5165 delete_insns_since (last
);
5166 restore_pending_stack_adjust (&save
);
5172 /* If the preferred op2/op3 order is not usable, retry with other
5173 operand order, perhaps it will expand successfully. */
5175 comp
.code
= orig_code
;
5176 else if ((reversed
=
5177 reversed_comparison_code_parts (orig_code
, comp
.op0
, comp
.op1
,
5180 comp
.code
= reversed
;
5183 std::swap (op2
, op3
);
5187 /* Helper function that, in addition to COMPARISON, also tries
5188 the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
5189 to when we pass the specific constituents of a comparison, no
5190 additional insns are emitted for it. It might still be necessary
5191 to emit more than one insn for the final conditional move, though. */
5194 emit_conditional_move (rtx target
, rtx comparison
, rtx rev_comparison
,
5195 rtx op2
, rtx op3
, machine_mode mode
)
5197 rtx res
= emit_conditional_move_1 (target
, comparison
, op2
, op3
, mode
);
5199 if (res
!= NULL_RTX
)
5202 return emit_conditional_move_1 (target
, rev_comparison
, op3
, op2
, mode
);
5205 /* Helper for emitting a conditional move. */
5208 emit_conditional_move_1 (rtx target
, rtx comparison
,
5209 rtx op2
, rtx op3
, machine_mode mode
)
5211 enum insn_code icode
;
5213 if (comparison
== NULL_RTX
|| !COMPARISON_P (comparison
))
5216 /* If the two source operands are identical, that's just a move.
5217 As the comparison comes in non-canonicalized, we must make
5218 sure not to discard any possible side effects. If there are
5219 side effects, just let the target handle it. */
5220 if (!side_effects_p (comparison
) && rtx_equal_p (op2
, op3
))
5223 target
= gen_reg_rtx (mode
);
5225 emit_move_insn (target
, op3
);
5229 if (mode
== VOIDmode
)
5230 mode
= GET_MODE (op2
);
5232 icode
= direct_optab_handler (movcc_optab
, mode
);
5234 if (icode
== CODE_FOR_nothing
)
5238 target
= gen_reg_rtx (mode
);
5240 class expand_operand ops
[4];
5242 create_output_operand (&ops
[0], target
, mode
);
5243 create_fixed_operand (&ops
[1], comparison
);
5244 create_input_operand (&ops
[2], op2
, mode
);
5245 create_input_operand (&ops
[3], op3
, mode
);
5247 if (maybe_expand_insn (icode
, 4, ops
))
5249 if (ops
[0].value
!= target
)
5250 convert_move (target
, ops
[0].value
, false);
5258 /* Emit a conditional negate or bitwise complement using the
5259 negcc or notcc optabs if available. Return NULL_RTX if such operations
5260 are not available. Otherwise return the RTX holding the result.
5261 TARGET is the desired destination of the result. COMP is the comparison
5262 on which to negate. If COND is true move into TARGET the negation
5263 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5264 CODE is either NEG or NOT. MODE is the machine mode in which the
5265 operation is performed. */
5268 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
5269 machine_mode mode
, rtx cond
, rtx op1
,
5272 optab op
= unknown_optab
;
5275 else if (code
== NOT
)
5280 insn_code icode
= direct_optab_handler (op
, mode
);
5282 if (icode
== CODE_FOR_nothing
)
5286 target
= gen_reg_rtx (mode
);
5288 rtx_insn
*last
= get_last_insn ();
5289 class expand_operand ops
[4];
5291 create_output_operand (&ops
[0], target
, mode
);
5292 create_fixed_operand (&ops
[1], cond
);
5293 create_input_operand (&ops
[2], op1
, mode
);
5294 create_input_operand (&ops
[3], op2
, mode
);
5296 if (maybe_expand_insn (icode
, 4, ops
))
5298 if (ops
[0].value
!= target
)
5299 convert_move (target
, ops
[0].value
, false);
5303 delete_insns_since (last
);
5307 /* Emit a conditional addition instruction if the machine supports one for that
5308 condition and machine mode.
5310 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5311 the mode to use should they be constants. If it is VOIDmode, they cannot
5314 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5315 should be stored there. MODE is the mode to use should they be constants.
5316 If it is VOIDmode, they cannot both be constants.
5318 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5319 is not supported. */
5322 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
5323 machine_mode cmode
, rtx op2
, rtx op3
,
5324 machine_mode mode
, int unsignedp
)
5328 enum insn_code icode
;
5330 /* If one operand is constant, make it the second one. Only do this
5331 if the other operand is not constant as well. */
5333 if (swap_commutative_operands_p (op0
, op1
))
5335 std::swap (op0
, op1
);
5336 code
= swap_condition (code
);
5339 /* get_condition will prefer to generate LT and GT even if the old
5340 comparison was against zero, so undo that canonicalization here since
5341 comparisons against zero are cheaper. */
5342 if (code
== LT
&& op1
== const1_rtx
)
5343 code
= LE
, op1
= const0_rtx
;
5344 else if (code
== GT
&& op1
== constm1_rtx
)
5345 code
= GE
, op1
= const0_rtx
;
5347 if (cmode
== VOIDmode
)
5348 cmode
= GET_MODE (op0
);
5350 if (mode
== VOIDmode
)
5351 mode
= GET_MODE (op2
);
5353 icode
= optab_handler (addcc_optab
, mode
);
5355 if (icode
== CODE_FOR_nothing
)
5359 target
= gen_reg_rtx (mode
);
5361 code
= unsignedp
? unsigned_condition (code
) : code
;
5362 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
5364 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5365 return NULL and let the caller figure out how best to deal with this
5367 if (!COMPARISON_P (comparison
))
5370 do_pending_stack_adjust ();
5371 last
= get_last_insn ();
5372 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
5373 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
5374 &comparison
, &cmode
);
5377 class expand_operand ops
[4];
5379 create_output_operand (&ops
[0], target
, mode
);
5380 create_fixed_operand (&ops
[1], comparison
);
5381 create_input_operand (&ops
[2], op2
, mode
);
5382 create_input_operand (&ops
[3], op3
, mode
);
5383 if (maybe_expand_insn (icode
, 4, ops
))
5385 if (ops
[0].value
!= target
)
5386 convert_move (target
, ops
[0].value
, false);
5390 delete_insns_since (last
);
5394 /* These functions attempt to generate an insn body, rather than
5395 emitting the insn, but if the gen function already emits them, we
5396 make no attempt to turn them back into naked patterns. */
5398 /* Generate and return an insn body to add Y to X. */
5401 gen_add2_insn (rtx x
, rtx y
)
5403 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
5405 gcc_assert (insn_operand_matches (icode
, 0, x
));
5406 gcc_assert (insn_operand_matches (icode
, 1, x
));
5407 gcc_assert (insn_operand_matches (icode
, 2, y
));
5409 return GEN_FCN (icode
) (x
, x
, y
);
5412 /* Generate and return an insn body to add r1 and c,
5413 storing the result in r0. */
5416 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
5418 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
5420 if (icode
== CODE_FOR_nothing
5421 || !insn_operand_matches (icode
, 0, r0
)
5422 || !insn_operand_matches (icode
, 1, r1
)
5423 || !insn_operand_matches (icode
, 2, c
))
5426 return GEN_FCN (icode
) (r0
, r1
, c
);
5430 have_add2_insn (rtx x
, rtx y
)
5432 enum insn_code icode
;
5434 gcc_assert (GET_MODE (x
) != VOIDmode
);
5436 icode
= optab_handler (add_optab
, GET_MODE (x
));
5438 if (icode
== CODE_FOR_nothing
)
5441 if (!insn_operand_matches (icode
, 0, x
)
5442 || !insn_operand_matches (icode
, 1, x
)
5443 || !insn_operand_matches (icode
, 2, y
))
5449 /* Generate and return an insn body to add Y to X. */
5452 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
5454 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
5456 gcc_assert (insn_operand_matches (icode
, 0, x
));
5457 gcc_assert (insn_operand_matches (icode
, 1, y
));
5458 gcc_assert (insn_operand_matches (icode
, 2, z
));
5460 return GEN_FCN (icode
) (x
, y
, z
);
5463 /* Return true if the target implements an addptr pattern and X, Y,
5464 and Z are valid for the pattern predicates. */
5467 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
5469 enum insn_code icode
;
5471 gcc_assert (GET_MODE (x
) != VOIDmode
);
5473 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
5475 if (icode
== CODE_FOR_nothing
)
5478 if (!insn_operand_matches (icode
, 0, x
)
5479 || !insn_operand_matches (icode
, 1, y
)
5480 || !insn_operand_matches (icode
, 2, z
))
5486 /* Generate and return an insn body to subtract Y from X. */
5489 gen_sub2_insn (rtx x
, rtx y
)
5491 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
5493 gcc_assert (insn_operand_matches (icode
, 0, x
));
5494 gcc_assert (insn_operand_matches (icode
, 1, x
));
5495 gcc_assert (insn_operand_matches (icode
, 2, y
));
5497 return GEN_FCN (icode
) (x
, x
, y
);
5500 /* Generate and return an insn body to subtract r1 and c,
5501 storing the result in r0. */
5504 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
5506 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
5508 if (icode
== CODE_FOR_nothing
5509 || !insn_operand_matches (icode
, 0, r0
)
5510 || !insn_operand_matches (icode
, 1, r1
)
5511 || !insn_operand_matches (icode
, 2, c
))
5514 return GEN_FCN (icode
) (r0
, r1
, c
);
5518 have_sub2_insn (rtx x
, rtx y
)
5520 enum insn_code icode
;
5522 gcc_assert (GET_MODE (x
) != VOIDmode
);
5524 icode
= optab_handler (sub_optab
, GET_MODE (x
));
5526 if (icode
== CODE_FOR_nothing
)
5529 if (!insn_operand_matches (icode
, 0, x
)
5530 || !insn_operand_matches (icode
, 1, x
)
5531 || !insn_operand_matches (icode
, 2, y
))
5537 /* Generate the body of an insn to extend Y (with mode MFROM)
5538 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5541 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
5542 machine_mode mfrom
, int unsignedp
)
5544 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
5545 return GEN_FCN (icode
) (x
, y
);
5548 /* Generate code to convert FROM to floating point
5549 and store in TO. FROM must be fixed point and not VOIDmode.
5550 UNSIGNEDP nonzero means regard FROM as unsigned.
5551 Normally this is done by correcting the final value
5552 if it is negative. */
5555 expand_float (rtx to
, rtx from
, int unsignedp
)
5557 enum insn_code icode
;
5559 scalar_mode from_mode
, to_mode
;
5560 machine_mode fmode
, imode
;
5561 bool can_do_signed
= false;
5563 /* Crash now, because we won't be able to decide which mode to use. */
5564 gcc_assert (GET_MODE (from
) != VOIDmode
);
5566 /* Look for an insn to do the conversion. Do it in the specified
5567 modes if possible; otherwise convert either input, output or both to
5568 wider mode. If the integer mode is wider than the mode of FROM,
5569 we can do the conversion signed even if the input is unsigned. */
5571 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
5572 FOR_EACH_MODE_FROM (imode
, GET_MODE (from
))
5574 int doing_unsigned
= unsignedp
;
5576 if (fmode
!= GET_MODE (to
)
5577 && (significand_size (fmode
)
5578 < GET_MODE_UNIT_PRECISION (GET_MODE (from
))))
5581 icode
= can_float_p (fmode
, imode
, unsignedp
);
5582 if (icode
== CODE_FOR_nothing
&& unsignedp
)
5584 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
5585 if (scode
!= CODE_FOR_nothing
)
5586 can_do_signed
= true;
5587 if (imode
!= GET_MODE (from
))
5588 icode
= scode
, doing_unsigned
= 0;
5591 if (icode
!= CODE_FOR_nothing
)
5593 if (imode
!= GET_MODE (from
))
5594 from
= convert_to_mode (imode
, from
, unsignedp
);
5596 if (fmode
!= GET_MODE (to
))
5597 target
= gen_reg_rtx (fmode
);
5599 emit_unop_insn (icode
, target
, from
,
5600 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
5603 convert_move (to
, target
, 0);
5608 /* Unsigned integer, and no way to convert directly. Convert as signed,
5609 then unconditionally adjust the result. */
5612 && is_a
<scalar_mode
> (GET_MODE (to
), &to_mode
)
5613 && is_a
<scalar_mode
> (GET_MODE (from
), &from_mode
))
5615 opt_scalar_mode fmode_iter
;
5616 rtx_code_label
*label
= gen_label_rtx ();
5618 REAL_VALUE_TYPE offset
;
5620 /* Look for a usable floating mode FMODE wider than the source and at
5621 least as wide as the target. Using FMODE will avoid rounding woes
5622 with unsigned values greater than the signed maximum value. */
5624 FOR_EACH_MODE_FROM (fmode_iter
, to_mode
)
5626 scalar_mode fmode
= fmode_iter
.require ();
5627 if (GET_MODE_PRECISION (from_mode
) < GET_MODE_BITSIZE (fmode
)
5628 && can_float_p (fmode
, from_mode
, 0) != CODE_FOR_nothing
)
5632 if (!fmode_iter
.exists (&fmode
))
5634 /* There is no such mode. Pretend the target is wide enough. */
5637 /* Avoid double-rounding when TO is narrower than FROM. */
5638 if ((significand_size (fmode
) + 1)
5639 < GET_MODE_PRECISION (from_mode
))
5642 rtx_code_label
*neglabel
= gen_label_rtx ();
5644 /* Don't use TARGET if it isn't a register, is a hard register,
5645 or is the wrong mode. */
5647 || REGNO (target
) < FIRST_PSEUDO_REGISTER
5648 || GET_MODE (target
) != fmode
)
5649 target
= gen_reg_rtx (fmode
);
5652 do_pending_stack_adjust ();
5654 /* Test whether the sign bit is set. */
5655 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
5658 /* The sign bit is not set. Convert as signed. */
5659 expand_float (target
, from
, 0);
5660 emit_jump_insn (targetm
.gen_jump (label
));
5663 /* The sign bit is set.
5664 Convert to a usable (positive signed) value by shifting right
5665 one bit, while remembering if a nonzero bit was shifted
5666 out; i.e., compute (from & 1) | (from >> 1). */
5668 emit_label (neglabel
);
5669 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
5670 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
5671 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
5672 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
5674 expand_float (target
, temp
, 0);
5676 /* Multiply by 2 to undo the shift above. */
5677 temp
= expand_binop (fmode
, add_optab
, target
, target
,
5678 target
, 0, OPTAB_LIB_WIDEN
);
5680 emit_move_insn (target
, temp
);
5682 do_pending_stack_adjust ();
5688 /* If we are about to do some arithmetic to correct for an
5689 unsigned operand, do it in a pseudo-register. */
5691 if (to_mode
!= fmode
5692 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
5693 target
= gen_reg_rtx (fmode
);
5695 /* Convert as signed integer to floating. */
5696 expand_float (target
, from
, 0);
5698 /* If FROM is negative (and therefore TO is negative),
5699 correct its value by 2**bitwidth. */
5701 do_pending_stack_adjust ();
5702 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, from_mode
,
5706 real_2expN (&offset
, GET_MODE_PRECISION (from_mode
), fmode
);
5707 temp
= expand_binop (fmode
, add_optab
, target
,
5708 const_double_from_real_value (offset
, fmode
),
5709 target
, 0, OPTAB_LIB_WIDEN
);
5711 emit_move_insn (target
, temp
);
5713 do_pending_stack_adjust ();
5718 /* No hardware instruction available; call a library routine. */
5723 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
5725 if (is_narrower_int_mode (GET_MODE (from
), SImode
))
5726 from
= convert_to_mode (SImode
, from
, unsignedp
);
5728 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5729 gcc_assert (libfunc
);
5733 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5734 GET_MODE (to
), from
, GET_MODE (from
));
5735 insns
= get_insns ();
5738 emit_libcall_block (insns
, target
, value
,
5739 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
5740 GET_MODE (to
), from
));
5745 /* Copy result to requested destination
5746 if we have been computing in a temp location. */
5750 if (GET_MODE (target
) == GET_MODE (to
))
5751 emit_move_insn (to
, target
);
5753 convert_move (to
, target
, 0);
5757 /* Generate code to convert FROM to fixed point and store in TO. FROM
5758 must be floating point. */
5761 expand_fix (rtx to
, rtx from
, int unsignedp
)
5763 enum insn_code icode
;
5765 machine_mode fmode
, imode
;
5766 opt_scalar_mode fmode_iter
;
5767 bool must_trunc
= false;
5769 /* We first try to find a pair of modes, one real and one integer, at
5770 least as wide as FROM and TO, respectively, in which we can open-code
5771 this conversion. If the integer mode is wider than the mode of TO,
5772 we can do the conversion either signed or unsigned. */
5774 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5775 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5777 int doing_unsigned
= unsignedp
;
5779 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
5780 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
5781 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
5783 if (icode
!= CODE_FOR_nothing
)
5785 rtx_insn
*last
= get_last_insn ();
5787 if (fmode
!= GET_MODE (from
))
5789 if (REAL_MODE_FORMAT (GET_MODE (from
))
5790 == &arm_bfloat_half_format
5791 && REAL_MODE_FORMAT (fmode
) == &ieee_single_format
)
5792 /* The BF -> SF conversions can be just a shift, doesn't
5793 need to handle sNANs. */
5795 int save_flag_finite_math_only
= flag_finite_math_only
;
5796 flag_finite_math_only
= true;
5797 from1
= convert_to_mode (fmode
, from
, 0);
5798 flag_finite_math_only
= save_flag_finite_math_only
;
5801 from1
= convert_to_mode (fmode
, from
, 0);
5806 rtx temp
= gen_reg_rtx (GET_MODE (from1
));
5807 from1
= expand_unop (GET_MODE (from1
), ftrunc_optab
, from1
,
5811 if (imode
!= GET_MODE (to
))
5812 target
= gen_reg_rtx (imode
);
5814 if (maybe_emit_unop_insn (icode
, target
, from1
,
5815 doing_unsigned
? UNSIGNED_FIX
: FIX
))
5818 convert_move (to
, target
, unsignedp
);
5821 delete_insns_since (last
);
5825 /* For an unsigned conversion, there is one more way to do it.
5826 If we have a signed conversion, we generate code that compares
5827 the real value to the largest representable positive number. If if
5828 is smaller, the conversion is done normally. Otherwise, subtract
5829 one plus the highest signed number, convert, and add it back.
5831 We only need to check all real modes, since we know we didn't find
5832 anything with a wider integer mode.
5834 This code used to extend FP value into mode wider than the destination.
5835 This is needed for decimal float modes which cannot accurately
5836 represent one plus the highest signed number of the same size, but
5837 not for binary modes. Consider, for instance conversion from SFmode
5840 The hot path through the code is dealing with inputs smaller than 2^63
5841 and doing just the conversion, so there is no bits to lose.
5843 In the other path we know the value is positive in the range 2^63..2^64-1
5844 inclusive. (as for other input overflow happens and result is undefined)
5845 So we know that the most important bit set in mantissa corresponds to
5846 2^63. The subtraction of 2^63 should not generate any rounding as it
5847 simply clears out that bit. The rest is trivial. */
5849 scalar_int_mode to_mode
;
5851 && is_a
<scalar_int_mode
> (GET_MODE (to
), &to_mode
)
5852 && HWI_COMPUTABLE_MODE_P (to_mode
))
5853 FOR_EACH_MODE_FROM (fmode_iter
, as_a
<scalar_mode
> (GET_MODE (from
)))
5855 scalar_mode fmode
= fmode_iter
.require ();
5856 if (CODE_FOR_nothing
!= can_fix_p (to_mode
, fmode
,
5858 && (!DECIMAL_FLOAT_MODE_P (fmode
)
5859 || (GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (to_mode
))))
5862 REAL_VALUE_TYPE offset
;
5864 rtx_code_label
*lab1
, *lab2
;
5867 bitsize
= GET_MODE_PRECISION (to_mode
);
5868 real_2expN (&offset
, bitsize
- 1, fmode
);
5869 limit
= const_double_from_real_value (offset
, fmode
);
5870 lab1
= gen_label_rtx ();
5871 lab2
= gen_label_rtx ();
5873 if (fmode
!= GET_MODE (from
))
5875 if (REAL_MODE_FORMAT (GET_MODE (from
))
5876 == &arm_bfloat_half_format
5877 && REAL_MODE_FORMAT (fmode
) == &ieee_single_format
)
5878 /* The BF -> SF conversions can be just a shift, doesn't
5879 need to handle sNANs. */
5881 int save_flag_finite_math_only
= flag_finite_math_only
;
5882 flag_finite_math_only
= true;
5883 from
= convert_to_mode (fmode
, from
, 0);
5884 flag_finite_math_only
= save_flag_finite_math_only
;
5887 from
= convert_to_mode (fmode
, from
, 0);
5890 /* See if we need to do the subtraction. */
5891 do_pending_stack_adjust ();
5892 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
,
5893 GET_MODE (from
), 0, lab1
);
5895 /* If not, do the signed "fix" and branch around fixup code. */
5896 expand_fix (to
, from
, 0);
5897 emit_jump_insn (targetm
.gen_jump (lab2
));
5900 /* Otherwise, subtract 2**(N-1), convert to signed number,
5901 then add 2**(N-1). Do the addition using XOR since this
5902 will often generate better code. */
5904 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
5905 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
5906 expand_fix (to
, target
, 0);
5907 target
= expand_binop (to_mode
, xor_optab
, to
,
5909 (HOST_WIDE_INT_1
<< (bitsize
- 1),
5911 to
, 1, OPTAB_LIB_WIDEN
);
5914 emit_move_insn (to
, target
);
5918 if (optab_handler (mov_optab
, to_mode
) != CODE_FOR_nothing
)
5920 /* Make a place for a REG_NOTE and add it. */
5921 insn
= emit_move_insn (to
, to
);
5922 set_dst_reg_note (insn
, REG_EQUAL
,
5923 gen_rtx_fmt_e (UNSIGNED_FIX
, to_mode
,
5933 if (REAL_MODE_FORMAT (GET_MODE (from
)) == &arm_bfloat_half_format
5934 && REAL_MODE_FORMAT (SFmode
) == &ieee_single_format
)
5935 /* We don't have BF -> TI library functions, use BF -> SF -> TI
5936 instead but the BF -> SF conversion can be just a shift, doesn't
5937 need to handle sNANs. */
5939 int save_flag_finite_math_only
= flag_finite_math_only
;
5940 flag_finite_math_only
= true;
5941 from
= convert_to_mode (SFmode
, from
, 0);
5942 flag_finite_math_only
= save_flag_finite_math_only
;
5943 expand_fix (to
, from
, unsignedp
);
5948 /* We can't do it with an insn, so use a library call. But first ensure
5949 that the mode of TO is at least as wide as SImode, since those are the
5950 only library calls we know about. */
5952 if (is_narrower_int_mode (GET_MODE (to
), SImode
))
5954 target
= gen_reg_rtx (SImode
);
5956 expand_fix (target
, from
, unsignedp
);
5964 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
5965 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5966 gcc_assert (libfunc
);
5970 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5971 GET_MODE (to
), from
, GET_MODE (from
));
5972 insns
= get_insns ();
5975 emit_libcall_block (insns
, target
, value
,
5976 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5977 GET_MODE (to
), from
));
5982 if (GET_MODE (to
) == GET_MODE (target
))
5983 emit_move_insn (to
, target
);
5985 convert_move (to
, target
, 0);
5990 /* Promote integer arguments for a libcall if necessary.
5991 emit_library_call_value cannot do the promotion because it does not
5992 know if it should do a signed or unsigned promotion. This is because
5993 there are no tree types defined for libcalls. */
5996 prepare_libcall_arg (rtx arg
, int uintp
)
5998 scalar_int_mode mode
;
5999 machine_mode arg_mode
;
6000 if (is_a
<scalar_int_mode
> (GET_MODE (arg
), &mode
))
6002 /* If we need to promote the integer function argument we need to do
6003 it here instead of inside emit_library_call_value because in
6004 emit_library_call_value we don't know if we should do a signed or
6005 unsigned promotion. */
6008 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
6009 &unsigned_p
, NULL_TREE
, 0);
6010 if (arg_mode
!= mode
)
6011 return convert_to_mode (arg_mode
, arg
, uintp
);
6016 /* Generate code to convert FROM or TO a fixed-point.
6017 If UINTP is true, either TO or FROM is an unsigned integer.
6018 If SATP is true, we need to saturate the result. */
6021 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
6023 machine_mode to_mode
= GET_MODE (to
);
6024 machine_mode from_mode
= GET_MODE (from
);
6026 enum rtx_code this_code
;
6027 enum insn_code code
;
6032 if (to_mode
== from_mode
)
6034 emit_move_insn (to
, from
);
6040 tab
= satp
? satfractuns_optab
: fractuns_optab
;
6041 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
6045 tab
= satp
? satfract_optab
: fract_optab
;
6046 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
6048 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
6049 if (code
!= CODE_FOR_nothing
)
6051 emit_unop_insn (code
, to
, from
, this_code
);
6055 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
6056 gcc_assert (libfunc
);
6058 from
= prepare_libcall_arg (from
, uintp
);
6059 from_mode
= GET_MODE (from
);
6062 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
6064 insns
= get_insns ();
6067 emit_libcall_block (insns
, to
, value
,
6068 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
6071 /* Generate code to convert FROM to fixed point and store in TO. FROM
6072 must be floating point, TO must be signed. Use the conversion optab
6073 TAB to do the conversion. */
6076 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
6078 enum insn_code icode
;
6080 machine_mode fmode
, imode
;
6082 /* We first try to find a pair of modes, one real and one integer, at
6083 least as wide as FROM and TO, respectively, in which we can open-code
6084 this conversion. If the integer mode is wider than the mode of TO,
6085 we can do the conversion either signed or unsigned. */
6087 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
6088 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
6090 icode
= convert_optab_handler (tab
, imode
, fmode
,
6091 insn_optimization_type ());
6092 if (icode
!= CODE_FOR_nothing
)
6094 rtx_insn
*last
= get_last_insn ();
6095 if (fmode
!= GET_MODE (from
))
6096 from
= convert_to_mode (fmode
, from
, 0);
6098 if (imode
!= GET_MODE (to
))
6099 target
= gen_reg_rtx (imode
);
6101 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
6103 delete_insns_since (last
);
6107 convert_move (to
, target
, 0);
6115 /* Report whether we have an instruction to perform the operation
6116 specified by CODE on operands of mode MODE. */
6118 have_insn_for (enum rtx_code code
, machine_mode mode
)
6120 return (code_to_optab (code
)
6121 && (optab_handler (code_to_optab (code
), mode
)
6122 != CODE_FOR_nothing
));
6125 /* Print information about the current contents of the optabs on
6129 debug_optab_libfuncs (void)
6133 /* Dump the arithmetic optabs. */
6134 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
6135 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
6137 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
6140 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
6141 fprintf (stderr
, "%s\t%s:\t%s\n",
6142 GET_RTX_NAME (optab_to_code ((optab
) i
)),
6148 /* Dump the conversion optabs. */
6149 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
6150 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
6151 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
6153 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
6157 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
6158 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
6159 GET_RTX_NAME (optab_to_code ((optab
) i
)),
6167 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6168 CODE. Return 0 on failure. */
6171 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
6173 machine_mode mode
= GET_MODE (op1
);
6174 enum insn_code icode
;
6178 if (mode
== VOIDmode
)
6181 icode
= optab_handler (ctrap_optab
, mode
);
6182 if (icode
== CODE_FOR_nothing
)
6185 /* Some targets only accept a zero trap code. */
6186 if (!insn_operand_matches (icode
, 3, tcode
))
6189 do_pending_stack_adjust ();
6191 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
6196 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
6199 /* If that failed, then give up. */
6207 insn
= get_insns ();
6212 /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
6213 or unsigned operation code. */
6216 get_rtx_code_1 (enum tree_code tcode
, bool unsignedp
)
6228 code
= unsignedp
? LTU
: LT
;
6231 code
= unsignedp
? LEU
: LE
;
6234 code
= unsignedp
? GTU
: GT
;
6237 code
= unsignedp
? GEU
: GE
;
6240 case UNORDERED_EXPR
:
6280 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6281 or unsigned operation code. */
6284 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
6286 enum rtx_code code
= get_rtx_code_1 (tcode
, unsignedp
);
6287 gcc_assert (code
!= UNKNOWN
);
6291 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6292 select signed or unsigned operators. OPNO holds the index of the
6293 first comparison operand for insn ICODE. Do not generate the
6294 compare instruction itself. */
6297 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
6298 tree t_op0
, tree t_op1
, bool unsignedp
,
6299 enum insn_code icode
, unsigned int opno
)
6301 class expand_operand ops
[2];
6302 rtx rtx_op0
, rtx_op1
;
6303 machine_mode m0
, m1
;
6304 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
6306 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
6308 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6309 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6310 cases, use the original mode. */
6311 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
6313 m0
= GET_MODE (rtx_op0
);
6315 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
6317 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
6319 m1
= GET_MODE (rtx_op1
);
6321 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
6323 create_input_operand (&ops
[0], rtx_op0
, m0
);
6324 create_input_operand (&ops
[1], rtx_op1
, m1
);
6325 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
6327 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
6330 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
6331 the first vec_perm operand, assuming the second operand (for left shift
6332 first operand) is a constant vector of zeros. Return the shift distance
6333 in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6334 mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6335 shift or vec_shl_optab for left shift. */
6337 shift_amt_for_vec_perm_mask (machine_mode mode
, const vec_perm_indices
&sel
,
6340 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (mode
);
6341 poly_int64 first
= sel
[0];
6342 if (maybe_ge (sel
[0], GET_MODE_NUNITS (mode
)))
6345 if (shift_optab
== vec_shl_optab
)
6348 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
6350 unsigned firstidx
= 0;
6351 for (unsigned int i
= 0; i
< nelt
; i
++)
6353 if (known_eq (sel
[i
], nelt
))
6355 if (i
== 0 || firstidx
)
6360 ? maybe_ne (sel
[i
], nelt
+ i
- firstidx
)
6361 : maybe_ge (sel
[i
], nelt
))
6369 else if (!sel
.series_p (0, 1, first
, 1))
6372 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
6374 for (unsigned int i
= 1; i
< nelt
; i
++)
6376 poly_int64 expected
= i
+ first
;
6377 /* Indices into the second vector are all equivalent. */
6378 if (maybe_lt (sel
[i
], nelt
)
6379 ? maybe_ne (sel
[i
], expected
)
6380 : maybe_lt (expected
, nelt
))
6385 return gen_int_shift_amount (mode
, first
* bitsize
);
6388 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6391 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
6392 rtx v0
, rtx v1
, rtx sel
)
6394 machine_mode tmode
= GET_MODE (target
);
6395 machine_mode smode
= GET_MODE (sel
);
6396 class expand_operand ops
[4];
6398 gcc_assert (GET_MODE_CLASS (smode
) == MODE_VECTOR_INT
6399 || related_int_vector_mode (tmode
).require () == smode
);
6400 create_output_operand (&ops
[0], target
, tmode
);
6401 create_input_operand (&ops
[3], sel
, smode
);
6403 /* Make an effort to preserve v0 == v1. The target expander is able to
6404 rely on this to determine if we're permuting a single input operand. */
6405 if (rtx_equal_p (v0
, v1
))
6407 if (!insn_operand_matches (icode
, 1, v0
))
6408 v0
= force_reg (tmode
, v0
);
6409 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
6410 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
6412 create_fixed_operand (&ops
[1], v0
);
6413 create_fixed_operand (&ops
[2], v0
);
6417 create_input_operand (&ops
[1], v0
, tmode
);
6418 create_input_operand (&ops
[2], v1
, tmode
);
6421 if (maybe_expand_insn (icode
, 4, ops
))
6422 return ops
[0].value
;
6426 /* Implement a permutation of vectors v0 and v1 using the permutation
6427 vector in SEL and return the result. Use TARGET to hold the result
6428 if nonnull and convenient.
6430 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
6431 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
6432 to have a particular mode. */
6435 expand_vec_perm_const (machine_mode mode
, rtx v0
, rtx v1
,
6436 const vec_perm_builder
&sel
, machine_mode sel_mode
,
6439 if (!target
|| !register_operand (target
, mode
))
6440 target
= gen_reg_rtx (mode
);
6442 /* Set QIMODE to a different vector mode with byte elements.
6443 If no such mode, or if MODE already has byte elements, use VOIDmode. */
6444 machine_mode qimode
;
6445 if (!qimode_for_vec_perm (mode
).exists (&qimode
))
6448 rtx_insn
*last
= get_last_insn ();
6450 bool single_arg_p
= rtx_equal_p (v0
, v1
);
6451 /* Always specify two input vectors here and leave the target to handle
6452 cases in which the inputs are equal. Not all backends can cope with
6453 the single-input representation when testing for a double-input
6454 target instruction. */
6455 vec_perm_indices
indices (sel
, 2, GET_MODE_NUNITS (mode
));
6457 /* See if this can be handled with a vec_shr or vec_shl. We only do this
6458 if the second (for vec_shr) or first (for vec_shl) vector is all
6460 insn_code shift_code
= CODE_FOR_nothing
;
6461 insn_code shift_code_qi
= CODE_FOR_nothing
;
6462 optab shift_optab
= unknown_optab
;
6464 if (v1
== CONST0_RTX (GET_MODE (v1
)))
6465 shift_optab
= vec_shr_optab
;
6466 else if (v0
== CONST0_RTX (GET_MODE (v0
)))
6468 shift_optab
= vec_shl_optab
;
6471 if (shift_optab
!= unknown_optab
)
6473 shift_code
= optab_handler (shift_optab
, mode
);
6474 shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
6475 ? optab_handler (shift_optab
, qimode
)
6476 : CODE_FOR_nothing
);
6478 if (shift_code
!= CODE_FOR_nothing
|| shift_code_qi
!= CODE_FOR_nothing
)
6480 rtx shift_amt
= shift_amt_for_vec_perm_mask (mode
, indices
, shift_optab
);
6483 class expand_operand ops
[3];
6484 if (shift_amt
== const0_rtx
)
6486 if (shift_code
!= CODE_FOR_nothing
)
6488 create_output_operand (&ops
[0], target
, mode
);
6489 create_input_operand (&ops
[1], v2
, mode
);
6490 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
6491 if (maybe_expand_insn (shift_code
, 3, ops
))
6492 return ops
[0].value
;
6494 if (shift_code_qi
!= CODE_FOR_nothing
)
6496 rtx tmp
= gen_reg_rtx (qimode
);
6497 create_output_operand (&ops
[0], tmp
, qimode
);
6498 create_input_operand (&ops
[1], gen_lowpart (qimode
, v2
), qimode
);
6499 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
6500 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
6501 return gen_lowpart (mode
, ops
[0].value
);
6506 if (targetm
.vectorize
.vec_perm_const
!= NULL
)
6511 gcc_checking_assert (GET_MODE (v0
) == GET_MODE (v1
));
6512 machine_mode op_mode
= GET_MODE (v0
);
6513 if (targetm
.vectorize
.vec_perm_const (mode
, op_mode
, target
, v0
, v1
,
6518 /* Fall back to a constant byte-based permutation. */
6519 vec_perm_indices qimode_indices
;
6520 rtx target_qi
= NULL_RTX
, v0_qi
= NULL_RTX
, v1_qi
= NULL_RTX
;
6521 if (qimode
!= VOIDmode
)
6523 qimode_indices
.new_expanded_vector (indices
, GET_MODE_UNIT_SIZE (mode
));
6524 target_qi
= gen_reg_rtx (qimode
);
6525 v0_qi
= gen_lowpart (qimode
, v0
);
6526 v1_qi
= gen_lowpart (qimode
, v1
);
6527 if (targetm
.vectorize
.vec_perm_const
!= NULL
6528 && targetm
.vectorize
.vec_perm_const (qimode
, qimode
, target_qi
, v0_qi
,
6529 v1_qi
, qimode_indices
))
6530 return gen_lowpart (mode
, target_qi
);
6533 v0
= force_reg (mode
, v0
);
6536 v1
= force_reg (mode
, v1
);
6538 /* Otherwise expand as a fully variable permuation. */
6540 /* The optabs are only defined for selectors with the same width
6541 as the values being permuted. */
6542 machine_mode required_sel_mode
;
6543 if (!related_int_vector_mode (mode
).exists (&required_sel_mode
))
6545 delete_insns_since (last
);
6549 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
6550 If that isn't the mode we want then we need to prove that using
6551 REQUIRED_SEL_MODE is OK. */
6552 if (sel_mode
!= required_sel_mode
)
6554 if (!selector_fits_mode_p (required_sel_mode
, indices
))
6556 delete_insns_since (last
);
6559 sel_mode
= required_sel_mode
;
6562 insn_code icode
= direct_optab_handler (vec_perm_optab
, mode
);
6563 if (icode
!= CODE_FOR_nothing
)
6565 rtx sel_rtx
= vec_perm_indices_to_rtx (sel_mode
, indices
);
6566 rtx tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel_rtx
);
6571 if (qimode
!= VOIDmode
6572 && selector_fits_mode_p (qimode
, qimode_indices
))
6574 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6575 if (icode
!= CODE_FOR_nothing
)
6577 rtx sel_qi
= vec_perm_indices_to_rtx (qimode
, qimode_indices
);
6578 rtx tmp
= expand_vec_perm_1 (icode
, target_qi
, v0_qi
, v1_qi
, sel_qi
);
6580 return gen_lowpart (mode
, tmp
);
6584 delete_insns_since (last
);
6588 /* Implement a permutation of vectors v0 and v1 using the permutation
6589 vector in SEL and return the result. Use TARGET to hold the result
6590 if nonnull and convenient.
6592 MODE is the mode of the vectors being permuted (V0 and V1).
6593 SEL must have the integer equivalent of MODE and is known to be
6594 unsuitable for permutes with a constant permutation vector. */
6597 expand_vec_perm_var (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
6599 enum insn_code icode
;
6603 u
= GET_MODE_UNIT_SIZE (mode
);
6605 if (!target
|| GET_MODE (target
) != mode
)
6606 target
= gen_reg_rtx (mode
);
6608 icode
= direct_optab_handler (vec_perm_optab
, mode
);
6609 if (icode
!= CODE_FOR_nothing
)
6611 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
6616 /* As a special case to aid several targets, lower the element-based
6617 permutation to a byte-based permutation and try again. */
6618 machine_mode qimode
;
6619 if (!qimode_for_vec_perm (mode
).exists (&qimode
)
6620 || maybe_gt (GET_MODE_NUNITS (qimode
), GET_MODE_MASK (QImode
) + 1))
6622 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6623 if (icode
== CODE_FOR_nothing
)
6626 /* Multiply each element by its byte size. */
6627 machine_mode selmode
= GET_MODE (sel
);
6629 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
6630 NULL
, 0, OPTAB_DIRECT
);
6632 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
6633 gen_int_shift_amount (selmode
, exact_log2 (u
)),
6634 NULL
, 0, OPTAB_DIRECT
);
6635 gcc_assert (sel
!= NULL
);
6637 /* Broadcast the low byte each element into each of its bytes.
6638 The encoding has U interleaved stepped patterns, one for each
6639 byte of an element. */
6640 vec_perm_builder
const_sel (GET_MODE_SIZE (mode
), u
, 3);
6641 unsigned int low_byte_in_u
= BYTES_BIG_ENDIAN
? u
- 1 : 0;
6642 for (i
= 0; i
< 3; ++i
)
6643 for (unsigned int j
= 0; j
< u
; ++j
)
6644 const_sel
.quick_push (i
* u
+ low_byte_in_u
);
6645 sel
= gen_lowpart (qimode
, sel
);
6646 sel
= expand_vec_perm_const (qimode
, sel
, sel
, const_sel
, qimode
, NULL
);
6647 gcc_assert (sel
!= NULL
);
6649 /* Add the byte offset to each byte element. */
6650 /* Note that the definition of the indicies here is memory ordering,
6651 so there should be no difference between big and little endian. */
6652 rtx_vector_builder
byte_indices (qimode
, u
, 1);
6653 for (i
= 0; i
< u
; ++i
)
6654 byte_indices
.quick_push (GEN_INT (i
));
6655 tmp
= byte_indices
.build ();
6656 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
6657 sel
, 0, OPTAB_DIRECT
);
6658 gcc_assert (sel_qi
!= NULL
);
6660 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
6661 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
6662 gen_lowpart (qimode
, v1
), sel_qi
);
6664 tmp
= gen_lowpart (mode
, tmp
);
6668 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
6669 Use TARGET for the result if nonnull and convenient. */
6672 expand_vec_series_expr (machine_mode vmode
, rtx op0
, rtx op1
, rtx target
)
6674 class expand_operand ops
[3];
6675 enum insn_code icode
;
6676 machine_mode emode
= GET_MODE_INNER (vmode
);
6678 icode
= direct_optab_handler (vec_series_optab
, vmode
);
6679 gcc_assert (icode
!= CODE_FOR_nothing
);
6681 create_output_operand (&ops
[0], target
, vmode
);
6682 create_input_operand (&ops
[1], op0
, emode
);
6683 create_input_operand (&ops
[2], op1
, emode
);
6685 expand_insn (icode
, 3, ops
);
6686 return ops
[0].value
;
6689 /* Generate insns for a vector comparison into a mask. */
6692 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
6694 class expand_operand ops
[4];
6695 enum insn_code icode
;
6697 machine_mode mask_mode
= TYPE_MODE (type
);
6701 enum tree_code tcode
;
6703 op0a
= TREE_OPERAND (exp
, 0);
6704 op0b
= TREE_OPERAND (exp
, 1);
6705 tcode
= TREE_CODE (exp
);
6707 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
6708 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
6710 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
6711 if (icode
== CODE_FOR_nothing
)
6713 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
6714 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
6715 if (icode
== CODE_FOR_nothing
)
6719 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
6720 unsignedp
, icode
, 2);
6721 create_output_operand (&ops
[0], target
, mask_mode
);
6722 create_fixed_operand (&ops
[1], comparison
);
6723 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
6724 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
6725 expand_insn (icode
, 4, ops
);
6726 return ops
[0].value
;
6729 /* Expand a highpart multiply. */
6732 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
6733 rtx target
, bool uns_p
)
6735 class expand_operand eops
[3];
6736 enum insn_code icode
;
6742 method
= can_mult_highpart_p (mode
, uns_p
);
6748 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
6749 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
6752 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
6753 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
6756 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
6757 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
6758 if (BYTES_BIG_ENDIAN
)
6759 std::swap (tab1
, tab2
);
6765 icode
= optab_handler (tab1
, mode
);
6766 wmode
= insn_data
[icode
].operand
[0].mode
;
6767 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode
),
6768 GET_MODE_NUNITS (mode
)));
6769 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode
), GET_MODE_SIZE (mode
)));
6771 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
6772 create_input_operand (&eops
[1], op0
, mode
);
6773 create_input_operand (&eops
[2], op1
, mode
);
6774 expand_insn (icode
, 3, eops
);
6775 m1
= gen_lowpart (mode
, eops
[0].value
);
6777 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
6778 create_input_operand (&eops
[1], op0
, mode
);
6779 create_input_operand (&eops
[2], op1
, mode
);
6780 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
6781 m2
= gen_lowpart (mode
, eops
[0].value
);
6783 vec_perm_builder sel
;
6786 /* The encoding has 2 interleaved stepped patterns. */
6787 sel
.new_vector (GET_MODE_NUNITS (mode
), 2, 3);
6788 for (i
= 0; i
< 6; ++i
)
6789 sel
.quick_push (!BYTES_BIG_ENDIAN
+ (i
& ~1)
6790 + ((i
& 1) ? GET_MODE_NUNITS (mode
) : 0));
6794 /* The encoding has a single interleaved stepped pattern. */
6795 sel
.new_vector (GET_MODE_NUNITS (mode
), 1, 3);
6796 for (i
= 0; i
< 3; ++i
)
6797 sel
.quick_push (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
6800 return expand_vec_perm_const (mode
, m1
, m2
, sel
, BLKmode
, target
);
6803 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6807 find_cc_set (rtx x
, const_rtx pat
, void *data
)
6809 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
6810 && GET_CODE (pat
) == SET
)
6812 rtx
*p_cc_reg
= (rtx
*) data
;
6813 gcc_assert (!*p_cc_reg
);
6818 /* This is a helper function for the other atomic operations. This function
6819 emits a loop that contains SEQ that iterates until a compare-and-swap
6820 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6821 a set of instructions that takes a value from OLD_REG as an input and
6822 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6823 set to the current contents of MEM. After SEQ, a compare-and-swap will
6824 attempt to update MEM with NEW_REG. The function returns true when the
6825 loop was generated successfully. */
6828 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
6830 machine_mode mode
= GET_MODE (mem
);
6831 rtx_code_label
*label
;
6832 rtx cmp_reg
, success
, oldval
;
6834 /* The loop we want to generate looks like
6840 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6844 Note that we only do the plain load from memory once. Subsequent
6845 iterations use the value loaded by the compare-and-swap pattern. */
6847 label
= gen_label_rtx ();
6848 cmp_reg
= gen_reg_rtx (mode
);
6850 emit_move_insn (cmp_reg
, mem
);
6852 emit_move_insn (old_reg
, cmp_reg
);
6858 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
6859 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
6863 if (oldval
!= cmp_reg
)
6864 emit_move_insn (cmp_reg
, oldval
);
6866 /* Mark this jump predicted not taken. */
6867 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
6868 GET_MODE (success
), 1, label
,
6869 profile_probability::guessed_never ());
6874 /* This function tries to emit an atomic_exchange intruction. VAL is written
6875 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6876 using TARGET if possible. */
6879 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6881 machine_mode mode
= GET_MODE (mem
);
6882 enum insn_code icode
;
6884 /* If the target supports the exchange directly, great. */
6885 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
6886 if (icode
!= CODE_FOR_nothing
)
6888 class expand_operand ops
[4];
6890 create_output_operand (&ops
[0], target
, mode
);
6891 create_fixed_operand (&ops
[1], mem
);
6892 create_input_operand (&ops
[2], val
, mode
);
6893 create_integer_operand (&ops
[3], model
);
6894 if (maybe_expand_insn (icode
, 4, ops
))
6895 return ops
[0].value
;
6901 /* This function tries to implement an atomic exchange operation using
6902 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6903 The previous contents of *MEM are returned, using TARGET if possible.
6904 Since this instructionn is an acquire barrier only, stronger memory
6905 models may require additional barriers to be emitted. */
6908 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
6909 enum memmodel model
)
6911 machine_mode mode
= GET_MODE (mem
);
6912 enum insn_code icode
;
6913 rtx_insn
*last_insn
= get_last_insn ();
6915 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
6917 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6918 exists, and the memory model is stronger than acquire, add a release
6919 barrier before the instruction. */
6921 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
6922 expand_mem_thread_fence (model
);
6924 if (icode
!= CODE_FOR_nothing
)
6926 class expand_operand ops
[3];
6927 create_output_operand (&ops
[0], target
, mode
);
6928 create_fixed_operand (&ops
[1], mem
);
6929 create_input_operand (&ops
[2], val
, mode
);
6930 if (maybe_expand_insn (icode
, 3, ops
))
6931 return ops
[0].value
;
6934 /* If an external test-and-set libcall is provided, use that instead of
6935 any external compare-and-swap that we might get from the compare-and-
6936 swap-loop expansion later. */
6937 if (!can_compare_and_swap_p (mode
, false))
6939 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
6940 if (libfunc
!= NULL
)
6944 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6945 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6946 mode
, addr
, ptr_mode
,
6951 /* If the test_and_set can't be emitted, eliminate any barrier that might
6952 have been emitted. */
6953 delete_insns_since (last_insn
);
6957 /* This function tries to implement an atomic exchange operation using a
6958 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6959 *MEM are returned, using TARGET if possible. No memory model is required
6960 since a compare_and_swap loop is seq-cst. */
6963 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
6965 machine_mode mode
= GET_MODE (mem
);
6967 if (can_compare_and_swap_p (mode
, true))
6969 if (!target
|| !register_operand (target
, mode
))
6970 target
= gen_reg_rtx (mode
);
6971 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
6978 /* This function tries to implement an atomic test-and-set operation
6979 using the atomic_test_and_set instruction pattern. A boolean value
6980 is returned from the operation, using TARGET if possible. */
6983 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6985 machine_mode pat_bool_mode
;
6986 class expand_operand ops
[3];
6988 if (!targetm
.have_atomic_test_and_set ())
6991 /* While we always get QImode from __atomic_test_and_set, we get
6992 other memory modes from __sync_lock_test_and_set. Note that we
6993 use no endian adjustment here. This matches the 4.6 behavior
6994 in the Sparc backend. */
6995 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
6996 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
6997 if (GET_MODE (mem
) != QImode
)
6998 mem
= adjust_address_nv (mem
, QImode
, 0);
7000 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
7001 create_output_operand (&ops
[0], target
, pat_bool_mode
);
7002 create_fixed_operand (&ops
[1], mem
);
7003 create_integer_operand (&ops
[2], model
);
7005 if (maybe_expand_insn (icode
, 3, ops
))
7006 return ops
[0].value
;
7010 /* This function expands the legacy _sync_lock test_and_set operation which is
7011 generally an atomic exchange. Some limited targets only allow the
7012 constant 1 to be stored. This is an ACQUIRE operation.
7014 TARGET is an optional place to stick the return value.
7015 MEM is where VAL is stored. */
7018 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
7022 /* Try an atomic_exchange first. */
7023 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
7027 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
7028 MEMMODEL_SYNC_ACQUIRE
);
7032 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
7036 /* If there are no other options, try atomic_test_and_set if the value
7037 being stored is 1. */
7038 if (val
== const1_rtx
)
7039 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
7044 /* This function expands the atomic test_and_set operation:
7045 atomically store a boolean TRUE into MEM and return the previous value.
7047 MEMMODEL is the memory model variant to use.
7048 TARGET is an optional place to stick the return value. */
7051 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
7053 machine_mode mode
= GET_MODE (mem
);
7054 rtx ret
, trueval
, subtarget
;
7056 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
7060 /* Be binary compatible with non-default settings of trueval, and different
7061 cpu revisions. E.g. one revision may have atomic-test-and-set, but
7062 another only has atomic-exchange. */
7063 if (targetm
.atomic_test_and_set_trueval
== 1)
7065 trueval
= const1_rtx
;
7066 subtarget
= target
? target
: gen_reg_rtx (mode
);
7070 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
7071 subtarget
= gen_reg_rtx (mode
);
7074 /* Try the atomic-exchange optab... */
7075 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
7077 /* ... then an atomic-compare-and-swap loop ... */
7079 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
7081 /* ... before trying the vaguely defined legacy lock_test_and_set. */
7083 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
7085 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7086 things with the value 1. Thus we try again without trueval. */
7087 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
7089 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
7093 /* Rectify the not-one trueval. */
7094 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
7102 /* This function expands the atomic exchange operation:
7103 atomically store VAL in MEM and return the previous value in MEM.
7105 MEMMODEL is the memory model variant to use.
7106 TARGET is an optional place to stick the return value. */
7109 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
7111 machine_mode mode
= GET_MODE (mem
);
7114 /* If loads are not atomic for the required size and we are not called to
7115 provide a __sync builtin, do not do anything so that we stay consistent
7116 with atomic loads of the same size. */
7117 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
7120 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7122 /* Next try a compare-and-swap loop for the exchange. */
7124 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
7129 /* This function expands the atomic compare exchange operation:
7131 *PTARGET_BOOL is an optional place to store the boolean success/failure.
7132 *PTARGET_OVAL is an optional place to store the old value from memory.
7133 Both target parameters may be NULL or const0_rtx to indicate that we do
7134 not care about that return value. Both target parameters are updated on
7135 success to the actual location of the corresponding result.
7137 MEMMODEL is the memory model variant to use.
7139 The return value of the function is true for success. */
7142 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
7143 rtx mem
, rtx expected
, rtx desired
,
7144 bool is_weak
, enum memmodel succ_model
,
7145 enum memmodel fail_model
)
7147 machine_mode mode
= GET_MODE (mem
);
7148 class expand_operand ops
[8];
7149 enum insn_code icode
;
7150 rtx target_oval
, target_bool
= NULL_RTX
;
7153 /* If loads are not atomic for the required size and we are not called to
7154 provide a __sync builtin, do not do anything so that we stay consistent
7155 with atomic loads of the same size. */
7156 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
7159 /* Load expected into a register for the compare and swap. */
7160 if (MEM_P (expected
))
7161 expected
= copy_to_reg (expected
);
7163 /* Make sure we always have some place to put the return oldval.
7164 Further, make sure that place is distinct from the input expected,
7165 just in case we need that path down below. */
7166 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
7167 ptarget_oval
= NULL
;
7169 if (ptarget_oval
== NULL
7170 || (target_oval
= *ptarget_oval
) == NULL
7171 || reg_overlap_mentioned_p (expected
, target_oval
))
7172 target_oval
= gen_reg_rtx (mode
);
7174 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
7175 if (icode
!= CODE_FOR_nothing
)
7177 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
7179 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
7180 ptarget_bool
= NULL
;
7182 /* Make sure we always have a place for the bool operand. */
7183 if (ptarget_bool
== NULL
7184 || (target_bool
= *ptarget_bool
) == NULL
7185 || GET_MODE (target_bool
) != bool_mode
)
7186 target_bool
= gen_reg_rtx (bool_mode
);
7188 /* Emit the compare_and_swap. */
7189 create_output_operand (&ops
[0], target_bool
, bool_mode
);
7190 create_output_operand (&ops
[1], target_oval
, mode
);
7191 create_fixed_operand (&ops
[2], mem
);
7192 create_input_operand (&ops
[3], expected
, mode
);
7193 create_input_operand (&ops
[4], desired
, mode
);
7194 create_integer_operand (&ops
[5], is_weak
);
7195 create_integer_operand (&ops
[6], succ_model
);
7196 create_integer_operand (&ops
[7], fail_model
);
7197 if (maybe_expand_insn (icode
, 8, ops
))
7199 /* Return success/failure. */
7200 target_bool
= ops
[0].value
;
7201 target_oval
= ops
[1].value
;
7206 /* Otherwise fall back to the original __sync_val_compare_and_swap
7207 which is always seq-cst. */
7208 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
7209 if (icode
!= CODE_FOR_nothing
)
7213 create_output_operand (&ops
[0], target_oval
, mode
);
7214 create_fixed_operand (&ops
[1], mem
);
7215 create_input_operand (&ops
[2], expected
, mode
);
7216 create_input_operand (&ops
[3], desired
, mode
);
7217 if (!maybe_expand_insn (icode
, 4, ops
))
7220 target_oval
= ops
[0].value
;
7222 /* If the caller isn't interested in the boolean return value,
7223 skip the computation of it. */
7224 if (ptarget_bool
== NULL
)
7227 /* Otherwise, work out if the compare-and-swap succeeded. */
7229 if (have_insn_for (COMPARE
, CCmode
))
7230 note_stores (get_last_insn (), find_cc_set
, &cc_reg
);
7233 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
7234 const0_rtx
, VOIDmode
, 0, 1);
7237 goto success_bool_from_val
;
7240 /* Also check for library support for __sync_val_compare_and_swap. */
7241 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
7242 if (libfunc
!= NULL
)
7244 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
7245 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
7246 mode
, addr
, ptr_mode
,
7247 expected
, mode
, desired
, mode
);
7248 emit_move_insn (target_oval
, target
);
7250 /* Compute the boolean return value only if requested. */
7252 goto success_bool_from_val
;
7260 success_bool_from_val
:
7261 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
7262 expected
, VOIDmode
, 1, 1);
7264 /* Make sure that the oval output winds up where the caller asked. */
7266 *ptarget_oval
= target_oval
;
7268 *ptarget_bool
= target_bool
;
7272 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
7275 expand_asm_memory_blockage (void)
7279 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
7280 rtvec_alloc (0), rtvec_alloc (0),
7281 rtvec_alloc (0), UNKNOWN_LOCATION
);
7282 MEM_VOLATILE_P (asm_op
) = 1;
7284 clob
= gen_rtx_SCRATCH (VOIDmode
);
7285 clob
= gen_rtx_MEM (BLKmode
, clob
);
7286 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
7288 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
7291 /* Do not propagate memory accesses across this point. */
7294 expand_memory_blockage (void)
7296 if (targetm
.have_memory_blockage ())
7297 emit_insn (targetm
.gen_memory_blockage ());
7299 expand_asm_memory_blockage ();
7302 /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7303 same time clobbering the register set specified by REGS. */
7306 expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs
)
7308 rtx asm_op
, clob_mem
;
7310 unsigned int num_of_regs
= 0;
7311 for (unsigned int i
= 0; i
< FIRST_PSEUDO_REGISTER
; i
++)
7312 if (TEST_HARD_REG_BIT (regs
, i
))
7315 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
7316 rtvec_alloc (0), rtvec_alloc (0),
7317 rtvec_alloc (0), UNKNOWN_LOCATION
);
7318 MEM_VOLATILE_P (asm_op
) = 1;
7320 rtvec v
= rtvec_alloc (num_of_regs
+ 2);
7322 clob_mem
= gen_rtx_SCRATCH (VOIDmode
);
7323 clob_mem
= gen_rtx_MEM (BLKmode
, clob_mem
);
7324 clob_mem
= gen_rtx_CLOBBER (VOIDmode
, clob_mem
);
7326 RTVEC_ELT (v
, 0) = asm_op
;
7327 RTVEC_ELT (v
, 1) = clob_mem
;
7329 if (num_of_regs
> 0)
7332 for (unsigned int i
= 0; i
< FIRST_PSEUDO_REGISTER
; i
++)
7333 if (TEST_HARD_REG_BIT (regs
, i
))
7335 RTVEC_ELT (v
, j
) = gen_rtx_CLOBBER (VOIDmode
, regno_reg_rtx
[i
]);
7338 gcc_assert (j
== (num_of_regs
+ 2));
7341 emit_insn (gen_rtx_PARALLEL (VOIDmode
, v
));
7344 /* This routine will either emit the mem_thread_fence pattern or issue a
7345 sync_synchronize to generate a fence for memory model MEMMODEL. */
7348 expand_mem_thread_fence (enum memmodel model
)
7350 if (is_mm_relaxed (model
))
7352 if (targetm
.have_mem_thread_fence ())
7354 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
7355 expand_memory_blockage ();
7357 else if (targetm
.have_memory_barrier ())
7358 emit_insn (targetm
.gen_memory_barrier ());
7359 else if (synchronize_libfunc
!= NULL_RTX
)
7360 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
);
7362 expand_memory_blockage ();
7365 /* Emit a signal fence with given memory model. */
7368 expand_mem_signal_fence (enum memmodel model
)
7370 /* No machine barrier is required to implement a signal fence, but
7371 a compiler memory barrier must be issued, except for relaxed MM. */
7372 if (!is_mm_relaxed (model
))
7373 expand_memory_blockage ();
7376 /* This function expands the atomic load operation:
7377 return the atomically loaded value in MEM.
7379 MEMMODEL is the memory model variant to use.
7380 TARGET is an option place to stick the return value. */
7383 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
7385 machine_mode mode
= GET_MODE (mem
);
7386 enum insn_code icode
;
7388 /* If the target supports the load directly, great. */
7389 icode
= direct_optab_handler (atomic_load_optab
, mode
);
7390 if (icode
!= CODE_FOR_nothing
)
7392 class expand_operand ops
[3];
7393 rtx_insn
*last
= get_last_insn ();
7394 if (is_mm_seq_cst (model
))
7395 expand_memory_blockage ();
7397 create_output_operand (&ops
[0], target
, mode
);
7398 create_fixed_operand (&ops
[1], mem
);
7399 create_integer_operand (&ops
[2], model
);
7400 if (maybe_expand_insn (icode
, 3, ops
))
7402 if (!is_mm_relaxed (model
))
7403 expand_memory_blockage ();
7404 return ops
[0].value
;
7406 delete_insns_since (last
);
7409 /* If the size of the object is greater than word size on this target,
7410 then we assume that a load will not be atomic. We could try to
7411 emulate a load with a compare-and-swap operation, but the store that
7412 doing this could result in would be incorrect if this is a volatile
7413 atomic load or targetting read-only-mapped memory. */
7414 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
7415 /* If there is no atomic load, leave the library call. */
7418 /* Otherwise assume loads are atomic, and emit the proper barriers. */
7419 if (!target
|| target
== const0_rtx
)
7420 target
= gen_reg_rtx (mode
);
7422 /* For SEQ_CST, emit a barrier before the load. */
7423 if (is_mm_seq_cst (model
))
7424 expand_mem_thread_fence (model
);
7426 emit_move_insn (target
, mem
);
7428 /* Emit the appropriate barrier after the load. */
7429 expand_mem_thread_fence (model
);
7434 /* This function expands the atomic store operation:
7435 Atomically store VAL in MEM.
7436 MEMMODEL is the memory model variant to use.
7437 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7438 function returns const0_rtx if a pattern was emitted. */
7441 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
7443 machine_mode mode
= GET_MODE (mem
);
7444 enum insn_code icode
;
7445 class expand_operand ops
[3];
7447 /* If the target supports the store directly, great. */
7448 icode
= direct_optab_handler (atomic_store_optab
, mode
);
7449 if (icode
!= CODE_FOR_nothing
)
7451 rtx_insn
*last
= get_last_insn ();
7452 if (!is_mm_relaxed (model
))
7453 expand_memory_blockage ();
7454 create_fixed_operand (&ops
[0], mem
);
7455 create_input_operand (&ops
[1], val
, mode
);
7456 create_integer_operand (&ops
[2], model
);
7457 if (maybe_expand_insn (icode
, 3, ops
))
7459 if (is_mm_seq_cst (model
))
7460 expand_memory_blockage ();
7463 delete_insns_since (last
);
7466 /* If using __sync_lock_release is a viable alternative, try it.
7467 Note that this will not be set to true if we are expanding a generic
7468 __atomic_store_n. */
7471 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
7472 if (icode
!= CODE_FOR_nothing
)
7474 create_fixed_operand (&ops
[0], mem
);
7475 create_input_operand (&ops
[1], const0_rtx
, mode
);
7476 if (maybe_expand_insn (icode
, 2, ops
))
7478 /* lock_release is only a release barrier. */
7479 if (is_mm_seq_cst (model
))
7480 expand_mem_thread_fence (model
);
7486 /* If the size of the object is greater than word size on this target,
7487 a default store will not be atomic. */
7488 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
7490 /* If loads are atomic or we are called to provide a __sync builtin,
7491 we can try a atomic_exchange and throw away the result. Otherwise,
7492 don't do anything so that we do not create an inconsistency between
7493 loads and stores. */
7494 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
7496 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
7498 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
7506 /* Otherwise assume stores are atomic, and emit the proper barriers. */
7507 expand_mem_thread_fence (model
);
7509 emit_move_insn (mem
, val
);
7511 /* For SEQ_CST, also emit a barrier after the store. */
7512 if (is_mm_seq_cst (model
))
7513 expand_mem_thread_fence (model
);
7519 /* Structure containing the pointers and values required to process the
7520 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7522 struct atomic_op_functions
7524 direct_optab mem_fetch_before
;
7525 direct_optab mem_fetch_after
;
7526 direct_optab mem_no_result
;
7529 direct_optab no_result
;
7530 enum rtx_code reverse_code
;
7534 /* Fill in structure pointed to by OP with the various optab entries for an
7535 operation of type CODE. */
7538 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
7540 gcc_assert (op
!= NULL
);
7542 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7543 in the source code during compilation, and the optab entries are not
7544 computable until runtime. Fill in the values at runtime. */
7548 op
->mem_fetch_before
= atomic_fetch_add_optab
;
7549 op
->mem_fetch_after
= atomic_add_fetch_optab
;
7550 op
->mem_no_result
= atomic_add_optab
;
7551 op
->fetch_before
= sync_old_add_optab
;
7552 op
->fetch_after
= sync_new_add_optab
;
7553 op
->no_result
= sync_add_optab
;
7554 op
->reverse_code
= MINUS
;
7557 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
7558 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
7559 op
->mem_no_result
= atomic_sub_optab
;
7560 op
->fetch_before
= sync_old_sub_optab
;
7561 op
->fetch_after
= sync_new_sub_optab
;
7562 op
->no_result
= sync_sub_optab
;
7563 op
->reverse_code
= PLUS
;
7566 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
7567 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
7568 op
->mem_no_result
= atomic_xor_optab
;
7569 op
->fetch_before
= sync_old_xor_optab
;
7570 op
->fetch_after
= sync_new_xor_optab
;
7571 op
->no_result
= sync_xor_optab
;
7572 op
->reverse_code
= XOR
;
7575 op
->mem_fetch_before
= atomic_fetch_and_optab
;
7576 op
->mem_fetch_after
= atomic_and_fetch_optab
;
7577 op
->mem_no_result
= atomic_and_optab
;
7578 op
->fetch_before
= sync_old_and_optab
;
7579 op
->fetch_after
= sync_new_and_optab
;
7580 op
->no_result
= sync_and_optab
;
7581 op
->reverse_code
= UNKNOWN
;
7584 op
->mem_fetch_before
= atomic_fetch_or_optab
;
7585 op
->mem_fetch_after
= atomic_or_fetch_optab
;
7586 op
->mem_no_result
= atomic_or_optab
;
7587 op
->fetch_before
= sync_old_ior_optab
;
7588 op
->fetch_after
= sync_new_ior_optab
;
7589 op
->no_result
= sync_ior_optab
;
7590 op
->reverse_code
= UNKNOWN
;
7593 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
7594 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
7595 op
->mem_no_result
= atomic_nand_optab
;
7596 op
->fetch_before
= sync_old_nand_optab
;
7597 op
->fetch_after
= sync_new_nand_optab
;
7598 op
->no_result
= sync_nand_optab
;
7599 op
->reverse_code
= UNKNOWN
;
7606 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7607 using memory order MODEL. If AFTER is true the operation needs to return
7608 the value of *MEM after the operation, otherwise the previous value.
7609 TARGET is an optional place to place the result. The result is unused if
7611 Return the result if there is a better sequence, otherwise NULL_RTX. */
7614 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7615 enum memmodel model
, bool after
)
7617 /* If the value is prefetched, or not used, it may be possible to replace
7618 the sequence with a native exchange operation. */
7619 if (!after
|| target
== const0_rtx
)
7621 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7622 if (code
== AND
&& val
== const0_rtx
)
7624 if (target
== const0_rtx
)
7625 target
= gen_reg_rtx (GET_MODE (mem
));
7626 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7629 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7630 if (code
== IOR
&& val
== constm1_rtx
)
7632 if (target
== const0_rtx
)
7633 target
= gen_reg_rtx (GET_MODE (mem
));
7634 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7641 /* Try to emit an instruction for a specific operation varaition.
7642 OPTAB contains the OP functions.
7643 TARGET is an optional place to return the result. const0_rtx means unused.
7644 MEM is the memory location to operate on.
7645 VAL is the value to use in the operation.
7646 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7647 MODEL is the memory model, if used.
7648 AFTER is true if the returned result is the value after the operation. */
7651 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
7652 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
7654 machine_mode mode
= GET_MODE (mem
);
7655 class expand_operand ops
[4];
7656 enum insn_code icode
;
7660 /* Check to see if there is a result returned. */
7661 if (target
== const0_rtx
)
7665 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
7666 create_integer_operand (&ops
[2], model
);
7671 icode
= direct_optab_handler (optab
->no_result
, mode
);
7675 /* Otherwise, we need to generate a result. */
7680 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
7681 : optab
->mem_fetch_before
, mode
);
7682 create_integer_operand (&ops
[3], model
);
7687 icode
= optab_handler (after
? optab
->fetch_after
7688 : optab
->fetch_before
, mode
);
7691 create_output_operand (&ops
[op_counter
++], target
, mode
);
7693 if (icode
== CODE_FOR_nothing
)
7696 create_fixed_operand (&ops
[op_counter
++], mem
);
7697 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7698 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
7700 if (maybe_expand_insn (icode
, num_ops
, ops
))
7701 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
7707 /* This function expands an atomic fetch_OP or OP_fetch operation:
7708 TARGET is an option place to stick the return value. const0_rtx indicates
7709 the result is unused.
7710 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7711 CODE is the operation being performed (OP)
7712 MEMMODEL is the memory model variant to use.
7713 AFTER is true to return the result of the operation (OP_fetch).
7714 AFTER is false to return the value before the operation (fetch_OP).
7716 This function will *only* generate instructions if there is a direct
7717 optab. No compare and swap loops or libcalls will be generated. */
7720 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
7721 enum rtx_code code
, enum memmodel model
,
7724 machine_mode mode
= GET_MODE (mem
);
7725 struct atomic_op_functions optab
;
7727 bool unused_result
= (target
== const0_rtx
);
7729 get_atomic_op_for_code (&optab
, code
);
7731 /* Check to see if there are any better instructions. */
7732 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
7736 /* Check for the case where the result isn't used and try those patterns. */
7739 /* Try the memory model variant first. */
7740 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
7744 /* Next try the old style withuot a memory model. */
7745 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
7749 /* There is no no-result pattern, so try patterns with a result. */
7753 /* Try the __atomic version. */
7754 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
7758 /* Try the older __sync version. */
7759 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
7763 /* If the fetch value can be calculated from the other variation of fetch,
7764 try that operation. */
7765 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
7767 /* Try the __atomic version, then the older __sync version. */
7768 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
7770 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
7774 /* If the result isn't used, no need to do compensation code. */
7778 /* Issue compensation code. Fetch_after == fetch_before OP val.
7779 Fetch_before == after REVERSE_OP val. */
7781 code
= optab
.reverse_code
;
7784 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
7785 true, OPTAB_LIB_WIDEN
);
7786 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
7789 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
7790 true, OPTAB_LIB_WIDEN
);
7795 /* No direct opcode can be generated. */
7801 /* This function expands an atomic fetch_OP or OP_fetch operation:
7802 TARGET is an option place to stick the return value. const0_rtx indicates
7803 the result is unused.
7804 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7805 CODE is the operation being performed (OP)
7806 MEMMODEL is the memory model variant to use.
7807 AFTER is true to return the result of the operation (OP_fetch).
7808 AFTER is false to return the value before the operation (fetch_OP). */
7810 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7811 enum memmodel model
, bool after
)
7813 machine_mode mode
= GET_MODE (mem
);
7815 bool unused_result
= (target
== const0_rtx
);
7817 /* If loads are not atomic for the required size and we are not called to
7818 provide a __sync builtin, do not do anything so that we stay consistent
7819 with atomic loads of the same size. */
7820 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
7823 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
7829 /* Add/sub can be implemented by doing the reverse operation with -(val). */
7830 if (code
== PLUS
|| code
== MINUS
)
7833 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
7836 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
7837 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
7841 /* PLUS worked so emit the insns and return. */
7848 /* PLUS did not work, so throw away the negation code and continue. */
7852 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7853 if (!can_compare_and_swap_p (mode
, false))
7857 enum rtx_code orig_code
= code
;
7858 struct atomic_op_functions optab
;
7860 get_atomic_op_for_code (&optab
, code
);
7861 libfunc
= optab_libfunc (after
? optab
.fetch_after
7862 : optab
.fetch_before
, mode
);
7864 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
7868 code
= optab
.reverse_code
;
7869 libfunc
= optab_libfunc (after
? optab
.fetch_before
7870 : optab
.fetch_after
, mode
);
7872 if (libfunc
!= NULL
)
7874 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
7875 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
7876 addr
, ptr_mode
, val
, mode
);
7878 if (!unused_result
&& fixup
)
7879 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
7880 true, OPTAB_LIB_WIDEN
);
7884 /* We need the original code for any further attempts. */
7888 /* If nothing else has succeeded, default to a compare and swap loop. */
7889 if (can_compare_and_swap_p (mode
, true))
7892 rtx t0
= gen_reg_rtx (mode
), t1
;
7896 /* If the result is used, get a register for it. */
7899 if (!target
|| !register_operand (target
, mode
))
7900 target
= gen_reg_rtx (mode
);
7901 /* If fetch_before, copy the value now. */
7903 emit_move_insn (target
, t0
);
7906 target
= const0_rtx
;
7911 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
7912 true, OPTAB_LIB_WIDEN
);
7913 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
7916 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
7919 /* For after, copy the value now. */
7920 if (!unused_result
&& after
)
7921 emit_move_insn (target
, t1
);
7922 insn
= get_insns ();
7925 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
7932 /* Return true if OPERAND is suitable for operand number OPNO of
7933 instruction ICODE. */
7936 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
7938 return (!insn_data
[(int) icode
].operand
[opno
].predicate
7939 || (insn_data
[(int) icode
].operand
[opno
].predicate
7940 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
7943 /* TARGET is a target of a multiword operation that we are going to
7944 implement as a series of word-mode operations. Return true if
7945 TARGET is suitable for this purpose. */
7948 valid_multiword_target_p (rtx target
)
7953 mode
= GET_MODE (target
);
7954 if (!GET_MODE_SIZE (mode
).is_constant (&size
))
7956 for (i
= 0; i
< size
; i
+= UNITS_PER_WORD
)
7957 if (!validate_subreg (word_mode
, mode
, target
, i
))
7962 /* Make OP describe an input operand that has value INTVAL and that has
7963 no inherent mode. This function should only be used for operands that
7964 are always expand-time constants. The backend may request that INTVAL
7965 be copied into a different kind of rtx, but it must specify the mode
7966 of that rtx if so. */
7969 create_integer_operand (class expand_operand
*op
, poly_int64 intval
)
7971 create_expand_operand (op
, EXPAND_INTEGER
,
7972 gen_int_mode (intval
, MAX_MODE_INT
),
7973 VOIDmode
, false, intval
);
7976 /* Like maybe_legitimize_operand, but do not change the code of the
7977 current rtx value. */
7980 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
7981 class expand_operand
*op
)
7983 /* See if the operand matches in its current form. */
7984 if (insn_operand_matches (icode
, opno
, op
->value
))
7987 /* If the operand is a memory whose address has no side effects,
7988 try forcing the address into a non-virtual pseudo register.
7989 The check for side effects is important because copy_to_mode_reg
7990 cannot handle things like auto-modified addresses. */
7991 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
7996 addr
= XEXP (mem
, 0);
7997 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
7998 && !side_effects_p (addr
))
8003 last
= get_last_insn ();
8004 mode
= get_address_mode (mem
);
8005 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
8006 if (insn_operand_matches (icode
, opno
, mem
))
8011 delete_insns_since (last
);
8018 /* Try to make OP match operand OPNO of instruction ICODE. Return true
8019 on success, storing the new operand value back in OP. */
8022 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
8023 class expand_operand
*op
)
8025 machine_mode mode
, imode
, tmode
;
8032 temporary_volatile_ok
v (true);
8033 return maybe_legitimize_operand_same_code (icode
, opno
, op
);
8037 gcc_assert (mode
!= VOIDmode
);
8039 && op
->value
!= const0_rtx
8040 && GET_MODE (op
->value
) == mode
8041 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
8044 op
->value
= gen_reg_rtx (mode
);
8050 gcc_assert (mode
!= VOIDmode
);
8051 gcc_assert (GET_MODE (op
->value
) == VOIDmode
8052 || GET_MODE (op
->value
) == mode
);
8053 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
8056 op
->value
= copy_to_mode_reg (mode
, op
->value
);
8059 case EXPAND_CONVERT_TO
:
8060 gcc_assert (mode
!= VOIDmode
);
8061 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
8064 case EXPAND_CONVERT_FROM
:
8065 if (GET_MODE (op
->value
) != VOIDmode
)
8066 mode
= GET_MODE (op
->value
);
8068 /* The caller must tell us what mode this value has. */
8069 gcc_assert (mode
!= VOIDmode
);
8071 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
8072 tmode
= (VECTOR_MODE_P (imode
) && !VECTOR_MODE_P (mode
)
8073 ? GET_MODE_INNER (imode
) : imode
);
8074 if (tmode
!= VOIDmode
&& tmode
!= mode
)
8076 op
->value
= convert_modes (tmode
, mode
, op
->value
, op
->unsigned_p
);
8079 if (imode
!= VOIDmode
&& imode
!= mode
)
8081 gcc_assert (VECTOR_MODE_P (imode
) && !VECTOR_MODE_P (mode
));
8082 op
->value
= expand_vector_broadcast (imode
, op
->value
);
8087 case EXPAND_ADDRESS
:
8088 op
->value
= convert_memory_address (as_a
<scalar_int_mode
> (mode
),
8092 case EXPAND_INTEGER
:
8093 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
8094 if (mode
!= VOIDmode
8095 && known_eq (trunc_int_for_mode (op
->int_value
, mode
),
8098 op
->value
= gen_int_mode (op
->int_value
, mode
);
8103 case EXPAND_UNDEFINED_INPUT
:
8104 /* See if the predicate accepts a SCRATCH rtx, which in this context
8105 indicates an undefined value. Use an uninitialized register if not. */
8106 if (!insn_operand_matches (icode
, opno
, op
->value
))
8108 op
->value
= gen_reg_rtx (op
->mode
);
8113 return insn_operand_matches (icode
, opno
, op
->value
);
8116 /* Make OP describe an input operand that should have the same value
8117 as VALUE, after any mode conversion that the target might request.
8118 TYPE is the type of VALUE. */
8121 create_convert_operand_from_type (class expand_operand
*op
,
8122 rtx value
, tree type
)
8124 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
8125 TYPE_UNSIGNED (type
));
8128 /* Return true if the requirements on operands OP1 and OP2 of instruction
8129 ICODE are similar enough for the result of legitimizing OP1 to be
8130 reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
8131 with OP1 and OP2 respectively. */
8134 can_reuse_operands_p (enum insn_code icode
,
8135 unsigned int opno1
, unsigned int opno2
,
8136 const class expand_operand
*op1
,
8137 const class expand_operand
*op2
)
8139 /* Check requirements that are common to all types. */
8140 if (op1
->type
!= op2
->type
8141 || op1
->mode
!= op2
->mode
8142 || (insn_data
[(int) icode
].operand
[opno1
].mode
8143 != insn_data
[(int) icode
].operand
[opno2
].mode
))
8146 /* Check the requirements for specific types. */
8150 case EXPAND_UNDEFINED_INPUT
:
8151 /* Outputs and undefined intputs must remain distinct. */
8156 case EXPAND_ADDRESS
:
8157 case EXPAND_INTEGER
:
8160 case EXPAND_CONVERT_TO
:
8161 case EXPAND_CONVERT_FROM
:
8162 return op1
->unsigned_p
== op2
->unsigned_p
;
8167 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8168 of instruction ICODE. Return true on success, leaving the new operand
8169 values in the OPS themselves. Emit no code on failure. */
8172 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
8173 unsigned int nops
, class expand_operand
*ops
)
8175 rtx_insn
*last
= get_last_insn ();
8176 rtx
*orig_values
= XALLOCAVEC (rtx
, nops
);
8177 for (unsigned int i
= 0; i
< nops
; i
++)
8179 orig_values
[i
] = ops
[i
].value
;
8181 /* First try reusing the result of an earlier legitimization.
8182 This avoids duplicate rtl and ensures that tied operands
8185 This search is linear, but NOPS is bounded at compile time
8186 to a small number (current a single digit). */
8189 if (can_reuse_operands_p (icode
, opno
+ j
, opno
+ i
, &ops
[j
], &ops
[i
])
8190 && rtx_equal_p (orig_values
[j
], orig_values
[i
])
8192 && insn_operand_matches (icode
, opno
+ i
, ops
[j
].value
))
8194 ops
[i
].value
= copy_rtx (ops
[j
].value
);
8198 /* Otherwise try legitimizing the operand on its own. */
8199 if (j
== i
&& !maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
8201 delete_insns_since (last
);
8208 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8209 as its operands. Return the instruction pattern on success,
8210 and emit any necessary set-up code. Return null and emit no
8214 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
8215 class expand_operand
*ops
)
8217 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
8218 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
8224 return GEN_FCN (icode
) ();
8226 return GEN_FCN (icode
) (ops
[0].value
);
8228 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
8230 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
8232 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8235 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8236 ops
[3].value
, ops
[4].value
);
8238 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8239 ops
[3].value
, ops
[4].value
, ops
[5].value
);
8241 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8242 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8245 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8246 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8247 ops
[6].value
, ops
[7].value
);
8249 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8250 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8251 ops
[6].value
, ops
[7].value
, ops
[8].value
);
8253 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8254 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8255 ops
[6].value
, ops
[7].value
, ops
[8].value
,
8258 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8259 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8260 ops
[6].value
, ops
[7].value
, ops
[8].value
,
8261 ops
[9].value
, ops
[10].value
);
8266 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8267 as its operands. Return true on success and emit no code on failure. */
8270 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
8271 class expand_operand
*ops
)
8273 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
8282 /* Like maybe_expand_insn, but for jumps. */
8285 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
8286 class expand_operand
*ops
)
8288 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
8291 emit_jump_insn (pat
);
8297 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8301 expand_insn (enum insn_code icode
, unsigned int nops
,
8302 class expand_operand
*ops
)
8304 if (!maybe_expand_insn (icode
, nops
, ops
))
8308 /* Like expand_insn, but for jumps. */
8311 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
8312 class expand_operand
*ops
)
8314 if (!maybe_expand_jump_insn (icode
, nops
, ops
))