1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2017 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
37 /* Include insn-config.h before expr.h so that HAVE_conditional_move
38 is properly defined. */
39 #include "stor-layout.h"
44 #include "optabs-tree.h"
47 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
49 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
50 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
52 /* Debug facility for use in GDB. */
53 void debug_optab_libfuncs (void);
55 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
56 the result of operation CODE applied to OP0 (and OP1 if it is a binary
59 If the last insn does not set TARGET, don't do anything, but return 1.
61 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
62 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
63 try again, ensuring that TARGET is not one of the operands. */
66 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
, rtx op1
)
72 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
74 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
75 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
76 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
77 && GET_RTX_CLASS (code
) != RTX_COMPARE
78 && GET_RTX_CLASS (code
) != RTX_UNARY
)
81 if (GET_CODE (target
) == ZERO_EXTRACT
)
84 for (last_insn
= insns
;
85 NEXT_INSN (last_insn
) != NULL_RTX
;
86 last_insn
= NEXT_INSN (last_insn
))
89 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
90 a value changing in the insn, so the note would be invalid for CSE. */
91 if (reg_overlap_mentioned_p (target
, op0
)
92 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
95 && (rtx_equal_p (target
, op0
)
96 || (op1
&& rtx_equal_p (target
, op1
))))
98 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
99 over expanding it as temp = MEM op X, MEM = temp. If the target
100 supports MEM = MEM op X instructions, it is sometimes too hard
101 to reconstruct that form later, especially if X is also a memory,
102 and due to multiple occurrences of addresses the address might
103 be forced into register unnecessarily.
104 Note that not emitting the REG_EQUIV note might inhibit
105 CSE in some cases. */
106 set
= single_set (last_insn
);
108 && GET_CODE (SET_SRC (set
)) == code
109 && MEM_P (SET_DEST (set
))
110 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
111 || (op1
&& rtx_equal_p (SET_DEST (set
),
112 XEXP (SET_SRC (set
), 1)))))
118 set
= set_for_reg_notes (last_insn
);
122 if (! rtx_equal_p (SET_DEST (set
), target
)
123 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
124 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
125 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
128 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
138 if (GET_MODE (op0
) != VOIDmode
&& GET_MODE (target
) != GET_MODE (op0
))
140 note
= gen_rtx_fmt_e (code
, GET_MODE (op0
), copy_rtx (op0
));
141 if (GET_MODE_SIZE (GET_MODE (op0
))
142 > GET_MODE_SIZE (GET_MODE (target
)))
143 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
144 note
, GET_MODE (op0
));
146 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
147 note
, GET_MODE (op0
));
152 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
156 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
158 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
163 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
164 for a widening operation would be. In most cases this would be OP0, but if
165 that's a constant it'll be VOIDmode, which isn't useful. */
168 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
170 machine_mode m0
= GET_MODE (op0
);
171 machine_mode m1
= GET_MODE (op1
);
174 if (m0
== VOIDmode
&& m1
== VOIDmode
)
176 else if (m0
== VOIDmode
|| GET_MODE_SIZE (m0
) < GET_MODE_SIZE (m1
))
181 if (GET_MODE_SIZE (result
) > GET_MODE_SIZE (to_mode
))
187 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
188 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
189 not actually do a sign-extend or zero-extend, but can leave the
190 higher-order bits of the result rtx undefined, for example, in the case
191 of logical operations, but not right shifts. */
194 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
195 int unsignedp
, int no_extend
)
199 /* If we don't have to extend and this is a constant, return it. */
200 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
203 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
204 extend since it will be more efficient to do so unless the signedness of
205 a promoted object differs from our extension. */
207 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
208 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
209 return convert_modes (mode
, oldmode
, op
, unsignedp
);
211 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
213 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
214 return gen_lowpart (mode
, force_reg (GET_MODE (op
), op
));
216 /* Otherwise, get an object of MODE, clobber it, and set the low-order
219 result
= gen_reg_rtx (mode
);
220 emit_clobber (result
);
221 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
225 /* Expand vector widening operations.
227 There are two different classes of operations handled here:
228 1) Operations whose result is wider than all the arguments to the operation.
229 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
230 In this case OP0 and optionally OP1 would be initialized,
231 but WIDE_OP wouldn't (not relevant for this case).
232 2) Operations whose result is of the same size as the last argument to the
233 operation, but wider than all the other arguments to the operation.
234 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
235 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
237 E.g, when called to expand the following operations, this is how
238 the arguments will be initialized:
240 widening-sum 2 oprnd0 - oprnd1
241 widening-dot-product 3 oprnd0 oprnd1 oprnd2
242 widening-mult 2 oprnd0 oprnd1 -
243 type-promotion (vec-unpack) 1 oprnd0 - - */
246 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
247 rtx target
, int unsignedp
)
249 struct expand_operand eops
[4];
250 tree oprnd0
, oprnd1
, oprnd2
;
251 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
252 optab widen_pattern_optab
;
253 enum insn_code icode
;
254 int nops
= TREE_CODE_LENGTH (ops
->code
);
258 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
259 widen_pattern_optab
=
260 optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
261 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
262 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
263 icode
= find_widening_optab_handler (widen_pattern_optab
,
264 TYPE_MODE (TREE_TYPE (ops
->op2
)),
267 icode
= optab_handler (widen_pattern_optab
, tmode0
);
268 gcc_assert (icode
!= CODE_FOR_nothing
);
273 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
276 /* The last operand is of a wider mode than the rest of the operands. */
281 gcc_assert (tmode1
== tmode0
);
284 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
288 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
289 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
291 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
293 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
294 expand_insn (icode
, op
, eops
);
295 return eops
[0].value
;
298 /* Generate code to perform an operation specified by TERNARY_OPTAB
299 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
301 UNSIGNEDP is for the case where we have to widen the operands
302 to perform the operation. It says to use zero-extension.
304 If TARGET is nonzero, the value
305 is generated there, if it is convenient to do so.
306 In all cases an rtx is returned for the locus of the value;
307 this may or may not be TARGET. */
310 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
311 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
313 struct expand_operand ops
[4];
314 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
316 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
318 create_output_operand (&ops
[0], target
, mode
);
319 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
320 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
321 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
322 expand_insn (icode
, 4, ops
);
327 /* Like expand_binop, but return a constant rtx if the result can be
328 calculated at compile time. The arguments and return value are
329 otherwise the same as for expand_binop. */
332 simplify_expand_binop (machine_mode mode
, optab binoptab
,
333 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
334 enum optab_methods methods
)
336 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
338 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
344 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
347 /* Like simplify_expand_binop, but always put the result in TARGET.
348 Return true if the expansion succeeded. */
351 force_expand_binop (machine_mode mode
, optab binoptab
,
352 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
353 enum optab_methods methods
)
355 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
356 target
, unsignedp
, methods
);
360 emit_move_insn (target
, x
);
364 /* Create a new vector value in VMODE with all elements set to OP. The
365 mode of OP must be the element mode of VMODE. If OP is a constant,
366 then the return value will be a constant. */
369 expand_vector_broadcast (machine_mode vmode
, rtx op
)
371 enum insn_code icode
;
376 gcc_checking_assert (VECTOR_MODE_P (vmode
));
378 n
= GET_MODE_NUNITS (vmode
);
379 vec
= rtvec_alloc (n
);
380 for (i
= 0; i
< n
; ++i
)
381 RTVEC_ELT (vec
, i
) = op
;
384 return gen_rtx_CONST_VECTOR (vmode
, vec
);
386 /* ??? If the target doesn't have a vec_init, then we have no easy way
387 of performing this operation. Most of this sort of generic support
388 is hidden away in the vector lowering support in gimple. */
389 icode
= optab_handler (vec_init_optab
, vmode
);
390 if (icode
== CODE_FOR_nothing
)
393 ret
= gen_reg_rtx (vmode
);
394 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
399 /* This subroutine of expand_doubleword_shift handles the cases in which
400 the effective shift value is >= BITS_PER_WORD. The arguments and return
401 value are the same as for the parent routine, except that SUPERWORD_OP1
402 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
403 INTO_TARGET may be null if the caller has decided to calculate it. */
406 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
407 rtx outof_target
, rtx into_target
,
408 int unsignedp
, enum optab_methods methods
)
410 if (into_target
!= 0)
411 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
412 into_target
, unsignedp
, methods
))
415 if (outof_target
!= 0)
417 /* For a signed right shift, we must fill OUTOF_TARGET with copies
418 of the sign bit, otherwise we must fill it with zeros. */
419 if (binoptab
!= ashr_optab
)
420 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
422 if (!force_expand_binop (word_mode
, binoptab
,
423 outof_input
, GEN_INT (BITS_PER_WORD
- 1),
424 outof_target
, unsignedp
, methods
))
430 /* This subroutine of expand_doubleword_shift handles the cases in which
431 the effective shift value is < BITS_PER_WORD. The arguments and return
432 value are the same as for the parent routine. */
435 expand_subword_shift (machine_mode op1_mode
, optab binoptab
,
436 rtx outof_input
, rtx into_input
, rtx op1
,
437 rtx outof_target
, rtx into_target
,
438 int unsignedp
, enum optab_methods methods
,
439 unsigned HOST_WIDE_INT shift_mask
)
441 optab reverse_unsigned_shift
, unsigned_shift
;
444 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
445 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
447 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
448 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
449 the opposite direction to BINOPTAB. */
450 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
452 carries
= outof_input
;
453 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
454 op1_mode
), op1_mode
);
455 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
460 /* We must avoid shifting by BITS_PER_WORD bits since that is either
461 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
462 has unknown behavior. Do a single shift first, then shift by the
463 remainder. It's OK to use ~OP1 as the remainder if shift counts
464 are truncated to the mode size. */
465 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
466 outof_input
, const1_rtx
, 0, unsignedp
, methods
);
467 if (shift_mask
== BITS_PER_WORD
- 1)
469 tmp
= immed_wide_int_const
470 (wi::minus_one (GET_MODE_PRECISION (op1_mode
)), op1_mode
);
471 tmp
= simplify_expand_binop (op1_mode
, xor_optab
, op1
, tmp
,
476 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
477 op1_mode
), op1_mode
);
478 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
482 if (tmp
== 0 || carries
== 0)
484 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
485 carries
, tmp
, 0, unsignedp
, methods
);
489 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
490 so the result can go directly into INTO_TARGET if convenient. */
491 tmp
= expand_binop (word_mode
, unsigned_shift
, into_input
, op1
,
492 into_target
, unsignedp
, methods
);
496 /* Now OR in the bits carried over from OUTOF_INPUT. */
497 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
498 into_target
, unsignedp
, methods
))
501 /* Use a standard word_mode shift for the out-of half. */
502 if (outof_target
!= 0)
503 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
504 outof_target
, unsignedp
, methods
))
511 /* Try implementing expand_doubleword_shift using conditional moves.
512 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
513 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
514 are the shift counts to use in the former and latter case. All other
515 arguments are the same as the parent routine. */
518 expand_doubleword_shift_condmove (machine_mode op1_mode
, optab binoptab
,
519 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
520 rtx outof_input
, rtx into_input
,
521 rtx subword_op1
, rtx superword_op1
,
522 rtx outof_target
, rtx into_target
,
523 int unsignedp
, enum optab_methods methods
,
524 unsigned HOST_WIDE_INT shift_mask
)
526 rtx outof_superword
, into_superword
;
528 /* Put the superword version of the output into OUTOF_SUPERWORD and
530 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
531 if (outof_target
!= 0 && subword_op1
== superword_op1
)
533 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
534 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
535 into_superword
= outof_target
;
536 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
537 outof_superword
, 0, unsignedp
, methods
))
542 into_superword
= gen_reg_rtx (word_mode
);
543 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
544 outof_superword
, into_superword
,
549 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
550 if (!expand_subword_shift (op1_mode
, binoptab
,
551 outof_input
, into_input
, subword_op1
,
552 outof_target
, into_target
,
553 unsignedp
, methods
, shift_mask
))
556 /* Select between them. Do the INTO half first because INTO_SUPERWORD
557 might be the current value of OUTOF_TARGET. */
558 if (!emit_conditional_move (into_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
559 into_target
, into_superword
, word_mode
, false))
562 if (outof_target
!= 0)
563 if (!emit_conditional_move (outof_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
564 outof_target
, outof_superword
,
571 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
572 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
573 input operand; the shift moves bits in the direction OUTOF_INPUT->
574 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
575 of the target. OP1 is the shift count and OP1_MODE is its mode.
576 If OP1 is constant, it will have been truncated as appropriate
577 and is known to be nonzero.
579 If SHIFT_MASK is zero, the result of word shifts is undefined when the
580 shift count is outside the range [0, BITS_PER_WORD). This routine must
581 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
583 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
584 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
585 fill with zeros or sign bits as appropriate.
587 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
588 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
589 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
590 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
593 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
594 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
595 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
596 function wants to calculate it itself.
598 Return true if the shift could be successfully synthesized. */
601 expand_doubleword_shift (machine_mode op1_mode
, optab binoptab
,
602 rtx outof_input
, rtx into_input
, rtx op1
,
603 rtx outof_target
, rtx into_target
,
604 int unsignedp
, enum optab_methods methods
,
605 unsigned HOST_WIDE_INT shift_mask
)
607 rtx superword_op1
, tmp
, cmp1
, cmp2
;
608 enum rtx_code cmp_code
;
610 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
611 fill the result with sign or zero bits as appropriate. If so, the value
612 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
613 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
614 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
616 This isn't worthwhile for constant shifts since the optimizers will
617 cope better with in-range shift counts. */
618 if (shift_mask
>= BITS_PER_WORD
620 && !CONSTANT_P (op1
))
622 if (!expand_doubleword_shift (op1_mode
, binoptab
,
623 outof_input
, into_input
, op1
,
625 unsignedp
, methods
, shift_mask
))
627 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
628 outof_target
, unsignedp
, methods
))
633 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
634 is true when the effective shift value is less than BITS_PER_WORD.
635 Set SUPERWORD_OP1 to the shift count that should be used to shift
636 OUTOF_INPUT into INTO_TARGET when the condition is false. */
637 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
638 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
640 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
641 is a subword shift count. */
642 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
644 cmp2
= CONST0_RTX (op1_mode
);
650 /* Set CMP1 to OP1 - BITS_PER_WORD. */
651 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
653 cmp2
= CONST0_RTX (op1_mode
);
655 superword_op1
= cmp1
;
660 /* If we can compute the condition at compile time, pick the
661 appropriate subroutine. */
662 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
663 if (tmp
!= 0 && CONST_INT_P (tmp
))
665 if (tmp
== const0_rtx
)
666 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
667 outof_target
, into_target
,
670 return expand_subword_shift (op1_mode
, binoptab
,
671 outof_input
, into_input
, op1
,
672 outof_target
, into_target
,
673 unsignedp
, methods
, shift_mask
);
676 /* Try using conditional moves to generate straight-line code. */
677 if (HAVE_conditional_move
)
679 rtx_insn
*start
= get_last_insn ();
680 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
681 cmp_code
, cmp1
, cmp2
,
682 outof_input
, into_input
,
684 outof_target
, into_target
,
685 unsignedp
, methods
, shift_mask
))
687 delete_insns_since (start
);
690 /* As a last resort, use branches to select the correct alternative. */
691 rtx_code_label
*subword_label
= gen_label_rtx ();
692 rtx_code_label
*done_label
= gen_label_rtx ();
695 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
697 profile_probability::uninitialized ());
700 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
701 outof_target
, into_target
,
705 emit_jump_insn (targetm
.gen_jump (done_label
));
707 emit_label (subword_label
);
709 if (!expand_subword_shift (op1_mode
, binoptab
,
710 outof_input
, into_input
, op1
,
711 outof_target
, into_target
,
712 unsignedp
, methods
, shift_mask
))
715 emit_label (done_label
);
719 /* Subroutine of expand_binop. Perform a double word multiplication of
720 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
721 as the target's word_mode. This function return NULL_RTX if anything
722 goes wrong, in which case it may have already emitted instructions
723 which need to be deleted.
725 If we want to multiply two two-word values and have normal and widening
726 multiplies of single-word values, we can do this with three smaller
729 The multiplication proceeds as follows:
730 _______________________
731 [__op0_high_|__op0_low__]
732 _______________________
733 * [__op1_high_|__op1_low__]
734 _______________________________________________
735 _______________________
736 (1) [__op0_low__*__op1_low__]
737 _______________________
738 (2a) [__op0_low__*__op1_high_]
739 _______________________
740 (2b) [__op0_high_*__op1_low__]
741 _______________________
742 (3) [__op0_high_*__op1_high_]
745 This gives a 4-word result. Since we are only interested in the
746 lower 2 words, partial result (3) and the upper words of (2a) and
747 (2b) don't need to be calculated. Hence (2a) and (2b) can be
748 calculated using non-widening multiplication.
750 (1), however, needs to be calculated with an unsigned widening
751 multiplication. If this operation is not directly supported we
752 try using a signed widening multiplication and adjust the result.
753 This adjustment works as follows:
755 If both operands are positive then no adjustment is needed.
757 If the operands have different signs, for example op0_low < 0 and
758 op1_low >= 0, the instruction treats the most significant bit of
759 op0_low as a sign bit instead of a bit with significance
760 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
761 with 2**BITS_PER_WORD - op0_low, and two's complements the
762 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
765 Similarly, if both operands are negative, we need to add
766 (op0_low + op1_low) * 2**BITS_PER_WORD.
768 We use a trick to adjust quickly. We logically shift op0_low right
769 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
770 op0_high (op1_high) before it is used to calculate 2b (2a). If no
771 logical shift exists, we do an arithmetic right shift and subtract
775 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
776 bool umulp
, enum optab_methods methods
)
778 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
779 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
780 rtx wordm1
= umulp
? NULL_RTX
: GEN_INT (BITS_PER_WORD
- 1);
781 rtx product
, adjust
, product_high
, temp
;
783 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
784 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
785 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
786 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
788 /* If we're using an unsigned multiply to directly compute the product
789 of the low-order words of the operands and perform any required
790 adjustments of the operands, we begin by trying two more multiplications
791 and then computing the appropriate sum.
793 We have checked above that the required addition is provided.
794 Full-word addition will normally always succeed, especially if
795 it is provided at all, so we don't worry about its failure. The
796 multiplication may well fail, however, so we do handle that. */
800 /* ??? This could be done with emit_store_flag where available. */
801 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
802 NULL_RTX
, 1, methods
);
804 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
805 NULL_RTX
, 0, OPTAB_DIRECT
);
808 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
809 NULL_RTX
, 0, methods
);
812 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
813 NULL_RTX
, 0, OPTAB_DIRECT
);
820 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
821 NULL_RTX
, 0, OPTAB_DIRECT
);
825 /* OP0_HIGH should now be dead. */
829 /* ??? This could be done with emit_store_flag where available. */
830 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
831 NULL_RTX
, 1, methods
);
833 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
834 NULL_RTX
, 0, OPTAB_DIRECT
);
837 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
838 NULL_RTX
, 0, methods
);
841 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
842 NULL_RTX
, 0, OPTAB_DIRECT
);
849 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
850 NULL_RTX
, 0, OPTAB_DIRECT
);
854 /* OP1_HIGH should now be dead. */
856 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
857 NULL_RTX
, 0, OPTAB_DIRECT
);
859 if (target
&& !REG_P (target
))
863 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
864 target
, 1, OPTAB_DIRECT
);
866 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
867 target
, 1, OPTAB_DIRECT
);
872 product_high
= operand_subword (product
, high
, 1, mode
);
873 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
874 NULL_RTX
, 0, OPTAB_DIRECT
);
875 emit_move_insn (product_high
, adjust
);
879 /* Wrapper around expand_binop which takes an rtx code to specify
880 the operation to perform, not an optab pointer. All other
881 arguments are the same. */
883 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
884 rtx op1
, rtx target
, int unsignedp
,
885 enum optab_methods methods
)
887 optab binop
= code_to_optab (code
);
890 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
893 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
894 binop. Order them according to commutative_operand_precedence and, if
895 possible, try to put TARGET or a pseudo first. */
897 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
899 int op0_prec
= commutative_operand_precedence (op0
);
900 int op1_prec
= commutative_operand_precedence (op1
);
902 if (op0_prec
< op1_prec
)
905 if (op0_prec
> op1_prec
)
908 /* With equal precedence, both orders are ok, but it is better if the
909 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
910 if (target
== 0 || REG_P (target
))
911 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
913 return rtx_equal_p (op1
, target
);
916 /* Return true if BINOPTAB implements a shift operation. */
919 shift_optab_p (optab binoptab
)
921 switch (optab_to_code (binoptab
))
937 /* Return true if BINOPTAB implements a commutative binary operation. */
940 commutative_optab_p (optab binoptab
)
942 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
943 || binoptab
== smul_widen_optab
944 || binoptab
== umul_widen_optab
945 || binoptab
== smul_highpart_optab
946 || binoptab
== umul_highpart_optab
);
949 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
950 optimizing, and if the operand is a constant that costs more than
951 1 instruction, force the constant into a register and return that
952 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
955 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
956 int opn
, rtx x
, bool unsignedp
)
958 bool speed
= optimize_insn_for_speed_p ();
963 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
964 > set_src_cost (x
, mode
, speed
)))
968 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
969 if (intval
!= INTVAL (x
))
970 x
= GEN_INT (intval
);
973 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
974 x
= force_reg (mode
, x
);
979 /* Helper function for expand_binop: handle the case where there
980 is an insn that directly implements the indicated operation.
981 Returns null if this is not possible. */
983 expand_binop_directly (machine_mode mode
, optab binoptab
,
985 rtx target
, int unsignedp
, enum optab_methods methods
,
988 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
989 enum insn_code icode
= find_widening_optab_handler (binoptab
, mode
,
991 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
992 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
993 machine_mode mode0
, mode1
, tmp_mode
;
994 struct expand_operand ops
[3];
997 rtx xop0
= op0
, xop1
= op1
;
998 bool canonicalize_op1
= false;
1000 /* If it is a commutative operator and the modes would match
1001 if we would swap the operands, we can save the conversions. */
1002 commutative_p
= commutative_optab_p (binoptab
);
1004 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1005 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode1
)
1006 std::swap (xop0
, xop1
);
1008 /* If we are optimizing, force expensive constants into a register. */
1009 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1010 if (!shift_optab_p (binoptab
))
1011 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1013 /* Shifts and rotates often use a different mode for op1 from op0;
1014 for VOIDmode constants we don't know the mode, so force it
1015 to be canonicalized using convert_modes. */
1016 canonicalize_op1
= true;
1018 /* In case the insn wants input operands in modes different from
1019 those of the actual operands, convert the operands. It would
1020 seem that we don't need to convert CONST_INTs, but we do, so
1021 that they're properly zero-extended, sign-extended or truncated
1024 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1025 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1027 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1031 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1032 ? GET_MODE (xop1
) : mode
);
1033 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1035 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1039 /* If operation is commutative,
1040 try to make the first operand a register.
1041 Even better, try to make it the same as the target.
1042 Also try to make the last operand a constant. */
1044 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1045 std::swap (xop0
, xop1
);
1047 /* Now, if insn's predicates don't allow our operands, put them into
1050 if (binoptab
== vec_pack_trunc_optab
1051 || binoptab
== vec_pack_usat_optab
1052 || binoptab
== vec_pack_ssat_optab
1053 || binoptab
== vec_pack_ufix_trunc_optab
1054 || binoptab
== vec_pack_sfix_trunc_optab
)
1056 /* The mode of the result is different then the mode of the
1058 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1059 if (VECTOR_MODE_P (mode
)
1060 && GET_MODE_NUNITS (tmp_mode
) != 2 * GET_MODE_NUNITS (mode
))
1062 delete_insns_since (last
);
1069 create_output_operand (&ops
[0], target
, tmp_mode
);
1070 create_input_operand (&ops
[1], xop0
, mode0
);
1071 create_input_operand (&ops
[2], xop1
, mode1
);
1072 pat
= maybe_gen_insn (icode
, 3, ops
);
1075 /* If PAT is composed of more than one insn, try to add an appropriate
1076 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1077 operand, call expand_binop again, this time without a target. */
1078 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1079 && ! add_equal_note (pat
, ops
[0].value
,
1080 optab_to_code (binoptab
),
1081 ops
[1].value
, ops
[2].value
))
1083 delete_insns_since (last
);
1084 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1085 unsignedp
, methods
);
1089 return ops
[0].value
;
1091 delete_insns_since (last
);
1095 /* Generate code to perform an operation specified by BINOPTAB
1096 on operands OP0 and OP1, with result having machine-mode MODE.
1098 UNSIGNEDP is for the case where we have to widen the operands
1099 to perform the operation. It says to use zero-extension.
1101 If TARGET is nonzero, the value
1102 is generated there, if it is convenient to do so.
1103 In all cases an rtx is returned for the locus of the value;
1104 this may or may not be TARGET. */
1107 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1108 rtx target
, int unsignedp
, enum optab_methods methods
)
1110 enum optab_methods next_methods
1111 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1112 ? OPTAB_WIDEN
: methods
);
1113 enum mode_class mclass
;
1114 machine_mode wider_mode
;
1117 rtx_insn
*entry_last
= get_last_insn ();
1120 mclass
= GET_MODE_CLASS (mode
);
1122 /* If subtracting an integer constant, convert this into an addition of
1123 the negated constant. */
1125 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1127 op1
= negate_rtx (mode
, op1
);
1128 binoptab
= add_optab
;
1130 /* For shifts, constant invalid op1 might be expanded from different
1131 mode than MODE. As those are invalid, force them to a register
1132 to avoid further problems during expansion. */
1133 else if (CONST_INT_P (op1
)
1134 && shift_optab_p (binoptab
)
1135 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1137 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1138 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1141 /* Record where to delete back to if we backtrack. */
1142 last
= get_last_insn ();
1144 /* If we can do it with a three-operand insn, do so. */
1146 if (methods
!= OPTAB_MUST_WIDEN
1147 && find_widening_optab_handler (binoptab
, mode
,
1148 widened_mode (mode
, op0
, op1
), 1)
1149 != CODE_FOR_nothing
)
1151 temp
= expand_binop_directly (mode
, binoptab
, op0
, op1
, target
,
1152 unsignedp
, methods
, last
);
1157 /* If we were trying to rotate, and that didn't work, try rotating
1158 the other direction before falling back to shifts and bitwise-or. */
1159 if (((binoptab
== rotl_optab
1160 && optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
1161 || (binoptab
== rotr_optab
1162 && optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
))
1163 && mclass
== MODE_INT
)
1165 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1167 unsigned int bits
= GET_MODE_PRECISION (mode
);
1169 if (CONST_INT_P (op1
))
1170 newop1
= GEN_INT (bits
- INTVAL (op1
));
1171 else if (targetm
.shift_truncation_mask (mode
) == bits
- 1)
1172 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1174 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1175 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1176 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1178 temp
= expand_binop_directly (mode
, otheroptab
, op0
, newop1
,
1179 target
, unsignedp
, methods
, last
);
1184 /* If this is a multiply, see if we can do a widening operation that
1185 takes operands of this mode and makes a wider mode. */
1187 if (binoptab
== smul_optab
1188 && GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
1189 && (widening_optab_handler ((unsignedp
? umul_widen_optab
1190 : smul_widen_optab
),
1191 GET_MODE_2XWIDER_MODE (mode
), mode
)
1192 != CODE_FOR_nothing
))
1194 temp
= expand_binop (GET_MODE_2XWIDER_MODE (mode
),
1195 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1196 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1200 if (GET_MODE_CLASS (mode
) == MODE_INT
1201 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1202 return gen_lowpart (mode
, temp
);
1204 return convert_to_mode (mode
, temp
, unsignedp
);
1208 /* If this is a vector shift by a scalar, see if we can do a vector
1209 shift by a vector. If so, broadcast the scalar into a vector. */
1210 if (mclass
== MODE_VECTOR_INT
)
1212 optab otheroptab
= unknown_optab
;
1214 if (binoptab
== ashl_optab
)
1215 otheroptab
= vashl_optab
;
1216 else if (binoptab
== ashr_optab
)
1217 otheroptab
= vashr_optab
;
1218 else if (binoptab
== lshr_optab
)
1219 otheroptab
= vlshr_optab
;
1220 else if (binoptab
== rotl_optab
)
1221 otheroptab
= vrotl_optab
;
1222 else if (binoptab
== rotr_optab
)
1223 otheroptab
= vrotr_optab
;
1225 if (otheroptab
&& optab_handler (otheroptab
, mode
) != CODE_FOR_nothing
)
1227 /* The scalar may have been extended to be too wide. Truncate
1228 it back to the proper size to fit in the broadcast vector. */
1229 machine_mode inner_mode
= GET_MODE_INNER (mode
);
1230 if (!CONST_INT_P (op1
)
1231 && (GET_MODE_BITSIZE (inner_mode
)
1232 < GET_MODE_BITSIZE (GET_MODE (op1
))))
1233 op1
= force_reg (inner_mode
,
1234 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1236 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1239 temp
= expand_binop_directly (mode
, otheroptab
, op0
, vop1
,
1240 target
, unsignedp
, methods
, last
);
1247 /* Look for a wider mode of the same class for which we think we
1248 can open-code the operation. Check for a widening multiply at the
1249 wider mode as well. */
1251 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1252 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1253 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
1254 wider_mode
!= VOIDmode
;
1255 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
1257 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1258 || (binoptab
== smul_optab
1259 && GET_MODE_WIDER_MODE (wider_mode
) != VOIDmode
1260 && (find_widening_optab_handler ((unsignedp
1262 : smul_widen_optab
),
1263 GET_MODE_WIDER_MODE (wider_mode
),
1265 != CODE_FOR_nothing
)))
1267 rtx xop0
= op0
, xop1
= op1
;
1270 /* For certain integer operations, we need not actually extend
1271 the narrow operands, as long as we will truncate
1272 the results to the same narrowness. */
1274 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1275 || binoptab
== xor_optab
1276 || binoptab
== add_optab
|| binoptab
== sub_optab
1277 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1278 && mclass
== MODE_INT
)
1281 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1283 if (binoptab
!= ashl_optab
)
1284 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1288 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1290 /* The second operand of a shift must always be extended. */
1291 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1292 no_extend
&& binoptab
!= ashl_optab
);
1294 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1295 unsignedp
, OPTAB_DIRECT
);
1298 if (mclass
!= MODE_INT
1299 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1302 target
= gen_reg_rtx (mode
);
1303 convert_move (target
, temp
, 0);
1307 return gen_lowpart (mode
, temp
);
1310 delete_insns_since (last
);
1314 /* If operation is commutative,
1315 try to make the first operand a register.
1316 Even better, try to make it the same as the target.
1317 Also try to make the last operand a constant. */
1318 if (commutative_optab_p (binoptab
)
1319 && swap_commutative_operands_with_target (target
, op0
, op1
))
1320 std::swap (op0
, op1
);
1322 /* These can be done a word at a time. */
1323 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1324 && mclass
== MODE_INT
1325 && GET_MODE_SIZE (mode
) > UNITS_PER_WORD
1326 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1331 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1332 won't be accurate, so use a new target. */
1336 || !valid_multiword_target_p (target
))
1337 target
= gen_reg_rtx (mode
);
1341 /* Do the actual arithmetic. */
1342 for (i
= 0; i
< GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
; i
++)
1344 rtx target_piece
= operand_subword (target
, i
, 1, mode
);
1345 rtx x
= expand_binop (word_mode
, binoptab
,
1346 operand_subword_force (op0
, i
, mode
),
1347 operand_subword_force (op1
, i
, mode
),
1348 target_piece
, unsignedp
, next_methods
);
1353 if (target_piece
!= x
)
1354 emit_move_insn (target_piece
, x
);
1357 insns
= get_insns ();
1360 if (i
== GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
)
1367 /* Synthesize double word shifts from single word shifts. */
1368 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1369 || binoptab
== ashr_optab
)
1370 && mclass
== MODE_INT
1371 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1372 && GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
1373 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
)
1374 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1375 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1376 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1378 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1379 machine_mode op1_mode
;
1381 double_shift_mask
= targetm
.shift_truncation_mask (mode
);
1382 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1383 op1_mode
= GET_MODE (op1
) != VOIDmode
? GET_MODE (op1
) : word_mode
;
1385 /* Apply the truncation to constant shifts. */
1386 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1387 op1
= GEN_INT (INTVAL (op1
) & double_shift_mask
);
1389 if (op1
== CONST0_RTX (op1_mode
))
1392 /* Make sure that this is a combination that expand_doubleword_shift
1393 can handle. See the comments there for details. */
1394 if (double_shift_mask
== 0
1395 || (shift_mask
== BITS_PER_WORD
- 1
1396 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1399 rtx into_target
, outof_target
;
1400 rtx into_input
, outof_input
;
1401 int left_shift
, outof_word
;
1403 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1404 won't be accurate, so use a new target. */
1408 || !valid_multiword_target_p (target
))
1409 target
= gen_reg_rtx (mode
);
1413 /* OUTOF_* is the word we are shifting bits away from, and
1414 INTO_* is the word that we are shifting bits towards, thus
1415 they differ depending on the direction of the shift and
1416 WORDS_BIG_ENDIAN. */
1418 left_shift
= binoptab
== ashl_optab
;
1419 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1421 outof_target
= operand_subword (target
, outof_word
, 1, mode
);
1422 into_target
= operand_subword (target
, 1 - outof_word
, 1, mode
);
1424 outof_input
= operand_subword_force (op0
, outof_word
, mode
);
1425 into_input
= operand_subword_force (op0
, 1 - outof_word
, mode
);
1427 if (expand_doubleword_shift (op1_mode
, binoptab
,
1428 outof_input
, into_input
, op1
,
1429 outof_target
, into_target
,
1430 unsignedp
, next_methods
, shift_mask
))
1432 insns
= get_insns ();
1442 /* Synthesize double word rotates from single word shifts. */
1443 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1444 && mclass
== MODE_INT
1445 && CONST_INT_P (op1
)
1446 && GET_MODE_PRECISION (mode
) == 2 * BITS_PER_WORD
1447 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1448 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1451 rtx into_target
, outof_target
;
1452 rtx into_input
, outof_input
;
1454 int shift_count
, left_shift
, outof_word
;
1456 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1457 won't be accurate, so use a new target. Do this also if target is not
1458 a REG, first because having a register instead may open optimization
1459 opportunities, and second because if target and op0 happen to be MEMs
1460 designating the same location, we would risk clobbering it too early
1461 in the code sequence we generate below. */
1466 || !valid_multiword_target_p (target
))
1467 target
= gen_reg_rtx (mode
);
1471 shift_count
= INTVAL (op1
);
1473 /* OUTOF_* is the word we are shifting bits away from, and
1474 INTO_* is the word that we are shifting bits towards, thus
1475 they differ depending on the direction of the shift and
1476 WORDS_BIG_ENDIAN. */
1478 left_shift
= (binoptab
== rotl_optab
);
1479 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1481 outof_target
= operand_subword (target
, outof_word
, 1, mode
);
1482 into_target
= operand_subword (target
, 1 - outof_word
, 1, mode
);
1484 outof_input
= operand_subword_force (op0
, outof_word
, mode
);
1485 into_input
= operand_subword_force (op0
, 1 - outof_word
, mode
);
1487 if (shift_count
== BITS_PER_WORD
)
1489 /* This is just a word swap. */
1490 emit_move_insn (outof_target
, into_input
);
1491 emit_move_insn (into_target
, outof_input
);
1496 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1497 rtx first_shift_count
, second_shift_count
;
1498 optab reverse_unsigned_shift
, unsigned_shift
;
1500 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1501 ? lshr_optab
: ashl_optab
);
1503 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1504 ? ashl_optab
: lshr_optab
);
1506 if (shift_count
> BITS_PER_WORD
)
1508 first_shift_count
= GEN_INT (shift_count
- BITS_PER_WORD
);
1509 second_shift_count
= GEN_INT (2 * BITS_PER_WORD
- shift_count
);
1513 first_shift_count
= GEN_INT (BITS_PER_WORD
- shift_count
);
1514 second_shift_count
= GEN_INT (shift_count
);
1517 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1518 outof_input
, first_shift_count
,
1519 NULL_RTX
, unsignedp
, next_methods
);
1520 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1521 into_input
, second_shift_count
,
1522 NULL_RTX
, unsignedp
, next_methods
);
1524 if (into_temp1
!= 0 && into_temp2
!= 0)
1525 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1526 into_target
, unsignedp
, next_methods
);
1530 if (inter
!= 0 && inter
!= into_target
)
1531 emit_move_insn (into_target
, inter
);
1533 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1534 into_input
, first_shift_count
,
1535 NULL_RTX
, unsignedp
, next_methods
);
1536 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1537 outof_input
, second_shift_count
,
1538 NULL_RTX
, unsignedp
, next_methods
);
1540 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1541 inter
= expand_binop (word_mode
, ior_optab
,
1542 outof_temp1
, outof_temp2
,
1543 outof_target
, unsignedp
, next_methods
);
1545 if (inter
!= 0 && inter
!= outof_target
)
1546 emit_move_insn (outof_target
, inter
);
1549 insns
= get_insns ();
1559 /* These can be done a word at a time by propagating carries. */
1560 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1561 && mclass
== MODE_INT
1562 && GET_MODE_SIZE (mode
) >= 2 * UNITS_PER_WORD
1563 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1566 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1567 const unsigned int nwords
= GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
;
1568 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1569 rtx xop0
, xop1
, xtarget
;
1571 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1572 value is one of those, use it. Otherwise, use 1 since it is the
1573 one easiest to get. */
1574 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1575 int normalizep
= STORE_FLAG_VALUE
;
1580 /* Prepare the operands. */
1581 xop0
= force_reg (mode
, op0
);
1582 xop1
= force_reg (mode
, op1
);
1584 xtarget
= gen_reg_rtx (mode
);
1586 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1589 /* Indicate for flow that the entire target reg is being set. */
1591 emit_clobber (xtarget
);
1593 /* Do the actual arithmetic. */
1594 for (i
= 0; i
< nwords
; i
++)
1596 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
1597 rtx target_piece
= operand_subword (xtarget
, index
, 1, mode
);
1598 rtx op0_piece
= operand_subword_force (xop0
, index
, mode
);
1599 rtx op1_piece
= operand_subword_force (xop1
, index
, mode
);
1602 /* Main add/subtract of the input operands. */
1603 x
= expand_binop (word_mode
, binoptab
,
1604 op0_piece
, op1_piece
,
1605 target_piece
, unsignedp
, next_methods
);
1611 /* Store carry from main add/subtract. */
1612 carry_out
= gen_reg_rtx (word_mode
);
1613 carry_out
= emit_store_flag_force (carry_out
,
1614 (binoptab
== add_optab
1617 word_mode
, 1, normalizep
);
1624 /* Add/subtract previous carry to main result. */
1625 newx
= expand_binop (word_mode
,
1626 normalizep
== 1 ? binoptab
: otheroptab
,
1628 NULL_RTX
, 1, next_methods
);
1632 /* Get out carry from adding/subtracting carry in. */
1633 rtx carry_tmp
= gen_reg_rtx (word_mode
);
1634 carry_tmp
= emit_store_flag_force (carry_tmp
,
1635 (binoptab
== add_optab
1638 word_mode
, 1, normalizep
);
1640 /* Logical-ior the two poss. carry together. */
1641 carry_out
= expand_binop (word_mode
, ior_optab
,
1642 carry_out
, carry_tmp
,
1643 carry_out
, 0, next_methods
);
1647 emit_move_insn (target_piece
, newx
);
1651 if (x
!= target_piece
)
1652 emit_move_insn (target_piece
, x
);
1655 carry_in
= carry_out
;
1658 if (i
== GET_MODE_BITSIZE (mode
) / (unsigned) BITS_PER_WORD
)
1660 if (optab_handler (mov_optab
, mode
) != CODE_FOR_nothing
1661 || ! rtx_equal_p (target
, xtarget
))
1663 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
1665 set_dst_reg_note (temp
, REG_EQUAL
,
1666 gen_rtx_fmt_ee (optab_to_code (binoptab
),
1667 mode
, copy_rtx (xop0
),
1678 delete_insns_since (last
);
1681 /* Attempt to synthesize double word multiplies using a sequence of word
1682 mode multiplications. We first attempt to generate a sequence using a
1683 more efficient unsigned widening multiply, and if that fails we then
1684 try using a signed widening multiply. */
1686 if (binoptab
== smul_optab
1687 && mclass
== MODE_INT
1688 && GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
1689 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
1690 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
1692 rtx product
= NULL_RTX
;
1693 if (widening_optab_handler (umul_widen_optab
, mode
, word_mode
)
1694 != CODE_FOR_nothing
)
1696 product
= expand_doubleword_mult (mode
, op0
, op1
, target
,
1699 delete_insns_since (last
);
1702 if (product
== NULL_RTX
1703 && widening_optab_handler (smul_widen_optab
, mode
, word_mode
)
1704 != CODE_FOR_nothing
)
1706 product
= expand_doubleword_mult (mode
, op0
, op1
, target
,
1709 delete_insns_since (last
);
1712 if (product
!= NULL_RTX
)
1714 if (optab_handler (mov_optab
, mode
) != CODE_FOR_nothing
)
1716 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
1718 set_dst_reg_note (move
,
1720 gen_rtx_fmt_ee (MULT
, mode
,
1723 target
? target
: product
);
1729 /* It can't be open-coded in this mode.
1730 Use a library call if one is available and caller says that's ok. */
1732 libfunc
= optab_libfunc (binoptab
, mode
);
1734 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
1738 machine_mode op1_mode
= mode
;
1743 if (shift_optab_p (binoptab
))
1745 op1_mode
= targetm
.libgcc_shift_count_mode ();
1746 /* Specify unsigned here,
1747 since negative shift counts are meaningless. */
1748 op1x
= convert_to_mode (op1_mode
, op1
, 1);
1751 if (GET_MODE (op0
) != VOIDmode
1752 && GET_MODE (op0
) != mode
)
1753 op0
= convert_to_mode (mode
, op0
, unsignedp
);
1755 /* Pass 1 for NO_QUEUE so we don't lose any increments
1756 if the libcall is cse'd or moved. */
1757 value
= emit_library_call_value (libfunc
,
1758 NULL_RTX
, LCT_CONST
, mode
, 2,
1759 op0
, mode
, op1x
, op1_mode
);
1761 insns
= get_insns ();
1764 bool trapv
= trapv_binoptab_p (binoptab
);
1765 target
= gen_reg_rtx (mode
);
1766 emit_libcall_block_1 (insns
, target
, value
,
1768 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
1769 mode
, op0
, op1
), trapv
);
1774 delete_insns_since (last
);
1776 /* It can't be done in this mode. Can we do it in a wider mode? */
1778 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
1779 || methods
== OPTAB_MUST_WIDEN
))
1781 /* Caller says, don't even try. */
1782 delete_insns_since (entry_last
);
1786 /* Compute the value of METHODS to pass to recursive calls.
1787 Don't allow widening to be tried recursively. */
1789 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
1791 /* Look for a wider mode of the same class for which it appears we can do
1794 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1796 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
1797 wider_mode
!= VOIDmode
;
1798 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
1800 if (find_widening_optab_handler (binoptab
, wider_mode
, mode
, 1)
1802 || (methods
== OPTAB_LIB
1803 && optab_libfunc (binoptab
, wider_mode
)))
1805 rtx xop0
= op0
, xop1
= op1
;
1808 /* For certain integer operations, we need not actually extend
1809 the narrow operands, as long as we will truncate
1810 the results to the same narrowness. */
1812 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1813 || binoptab
== xor_optab
1814 || binoptab
== add_optab
|| binoptab
== sub_optab
1815 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1816 && mclass
== MODE_INT
)
1819 xop0
= widen_operand (xop0
, wider_mode
, mode
,
1820 unsignedp
, no_extend
);
1822 /* The second operand of a shift must always be extended. */
1823 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1824 no_extend
&& binoptab
!= ashl_optab
);
1826 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1827 unsignedp
, methods
);
1830 if (mclass
!= MODE_INT
1831 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1834 target
= gen_reg_rtx (mode
);
1835 convert_move (target
, temp
, 0);
1839 return gen_lowpart (mode
, temp
);
1842 delete_insns_since (last
);
1847 delete_insns_since (entry_last
);
1851 /* Expand a binary operator which has both signed and unsigned forms.
1852 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1855 If we widen unsigned operands, we may use a signed wider operation instead
1856 of an unsigned wider operation, since the result would be the same. */
1859 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
1860 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
1861 enum optab_methods methods
)
1864 optab direct_optab
= unsignedp
? uoptab
: soptab
;
1867 /* Do it without widening, if possible. */
1868 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1869 unsignedp
, OPTAB_DIRECT
);
1870 if (temp
|| methods
== OPTAB_DIRECT
)
1873 /* Try widening to a signed int. Disable any direct use of any
1874 signed insn in the current mode. */
1875 save_enable
= swap_optab_enable (soptab
, mode
, false);
1877 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1878 unsignedp
, OPTAB_WIDEN
);
1880 /* For unsigned operands, try widening to an unsigned int. */
1881 if (!temp
&& unsignedp
)
1882 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1883 unsignedp
, OPTAB_WIDEN
);
1884 if (temp
|| methods
== OPTAB_WIDEN
)
1887 /* Use the right width libcall if that exists. */
1888 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1889 unsignedp
, OPTAB_LIB
);
1890 if (temp
|| methods
== OPTAB_LIB
)
1893 /* Must widen and use a libcall, use either signed or unsigned. */
1894 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1895 unsignedp
, methods
);
1896 if (!temp
&& unsignedp
)
1897 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1898 unsignedp
, methods
);
1901 /* Undo the fiddling above. */
1903 swap_optab_enable (soptab
, mode
, true);
1907 /* Generate code to perform an operation specified by UNOPPTAB
1908 on operand OP0, with two results to TARG0 and TARG1.
1909 We assume that the order of the operands for the instruction
1910 is TARG0, TARG1, OP0.
1912 Either TARG0 or TARG1 may be zero, but what that means is that
1913 the result is not actually wanted. We will generate it into
1914 a dummy pseudo-reg and discard it. They may not both be zero.
1916 Returns 1 if this operation can be performed; 0 if not. */
1919 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
1922 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
1923 enum mode_class mclass
;
1924 machine_mode wider_mode
;
1925 rtx_insn
*entry_last
= get_last_insn ();
1928 mclass
= GET_MODE_CLASS (mode
);
1931 targ0
= gen_reg_rtx (mode
);
1933 targ1
= gen_reg_rtx (mode
);
1935 /* Record where to go back to if we fail. */
1936 last
= get_last_insn ();
1938 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
1940 struct expand_operand ops
[3];
1941 enum insn_code icode
= optab_handler (unoptab
, mode
);
1943 create_fixed_operand (&ops
[0], targ0
);
1944 create_fixed_operand (&ops
[1], targ1
);
1945 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
1946 if (maybe_expand_insn (icode
, 3, ops
))
1950 /* It can't be done in this mode. Can we do it in a wider mode? */
1952 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1954 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
1955 wider_mode
!= VOIDmode
;
1956 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
1958 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
1960 rtx t0
= gen_reg_rtx (wider_mode
);
1961 rtx t1
= gen_reg_rtx (wider_mode
);
1962 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
1964 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
1966 convert_move (targ0
, t0
, unsignedp
);
1967 convert_move (targ1
, t1
, unsignedp
);
1971 delete_insns_since (last
);
1976 delete_insns_since (entry_last
);
1980 /* Generate code to perform an operation specified by BINOPTAB
1981 on operands OP0 and OP1, with two results to TARG1 and TARG2.
1982 We assume that the order of the operands for the instruction
1983 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
1984 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
1986 Either TARG0 or TARG1 may be zero, but what that means is that
1987 the result is not actually wanted. We will generate it into
1988 a dummy pseudo-reg and discard it. They may not both be zero.
1990 Returns 1 if this operation can be performed; 0 if not. */
1993 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
1996 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
1997 enum mode_class mclass
;
1998 machine_mode wider_mode
;
1999 rtx_insn
*entry_last
= get_last_insn ();
2002 mclass
= GET_MODE_CLASS (mode
);
2005 targ0
= gen_reg_rtx (mode
);
2007 targ1
= gen_reg_rtx (mode
);
2009 /* Record where to go back to if we fail. */
2010 last
= get_last_insn ();
2012 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2014 struct expand_operand ops
[4];
2015 enum insn_code icode
= optab_handler (binoptab
, mode
);
2016 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2017 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2018 rtx xop0
= op0
, xop1
= op1
;
2020 /* If we are optimizing, force expensive constants into a register. */
2021 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2022 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2024 create_fixed_operand (&ops
[0], targ0
);
2025 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2026 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
2027 create_fixed_operand (&ops
[3], targ1
);
2028 if (maybe_expand_insn (icode
, 4, ops
))
2030 delete_insns_since (last
);
2033 /* It can't be done in this mode. Can we do it in a wider mode? */
2035 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2037 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
2038 wider_mode
!= VOIDmode
;
2039 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2041 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2043 rtx t0
= gen_reg_rtx (wider_mode
);
2044 rtx t1
= gen_reg_rtx (wider_mode
);
2045 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2046 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2048 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2051 convert_move (targ0
, t0
, unsignedp
);
2052 convert_move (targ1
, t1
, unsignedp
);
2056 delete_insns_since (last
);
2061 delete_insns_since (entry_last
);
2065 /* Expand the two-valued library call indicated by BINOPTAB, but
2066 preserve only one of the values. If TARG0 is non-NULL, the first
2067 value is placed into TARG0; otherwise the second value is placed
2068 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2069 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2070 This routine assumes that the value returned by the library call is
2071 as if the return value was of an integral mode twice as wide as the
2072 mode of OP0. Returns 1 if the call was successful. */
2075 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2076 rtx targ0
, rtx targ1
, enum rtx_code code
)
2079 machine_mode libval_mode
;
2084 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2085 gcc_assert (!targ0
!= !targ1
);
2087 mode
= GET_MODE (op0
);
2088 libfunc
= optab_libfunc (binoptab
, mode
);
2092 /* The value returned by the library function will have twice as
2093 many bits as the nominal MODE. */
2094 libval_mode
= smallest_mode_for_size (2 * GET_MODE_BITSIZE (mode
),
2097 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2101 /* Get the part of VAL containing the value that we want. */
2102 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2103 targ0
? 0 : GET_MODE_SIZE (mode
));
2104 insns
= get_insns ();
2106 /* Move the into the desired location. */
2107 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2108 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2114 /* Wrapper around expand_unop which takes an rtx code to specify
2115 the operation to perform, not an optab pointer. All other
2116 arguments are the same. */
2118 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2119 rtx target
, int unsignedp
)
2121 optab unop
= code_to_optab (code
);
2124 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2130 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2132 A similar operation can be used for clrsb. UNOPTAB says which operation
2133 we are trying to expand. */
2135 widen_leading (machine_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2137 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2138 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2140 machine_mode wider_mode
;
2141 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
2142 wider_mode
!= VOIDmode
;
2143 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2145 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2150 last
= get_last_insn ();
2153 target
= gen_reg_rtx (mode
);
2154 xop0
= widen_operand (op0
, wider_mode
, mode
,
2155 unoptab
!= clrsb_optab
, false);
2156 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2157 unoptab
!= clrsb_optab
);
2160 (wider_mode
, sub_optab
, temp
,
2161 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2162 - GET_MODE_PRECISION (mode
),
2164 target
, true, OPTAB_DIRECT
);
2166 delete_insns_since (last
);
2175 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2176 quantities, choosing which based on whether the high word is nonzero. */
2178 expand_doubleword_clz (machine_mode mode
, rtx op0
, rtx target
)
2180 rtx xop0
= force_reg (mode
, op0
);
2181 rtx subhi
= gen_highpart (word_mode
, xop0
);
2182 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2183 rtx_code_label
*hi0_label
= gen_label_rtx ();
2184 rtx_code_label
*after_label
= gen_label_rtx ();
2188 /* If we were not given a target, use a word_mode register, not a
2189 'mode' register. The result will fit, and nobody is expecting
2190 anything bigger (the return type of __builtin_clz* is int). */
2192 target
= gen_reg_rtx (word_mode
);
2194 /* In any case, write to a word_mode scratch in both branches of the
2195 conditional, so we can ensure there is a single move insn setting
2196 'target' to tag a REG_EQUAL note on. */
2197 result
= gen_reg_rtx (word_mode
);
2201 /* If the high word is not equal to zero,
2202 then clz of the full value is clz of the high word. */
2203 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2204 word_mode
, true, hi0_label
);
2206 temp
= expand_unop_direct (word_mode
, clz_optab
, subhi
, result
, true);
2211 convert_move (result
, temp
, true);
2213 emit_jump_insn (targetm
.gen_jump (after_label
));
2216 /* Else clz of the full value is clz of the low word plus the number
2217 of bits in the high word. */
2218 emit_label (hi0_label
);
2220 temp
= expand_unop_direct (word_mode
, clz_optab
, sublo
, 0, true);
2223 temp
= expand_binop (word_mode
, add_optab
, temp
,
2224 gen_int_mode (GET_MODE_BITSIZE (word_mode
), word_mode
),
2225 result
, true, OPTAB_DIRECT
);
2229 convert_move (result
, temp
, true);
2231 emit_label (after_label
);
2232 convert_move (target
, result
, true);
2237 add_equal_note (seq
, target
, CLZ
, xop0
, 0);
2246 /* Try calculating popcount of a double-word quantity as two popcount's of
2247 word-sized quantities and summing up the results. */
2249 expand_doubleword_popcount (machine_mode mode
, rtx op0
, rtx target
)
2256 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2257 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2259 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2260 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2268 /* If we were not given a target, use a word_mode register, not a
2269 'mode' register. The result will fit, and nobody is expecting
2270 anything bigger (the return type of __builtin_popcount* is int). */
2272 target
= gen_reg_rtx (word_mode
);
2274 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2279 add_equal_note (seq
, t
, POPCOUNT
, op0
, 0);
2287 (parity:narrow (low (x) ^ high (x))) */
2289 expand_doubleword_parity (machine_mode mode
, rtx op0
, rtx target
)
2291 rtx t
= expand_binop (word_mode
, xor_optab
,
2292 operand_subword_force (op0
, 0, mode
),
2293 operand_subword_force (op0
, 1, mode
),
2294 NULL_RTX
, 0, OPTAB_DIRECT
);
2295 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2301 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2303 widen_bswap (machine_mode mode
, rtx op0
, rtx target
)
2305 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2306 machine_mode wider_mode
;
2310 if (!CLASS_HAS_WIDER_MODES_P (mclass
))
2313 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
2314 wider_mode
!= VOIDmode
;
2315 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2316 if (optab_handler (bswap_optab
, wider_mode
) != CODE_FOR_nothing
)
2321 last
= get_last_insn ();
2323 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2324 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2326 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2327 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2329 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2330 GET_MODE_BITSIZE (wider_mode
)
2331 - GET_MODE_BITSIZE (mode
),
2337 target
= gen_reg_rtx (mode
);
2338 emit_move_insn (target
, gen_lowpart (mode
, x
));
2341 delete_insns_since (last
);
2346 /* Try calculating bswap as two bswaps of two word-sized operands. */
2349 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2353 t1
= expand_unop (word_mode
, bswap_optab
,
2354 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2355 t0
= expand_unop (word_mode
, bswap_optab
,
2356 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2358 if (target
== 0 || !valid_multiword_target_p (target
))
2359 target
= gen_reg_rtx (mode
);
2361 emit_clobber (target
);
2362 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2363 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2368 /* Try calculating (parity x) as (and (popcount x) 1), where
2369 popcount can also be done in a wider mode. */
2371 expand_parity (machine_mode mode
, rtx op0
, rtx target
)
2373 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2374 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2376 machine_mode wider_mode
;
2377 for (wider_mode
= mode
; wider_mode
!= VOIDmode
;
2378 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2380 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2385 last
= get_last_insn ();
2387 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2388 target
= gen_reg_rtx (wider_mode
);
2390 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2391 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2394 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2395 target
, true, OPTAB_DIRECT
);
2399 if (mclass
!= MODE_INT
2400 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2401 return convert_to_mode (mode
, temp
, 0);
2403 return gen_lowpart (mode
, temp
);
2406 delete_insns_since (last
);
2413 /* Try calculating ctz(x) as K - clz(x & -x) ,
2414 where K is GET_MODE_PRECISION(mode) - 1.
2416 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2417 don't have to worry about what the hardware does in that case. (If
2418 the clz instruction produces the usual value at 0, which is K, the
2419 result of this code sequence will be -1; expand_ffs, below, relies
2420 on this. It might be nice to have it be K instead, for consistency
2421 with the (very few) processors that provide a ctz with a defined
2422 value, but that would take one more instruction, and it would be
2423 less convenient for expand_ffs anyway. */
2426 expand_ctz (machine_mode mode
, rtx op0
, rtx target
)
2431 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2436 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2438 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2439 true, OPTAB_DIRECT
);
2441 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2443 temp
= expand_binop (mode
, sub_optab
,
2444 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
2446 true, OPTAB_DIRECT
);
2456 add_equal_note (seq
, temp
, CTZ
, op0
, 0);
2462 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2463 else with the sequence used by expand_clz.
2465 The ffs builtin promises to return zero for a zero value and ctz/clz
2466 may have an undefined value in that case. If they do not give us a
2467 convenient value, we have to generate a test and branch. */
2469 expand_ffs (machine_mode mode
, rtx op0
, rtx target
)
2471 HOST_WIDE_INT val
= 0;
2472 bool defined_at_zero
= false;
2476 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
2480 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
2484 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
2486 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
2489 temp
= expand_ctz (mode
, op0
, 0);
2493 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
2495 defined_at_zero
= true;
2496 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
2502 if (defined_at_zero
&& val
== -1)
2503 /* No correction needed at zero. */;
2506 /* We don't try to do anything clever with the situation found
2507 on some processors (eg Alpha) where ctz(0:mode) ==
2508 bitsize(mode). If someone can think of a way to send N to -1
2509 and leave alone all values in the range 0..N-1 (where N is a
2510 power of two), cheaper than this test-and-branch, please add it.
2512 The test-and-branch is done after the operation itself, in case
2513 the operation sets condition codes that can be recycled for this.
2514 (This is true on i386, for instance.) */
2516 rtx_code_label
*nonzero_label
= gen_label_rtx ();
2517 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
2518 mode
, true, nonzero_label
);
2520 convert_move (temp
, GEN_INT (-1), false);
2521 emit_label (nonzero_label
);
2524 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2525 to produce a value in the range 0..bitsize. */
2526 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
2527 target
, false, OPTAB_DIRECT
);
2534 add_equal_note (seq
, temp
, FFS
, op0
, 0);
2543 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2544 conditions, VAL may already be a SUBREG against which we cannot generate
2545 a further SUBREG. In this case, we expect forcing the value into a
2546 register will work around the situation. */
2549 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
2553 ret
= lowpart_subreg (omode
, val
, imode
);
2556 val
= force_reg (imode
, val
);
2557 ret
= lowpart_subreg (omode
, val
, imode
);
2558 gcc_assert (ret
!= NULL
);
2563 /* Expand a floating point absolute value or negation operation via a
2564 logical operation on the sign bit. */
2567 expand_absneg_bit (enum rtx_code code
, machine_mode mode
,
2568 rtx op0
, rtx target
)
2570 const struct real_format
*fmt
;
2571 int bitpos
, word
, nwords
, i
;
2576 /* The format has to have a simple sign bit. */
2577 fmt
= REAL_MODE_FORMAT (mode
);
2581 bitpos
= fmt
->signbit_rw
;
2585 /* Don't create negative zeros if the format doesn't support them. */
2586 if (code
== NEG
&& !fmt
->has_signed_zero
)
2589 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
2591 imode
= int_mode_for_mode (mode
);
2592 if (imode
== BLKmode
)
2601 if (FLOAT_WORDS_BIG_ENDIAN
)
2602 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
2604 word
= bitpos
/ BITS_PER_WORD
;
2605 bitpos
= bitpos
% BITS_PER_WORD
;
2606 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
2609 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
2615 || (nwords
> 1 && !valid_multiword_target_p (target
)))
2616 target
= gen_reg_rtx (mode
);
2622 for (i
= 0; i
< nwords
; ++i
)
2624 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
2625 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
2629 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2631 immed_wide_int_const (mask
, imode
),
2632 targ_piece
, 1, OPTAB_LIB_WIDEN
);
2633 if (temp
!= targ_piece
)
2634 emit_move_insn (targ_piece
, temp
);
2637 emit_move_insn (targ_piece
, op0_piece
);
2640 insns
= get_insns ();
2647 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2648 gen_lowpart (imode
, op0
),
2649 immed_wide_int_const (mask
, imode
),
2650 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
2651 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
2653 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
2654 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
2661 /* As expand_unop, but will fail rather than attempt the operation in a
2662 different mode or with a libcall. */
2664 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2667 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2669 struct expand_operand ops
[2];
2670 enum insn_code icode
= optab_handler (unoptab
, mode
);
2671 rtx_insn
*last
= get_last_insn ();
2674 create_output_operand (&ops
[0], target
, mode
);
2675 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2676 pat
= maybe_gen_insn (icode
, 2, ops
);
2679 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
2680 && ! add_equal_note (pat
, ops
[0].value
,
2681 optab_to_code (unoptab
),
2682 ops
[1].value
, NULL_RTX
))
2684 delete_insns_since (last
);
2685 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
2690 return ops
[0].value
;
2696 /* Generate code to perform an operation specified by UNOPTAB
2697 on operand OP0, with result having machine-mode MODE.
2699 UNSIGNEDP is for the case where we have to widen the operands
2700 to perform the operation. It says to use zero-extension.
2702 If TARGET is nonzero, the value
2703 is generated there, if it is convenient to do so.
2704 In all cases an rtx is returned for the locus of the value;
2705 this may or may not be TARGET. */
2708 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2711 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2712 machine_mode wider_mode
;
2716 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
2720 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2722 /* Widening (or narrowing) clz needs special treatment. */
2723 if (unoptab
== clz_optab
)
2725 temp
= widen_leading (mode
, op0
, target
, unoptab
);
2729 if (GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
2730 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2732 temp
= expand_doubleword_clz (mode
, op0
, target
);
2740 if (unoptab
== clrsb_optab
)
2742 temp
= widen_leading (mode
, op0
, target
, unoptab
);
2748 if (unoptab
== popcount_optab
2749 && GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
2750 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2751 && optimize_insn_for_speed_p ())
2753 temp
= expand_doubleword_popcount (mode
, op0
, target
);
2758 if (unoptab
== parity_optab
2759 && GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
2760 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2761 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
2762 && optimize_insn_for_speed_p ())
2764 temp
= expand_doubleword_parity (mode
, op0
, target
);
2769 /* Widening (or narrowing) bswap needs special treatment. */
2770 if (unoptab
== bswap_optab
)
2772 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2773 or ROTATERT. First try these directly; if this fails, then try the
2774 obvious pair of shifts with allowed widening, as this will probably
2775 be always more efficient than the other fallback methods. */
2781 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
2783 temp
= expand_binop (mode
, rotl_optab
, op0
, GEN_INT (8), target
,
2784 unsignedp
, OPTAB_DIRECT
);
2789 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
2791 temp
= expand_binop (mode
, rotr_optab
, op0
, GEN_INT (8), target
,
2792 unsignedp
, OPTAB_DIRECT
);
2797 last
= get_last_insn ();
2799 temp1
= expand_binop (mode
, ashl_optab
, op0
, GEN_INT (8), NULL_RTX
,
2800 unsignedp
, OPTAB_WIDEN
);
2801 temp2
= expand_binop (mode
, lshr_optab
, op0
, GEN_INT (8), NULL_RTX
,
2802 unsignedp
, OPTAB_WIDEN
);
2805 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
2806 unsignedp
, OPTAB_WIDEN
);
2811 delete_insns_since (last
);
2814 temp
= widen_bswap (mode
, op0
, target
);
2818 if (GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
2819 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2821 temp
= expand_doubleword_bswap (mode
, op0
, target
);
2829 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2830 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
2831 wider_mode
!= VOIDmode
;
2832 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2834 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2837 rtx_insn
*last
= get_last_insn ();
2839 /* For certain operations, we need not actually extend
2840 the narrow operand, as long as we will truncate the
2841 results to the same narrowness. */
2843 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
2844 (unoptab
== neg_optab
2845 || unoptab
== one_cmpl_optab
)
2846 && mclass
== MODE_INT
);
2848 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2853 if (mclass
!= MODE_INT
2854 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2857 target
= gen_reg_rtx (mode
);
2858 convert_move (target
, temp
, 0);
2862 return gen_lowpart (mode
, temp
);
2865 delete_insns_since (last
);
2869 /* These can be done a word at a time. */
2870 if (unoptab
== one_cmpl_optab
2871 && mclass
== MODE_INT
2872 && GET_MODE_SIZE (mode
) > UNITS_PER_WORD
2873 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2878 if (target
== 0 || target
== op0
|| !valid_multiword_target_p (target
))
2879 target
= gen_reg_rtx (mode
);
2883 /* Do the actual arithmetic. */
2884 for (i
= 0; i
< GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
; i
++)
2886 rtx target_piece
= operand_subword (target
, i
, 1, mode
);
2887 rtx x
= expand_unop (word_mode
, unoptab
,
2888 operand_subword_force (op0
, i
, mode
),
2889 target_piece
, unsignedp
);
2891 if (target_piece
!= x
)
2892 emit_move_insn (target_piece
, x
);
2895 insns
= get_insns ();
2902 if (optab_to_code (unoptab
) == NEG
)
2904 /* Try negating floating point values by flipping the sign bit. */
2905 if (SCALAR_FLOAT_MODE_P (mode
))
2907 temp
= expand_absneg_bit (NEG
, mode
, op0
, target
);
2912 /* If there is no negation pattern, and we have no negative zero,
2913 try subtracting from zero. */
2914 if (!HONOR_SIGNED_ZEROS (mode
))
2916 temp
= expand_binop (mode
, (unoptab
== negv_optab
2917 ? subv_optab
: sub_optab
),
2918 CONST0_RTX (mode
), op0
, target
,
2919 unsignedp
, OPTAB_DIRECT
);
2925 /* Try calculating parity (x) as popcount (x) % 2. */
2926 if (unoptab
== parity_optab
)
2928 temp
= expand_parity (mode
, op0
, target
);
2933 /* Try implementing ffs (x) in terms of clz (x). */
2934 if (unoptab
== ffs_optab
)
2936 temp
= expand_ffs (mode
, op0
, target
);
2941 /* Try implementing ctz (x) in terms of clz (x). */
2942 if (unoptab
== ctz_optab
)
2944 temp
= expand_ctz (mode
, op0
, target
);
2950 /* Now try a library call in this mode. */
2951 libfunc
= optab_libfunc (unoptab
, mode
);
2957 machine_mode outmode
= mode
;
2959 /* All of these functions return small values. Thus we choose to
2960 have them return something that isn't a double-word. */
2961 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
2962 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
2963 || unoptab
== parity_optab
)
2965 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
2966 optab_libfunc (unoptab
, mode
)));
2970 /* Pass 1 for NO_QUEUE so we don't lose any increments
2971 if the libcall is cse'd or moved. */
2972 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
2974 insns
= get_insns ();
2977 target
= gen_reg_rtx (outmode
);
2978 bool trapv
= trapv_unoptab_p (unoptab
);
2980 eq_value
= NULL_RTX
;
2983 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
2984 if (GET_MODE_SIZE (outmode
) < GET_MODE_SIZE (mode
))
2985 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
2986 else if (GET_MODE_SIZE (outmode
) > GET_MODE_SIZE (mode
))
2987 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
2988 outmode
, eq_value
, mode
);
2990 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
2995 /* It can't be done in this mode. Can we do it in a wider mode? */
2997 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2999 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
3000 wider_mode
!= VOIDmode
;
3001 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
3003 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
3004 || optab_libfunc (unoptab
, wider_mode
))
3007 rtx_insn
*last
= get_last_insn ();
3009 /* For certain operations, we need not actually extend
3010 the narrow operand, as long as we will truncate the
3011 results to the same narrowness. */
3012 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3013 (unoptab
== neg_optab
3014 || unoptab
== one_cmpl_optab
3015 || unoptab
== bswap_optab
)
3016 && mclass
== MODE_INT
);
3018 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3021 /* If we are generating clz using wider mode, adjust the
3022 result. Similarly for clrsb. */
3023 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3026 (wider_mode
, sub_optab
, temp
,
3027 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
3028 - GET_MODE_PRECISION (mode
),
3030 target
, true, OPTAB_DIRECT
);
3032 /* Likewise for bswap. */
3033 if (unoptab
== bswap_optab
&& temp
!= 0)
3035 gcc_assert (GET_MODE_PRECISION (wider_mode
)
3036 == GET_MODE_BITSIZE (wider_mode
)
3037 && GET_MODE_PRECISION (mode
)
3038 == GET_MODE_BITSIZE (mode
));
3040 temp
= expand_shift (RSHIFT_EXPR
, wider_mode
, temp
,
3041 GET_MODE_BITSIZE (wider_mode
)
3042 - GET_MODE_BITSIZE (mode
),
3048 if (mclass
!= MODE_INT
)
3051 target
= gen_reg_rtx (mode
);
3052 convert_move (target
, temp
, 0);
3056 return gen_lowpart (mode
, temp
);
3059 delete_insns_since (last
);
3064 /* One final attempt at implementing negation via subtraction,
3065 this time allowing widening of the operand. */
3066 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3069 temp
= expand_binop (mode
,
3070 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3071 CONST0_RTX (mode
), op0
,
3072 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3080 /* Emit code to compute the absolute value of OP0, with result to
3081 TARGET if convenient. (TARGET may be 0.) The return value says
3082 where the result actually is to be found.
3084 MODE is the mode of the operand; the mode of the result is
3085 different but can be deduced from MODE.
3090 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3091 int result_unsignedp
)
3095 if (GET_MODE_CLASS (mode
) != MODE_INT
3097 result_unsignedp
= 1;
3099 /* First try to do it with a special abs instruction. */
3100 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3105 /* For floating point modes, try clearing the sign bit. */
3106 if (SCALAR_FLOAT_MODE_P (mode
))
3108 temp
= expand_absneg_bit (ABS
, mode
, op0
, target
);
3113 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3114 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3115 && !HONOR_SIGNED_ZEROS (mode
))
3117 rtx_insn
*last
= get_last_insn ();
3119 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3122 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3128 delete_insns_since (last
);
3131 /* If this machine has expensive jumps, we can do integer absolute
3132 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3133 where W is the width of MODE. */
3135 if (GET_MODE_CLASS (mode
) == MODE_INT
3136 && BRANCH_COST (optimize_insn_for_speed_p (),
3139 rtx extended
= expand_shift (RSHIFT_EXPR
, mode
, op0
,
3140 GET_MODE_PRECISION (mode
) - 1,
3143 temp
= expand_binop (mode
, xor_optab
, extended
, op0
, target
, 0,
3146 temp
= expand_binop (mode
, result_unsignedp
? sub_optab
: subv_optab
,
3147 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3157 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3158 int result_unsignedp
, int safe
)
3161 rtx_code_label
*op1
;
3163 if (GET_MODE_CLASS (mode
) != MODE_INT
3165 result_unsignedp
= 1;
3167 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3171 /* If that does not win, use conditional jump and negate. */
3173 /* It is safe to use the target if it is the same
3174 as the source if this is also a pseudo register */
3175 if (op0
== target
&& REG_P (op0
)
3176 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3179 op1
= gen_label_rtx ();
3180 if (target
== 0 || ! safe
3181 || GET_MODE (target
) != mode
3182 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3184 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3185 target
= gen_reg_rtx (mode
);
3187 emit_move_insn (target
, op0
);
3190 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3191 NULL_RTX
, NULL
, op1
,
3192 profile_probability::uninitialized ());
3194 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3197 emit_move_insn (target
, op0
);
3203 /* Emit code to compute the one's complement absolute value of OP0
3204 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3205 (TARGET may be NULL_RTX.) The return value says where the result
3206 actually is to be found.
3208 MODE is the mode of the operand; the mode of the result is
3209 different but can be deduced from MODE. */
3212 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3216 /* Not applicable for floating point modes. */
3217 if (FLOAT_MODE_P (mode
))
3220 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3221 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3223 rtx_insn
*last
= get_last_insn ();
3225 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3227 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3233 delete_insns_since (last
);
3236 /* If this machine has expensive jumps, we can do one's complement
3237 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3239 if (GET_MODE_CLASS (mode
) == MODE_INT
3240 && BRANCH_COST (optimize_insn_for_speed_p (),
3243 rtx extended
= expand_shift (RSHIFT_EXPR
, mode
, op0
,
3244 GET_MODE_PRECISION (mode
) - 1,
3247 temp
= expand_binop (mode
, xor_optab
, extended
, op0
, target
, 0,
3257 /* A subroutine of expand_copysign, perform the copysign operation using the
3258 abs and neg primitives advertised to exist on the target. The assumption
3259 is that we have a split register file, and leaving op0 in fp registers,
3260 and not playing with subregs so much, will help the register allocator. */
3263 expand_copysign_absneg (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
3264 int bitpos
, bool op0_is_abs
)
3267 enum insn_code icode
;
3269 rtx_code_label
*label
;
3274 /* Check if the back end provides an insn that handles signbit for the
3276 icode
= optab_handler (signbit_optab
, mode
);
3277 if (icode
!= CODE_FOR_nothing
)
3279 imode
= insn_data
[(int) icode
].operand
[0].mode
;
3280 sign
= gen_reg_rtx (imode
);
3281 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3285 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3287 imode
= int_mode_for_mode (mode
);
3288 if (imode
== BLKmode
)
3290 op1
= gen_lowpart (imode
, op1
);
3297 if (FLOAT_WORDS_BIG_ENDIAN
)
3298 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3300 word
= bitpos
/ BITS_PER_WORD
;
3301 bitpos
= bitpos
% BITS_PER_WORD
;
3302 op1
= operand_subword_force (op1
, word
, mode
);
3305 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3306 sign
= expand_binop (imode
, and_optab
, op1
,
3307 immed_wide_int_const (mask
, imode
),
3308 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3313 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3320 if (target
== NULL_RTX
)
3321 target
= copy_to_reg (op0
);
3323 emit_move_insn (target
, op0
);
3326 label
= gen_label_rtx ();
3327 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3329 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3330 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3332 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3334 emit_move_insn (target
, op0
);
3342 /* A subroutine of expand_copysign, perform the entire copysign operation
3343 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3344 is true if op0 is known to have its sign bit clear. */
3347 expand_copysign_bit (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
3348 int bitpos
, bool op0_is_abs
)
3351 int word
, nwords
, i
;
3355 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3357 imode
= int_mode_for_mode (mode
);
3358 if (imode
== BLKmode
)
3367 if (FLOAT_WORDS_BIG_ENDIAN
)
3368 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3370 word
= bitpos
/ BITS_PER_WORD
;
3371 bitpos
= bitpos
% BITS_PER_WORD
;
3372 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3375 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3380 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3381 target
= gen_reg_rtx (mode
);
3387 for (i
= 0; i
< nwords
; ++i
)
3389 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3390 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3396 = expand_binop (imode
, and_optab
, op0_piece
,
3397 immed_wide_int_const (~mask
, imode
),
3398 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3399 op1
= expand_binop (imode
, and_optab
,
3400 operand_subword_force (op1
, i
, mode
),
3401 immed_wide_int_const (mask
, imode
),
3402 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3404 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
3405 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3406 if (temp
!= targ_piece
)
3407 emit_move_insn (targ_piece
, temp
);
3410 emit_move_insn (targ_piece
, op0_piece
);
3413 insns
= get_insns ();
3420 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
3421 immed_wide_int_const (mask
, imode
),
3422 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3424 op0
= gen_lowpart (imode
, op0
);
3426 op0
= expand_binop (imode
, and_optab
, op0
,
3427 immed_wide_int_const (~mask
, imode
),
3428 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3430 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
3431 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3432 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3438 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3439 scalar floating point mode. Return NULL if we do not know how to
3440 expand the operation inline. */
3443 expand_copysign (rtx op0
, rtx op1
, rtx target
)
3445 machine_mode mode
= GET_MODE (op0
);
3446 const struct real_format
*fmt
;
3450 gcc_assert (SCALAR_FLOAT_MODE_P (mode
));
3451 gcc_assert (GET_MODE (op1
) == mode
);
3453 /* First try to do it with a special instruction. */
3454 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
3455 target
, 0, OPTAB_DIRECT
);
3459 fmt
= REAL_MODE_FORMAT (mode
);
3460 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
3464 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3466 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
3467 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
3471 if (fmt
->signbit_ro
>= 0
3472 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
3473 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
3474 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
3476 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
3477 fmt
->signbit_ro
, op0_is_abs
);
3482 if (fmt
->signbit_rw
< 0)
3484 return expand_copysign_bit (mode
, op0
, op1
, target
,
3485 fmt
->signbit_rw
, op0_is_abs
);
3488 /* Generate an instruction whose insn-code is INSN_CODE,
3489 with two operands: an output TARGET and an input OP0.
3490 TARGET *must* be nonzero, and the output is always stored there.
3491 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3492 the value that is stored into TARGET.
3494 Return false if expansion failed. */
3497 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
3500 struct expand_operand ops
[2];
3503 create_output_operand (&ops
[0], target
, GET_MODE (target
));
3504 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
3505 pat
= maybe_gen_insn (icode
, 2, ops
);
3509 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3511 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
);
3515 if (ops
[0].value
!= target
)
3516 emit_move_insn (target
, ops
[0].value
);
3519 /* Generate an instruction whose insn-code is INSN_CODE,
3520 with two operands: an output TARGET and an input OP0.
3521 TARGET *must* be nonzero, and the output is always stored there.
3522 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3523 the value that is stored into TARGET. */
3526 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
3528 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
3532 struct no_conflict_data
3535 rtx_insn
*first
, *insn
;
3539 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3540 the currently examined clobber / store has to stay in the list of
3541 insns that constitute the actual libcall block. */
3543 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
3545 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
3547 /* If this inns directly contributes to setting the target, it must stay. */
3548 if (reg_overlap_mentioned_p (p
->target
, dest
))
3549 p
->must_stay
= true;
3550 /* If we haven't committed to keeping any other insns in the list yet,
3551 there is nothing more to check. */
3552 else if (p
->insn
== p
->first
)
3554 /* If this insn sets / clobbers a register that feeds one of the insns
3555 already in the list, this insn has to stay too. */
3556 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
3557 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
3558 || reg_used_between_p (dest
, p
->first
, p
->insn
)
3559 /* Likewise if this insn depends on a register set by a previous
3560 insn in the list, or if it sets a result (presumably a hard
3561 register) that is set or clobbered by a previous insn.
3562 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3563 SET_DEST perform the former check on the address, and the latter
3564 check on the MEM. */
3565 || (GET_CODE (set
) == SET
3566 && (modified_in_p (SET_SRC (set
), p
->first
)
3567 || modified_in_p (SET_DEST (set
), p
->first
)
3568 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
3569 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
3570 p
->must_stay
= true;
3574 /* Emit code to make a call to a constant function or a library call.
3576 INSNS is a list containing all insns emitted in the call.
3577 These insns leave the result in RESULT. Our block is to copy RESULT
3578 to TARGET, which is logically equivalent to EQUIV.
3580 We first emit any insns that set a pseudo on the assumption that these are
3581 loading constants into registers; doing so allows them to be safely cse'ed
3582 between blocks. Then we emit all the other insns in the block, followed by
3583 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3584 note with an operand of EQUIV. */
3587 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
3588 bool equiv_may_trap
)
3590 rtx final_dest
= target
;
3591 rtx_insn
*next
, *last
, *insn
;
3593 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3594 into a MEM later. Protect the libcall block from this change. */
3595 if (! REG_P (target
) || REG_USERVAR_P (target
))
3596 target
= gen_reg_rtx (GET_MODE (target
));
3598 /* If we're using non-call exceptions, a libcall corresponding to an
3599 operation that may trap may also trap. */
3600 /* ??? See the comment in front of make_reg_eh_region_note. */
3601 if (cfun
->can_throw_non_call_exceptions
3602 && (equiv_may_trap
|| may_trap_p (equiv
)))
3604 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3607 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
3610 int lp_nr
= INTVAL (XEXP (note
, 0));
3611 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
3612 remove_note (insn
, note
);
3618 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3619 reg note to indicate that this call cannot throw or execute a nonlocal
3620 goto (unless there is already a REG_EH_REGION note, in which case
3622 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3624 make_reg_eh_region_note_nothrow_nononlocal (insn
);
3627 /* First emit all insns that set pseudos. Remove them from the list as
3628 we go. Avoid insns that set pseudos which were referenced in previous
3629 insns. These can be generated by move_by_pieces, for example,
3630 to update an address. Similarly, avoid insns that reference things
3631 set in previous insns. */
3633 for (insn
= insns
; insn
; insn
= next
)
3635 rtx set
= single_set (insn
);
3637 next
= NEXT_INSN (insn
);
3639 if (set
!= 0 && REG_P (SET_DEST (set
))
3640 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
3642 struct no_conflict_data data
;
3644 data
.target
= const0_rtx
;
3648 note_stores (PATTERN (insn
), no_conflict_move_test
, &data
);
3649 if (! data
.must_stay
)
3651 if (PREV_INSN (insn
))
3652 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
3657 SET_PREV_INSN (next
) = PREV_INSN (insn
);
3663 /* Some ports use a loop to copy large arguments onto the stack.
3664 Don't move anything outside such a loop. */
3669 /* Write the remaining insns followed by the final copy. */
3670 for (insn
= insns
; insn
; insn
= next
)
3672 next
= NEXT_INSN (insn
);
3677 last
= emit_move_insn (target
, result
);
3679 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
3681 if (final_dest
!= target
)
3682 emit_move_insn (final_dest
, target
);
3686 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
3688 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
3691 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3692 PURPOSE describes how this comparison will be used. CODE is the rtx
3693 comparison code we will be using.
3695 ??? Actually, CODE is slightly weaker than that. A target is still
3696 required to implement all of the normal bcc operations, but not
3697 required to implement all (or any) of the unordered bcc operations. */
3700 can_compare_p (enum rtx_code code
, machine_mode mode
,
3701 enum can_compare_purpose purpose
)
3704 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
3707 enum insn_code icode
;
3709 if (purpose
== ccp_jump
3710 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
3711 && insn_operand_matches (icode
, 0, test
))
3713 if (purpose
== ccp_store_flag
3714 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
3715 && insn_operand_matches (icode
, 1, test
))
3717 if (purpose
== ccp_cmov
3718 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
3721 mode
= GET_MODE_WIDER_MODE (mode
);
3722 PUT_MODE (test
, mode
);
3724 while (mode
!= VOIDmode
);
3729 /* This function is called when we are going to emit a compare instruction that
3730 compares the values found in X and Y, using the rtl operator COMPARISON.
3732 If they have mode BLKmode, then SIZE specifies the size of both operands.
3734 UNSIGNEDP nonzero says that the operands are unsigned;
3735 this matters if they need to be widened (as given by METHODS).
3737 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3738 if we failed to produce one.
3740 *PMODE is the mode of the inputs (in case they are const_int).
3742 This function performs all the setup necessary so that the caller only has
3743 to emit a single comparison insn. This setup can involve doing a BLKmode
3744 comparison or emitting a library call to perform the comparison if no insn
3745 is available to handle it.
3746 The values which are passed in through pointers can be modified; the caller
3747 should perform the comparison on the modified values. Constant
3748 comparisons must have already been folded. */
3751 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
3752 int unsignedp
, enum optab_methods methods
,
3753 rtx
*ptest
, machine_mode
*pmode
)
3755 machine_mode mode
= *pmode
;
3757 machine_mode cmp_mode
;
3758 enum mode_class mclass
;
3760 /* The other methods are not needed. */
3761 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
3762 || methods
== OPTAB_LIB_WIDEN
);
3764 /* If we are optimizing, force expensive constants into a register. */
3765 if (CONSTANT_P (x
) && optimize
3766 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
3767 > COSTS_N_INSNS (1)))
3768 x
= force_reg (mode
, x
);
3770 if (CONSTANT_P (y
) && optimize
3771 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
3772 > COSTS_N_INSNS (1)))
3773 y
= force_reg (mode
, y
);
3776 /* Make sure if we have a canonical comparison. The RTL
3777 documentation states that canonical comparisons are required only
3778 for targets which have cc0. */
3779 gcc_assert (!CONSTANT_P (x
) || CONSTANT_P (y
));
3782 /* Don't let both operands fail to indicate the mode. */
3783 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
3784 x
= force_reg (mode
, x
);
3785 if (mode
== VOIDmode
)
3786 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
3788 /* Handle all BLKmode compares. */
3790 if (mode
== BLKmode
)
3792 machine_mode result_mode
;
3793 enum insn_code cmp_code
;
3796 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
3800 /* Try to use a memory block compare insn - either cmpstr
3801 or cmpmem will do. */
3802 for (cmp_mode
= GET_CLASS_NARROWEST_MODE (MODE_INT
);
3803 cmp_mode
!= VOIDmode
;
3804 cmp_mode
= GET_MODE_WIDER_MODE (cmp_mode
))
3806 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
3807 if (cmp_code
== CODE_FOR_nothing
)
3808 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
3809 if (cmp_code
== CODE_FOR_nothing
)
3810 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
3811 if (cmp_code
== CODE_FOR_nothing
)
3814 /* Must make sure the size fits the insn's mode. */
3815 if ((CONST_INT_P (size
)
3816 && INTVAL (size
) >= (1 << GET_MODE_BITSIZE (cmp_mode
)))
3817 || (GET_MODE_BITSIZE (GET_MODE (size
))
3818 > GET_MODE_BITSIZE (cmp_mode
)))
3821 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
3822 result
= gen_reg_rtx (result_mode
);
3823 size
= convert_to_mode (cmp_mode
, size
, 1);
3824 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
3826 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
3827 *pmode
= result_mode
;
3831 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
3834 /* Otherwise call a library function. */
3835 result
= emit_block_comp_via_libcall (XEXP (x
, 0), XEXP (y
, 0), size
);
3839 mode
= TYPE_MODE (integer_type_node
);
3840 methods
= OPTAB_LIB_WIDEN
;
3844 /* Don't allow operands to the compare to trap, as that can put the
3845 compare and branch in different basic blocks. */
3846 if (cfun
->can_throw_non_call_exceptions
)
3849 x
= force_reg (mode
, x
);
3851 y
= force_reg (mode
, y
);
3854 if (GET_MODE_CLASS (mode
) == MODE_CC
)
3856 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
3857 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3858 gcc_assert (icode
!= CODE_FOR_nothing
3859 && insn_operand_matches (icode
, 0, test
));
3864 mclass
= GET_MODE_CLASS (mode
);
3865 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3869 enum insn_code icode
;
3870 icode
= optab_handler (cbranch_optab
, cmp_mode
);
3871 if (icode
!= CODE_FOR_nothing
3872 && insn_operand_matches (icode
, 0, test
))
3874 rtx_insn
*last
= get_last_insn ();
3875 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
3876 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
3878 && insn_operand_matches (icode
, 1, op0
)
3879 && insn_operand_matches (icode
, 2, op1
))
3881 XEXP (test
, 0) = op0
;
3882 XEXP (test
, 1) = op1
;
3887 delete_insns_since (last
);
3890 if (methods
== OPTAB_DIRECT
|| !CLASS_HAS_WIDER_MODES_P (mclass
))
3892 cmp_mode
= GET_MODE_WIDER_MODE (cmp_mode
);
3894 while (cmp_mode
!= VOIDmode
);
3896 if (methods
!= OPTAB_LIB_WIDEN
)
3899 if (!SCALAR_FLOAT_MODE_P (mode
))
3902 machine_mode ret_mode
;
3904 /* Handle a libcall just for the mode we are using. */
3905 libfunc
= optab_libfunc (cmp_optab
, mode
);
3906 gcc_assert (libfunc
);
3908 /* If we want unsigned, and this mode has a distinct unsigned
3909 comparison routine, use that. */
3912 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
3917 ret_mode
= targetm
.libgcc_cmp_return_mode ();
3918 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
3919 ret_mode
, 2, x
, mode
, y
, mode
);
3921 /* There are two kinds of comparison routines. Biased routines
3922 return 0/1/2, and unbiased routines return -1/0/1. Other parts
3923 of gcc expect that the comparison operation is equivalent
3924 to the modified comparison. For signed comparisons compare the
3925 result against 1 in the biased case, and zero in the unbiased
3926 case. For unsigned comparisons always compare against 1 after
3927 biasing the unbiased result by adding 1. This gives us a way to
3929 The comparisons in the fixed-point helper library are always
3934 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
3937 x
= plus_constant (ret_mode
, result
, 1);
3943 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
3947 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
3955 /* Before emitting an insn with code ICODE, make sure that X, which is going
3956 to be used for operand OPNUM of the insn, is converted from mode MODE to
3957 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
3958 that it is accepted by the operand predicate. Return the new value. */
3961 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
3962 machine_mode wider_mode
, int unsignedp
)
3964 if (mode
!= wider_mode
)
3965 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
3967 if (!insn_operand_matches (icode
, opnum
, x
))
3969 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
3970 if (reload_completed
)
3972 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
3974 x
= copy_to_mode_reg (op_mode
, x
);
3980 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
3981 we can do the branch. */
3984 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
3985 profile_probability prob
)
3987 machine_mode optab_mode
;
3988 enum mode_class mclass
;
3989 enum insn_code icode
;
3992 mclass
= GET_MODE_CLASS (mode
);
3993 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
3994 icode
= optab_handler (cbranch_optab
, optab_mode
);
3996 gcc_assert (icode
!= CODE_FOR_nothing
);
3997 gcc_assert (insn_operand_matches (icode
, 0, test
));
3998 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
3999 XEXP (test
, 1), label
));
4000 if (prob
.initialized_p ()
4001 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4004 && any_condjump_p (insn
)
4005 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4006 add_int_reg_note (insn
, REG_BR_PROB
, prob
.to_reg_br_prob_base ());
4009 /* Generate code to compare X with Y so that the condition codes are
4010 set and to jump to LABEL if the condition is true. If X is a
4011 constant and Y is not a constant, then the comparison is swapped to
4012 ensure that the comparison RTL has the canonical form.
4014 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4015 need to be widened. UNSIGNEDP is also used to select the proper
4016 branch condition code.
4018 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4020 MODE is the mode of the inputs (in case they are const_int).
4022 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4023 It will be potentially converted into an unsigned variant based on
4024 UNSIGNEDP to select a proper jump instruction.
4026 PROB is the probability of jumping to LABEL. */
4029 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4030 machine_mode mode
, int unsignedp
, rtx label
,
4031 profile_probability prob
)
4033 rtx op0
= x
, op1
= y
;
4036 /* Swap operands and condition to ensure canonical RTL. */
4037 if (swap_commutative_operands_p (x
, y
)
4038 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4041 comparison
= swap_condition (comparison
);
4044 /* If OP0 is still a constant, then both X and Y must be constants
4045 or the opposite comparison is not supported. Force X into a register
4046 to create canonical RTL. */
4047 if (CONSTANT_P (op0
))
4048 op0
= force_reg (mode
, op0
);
4051 comparison
= unsigned_condition (comparison
);
4053 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4055 emit_cmp_and_jump_insn_1 (test
, mode
, label
, prob
);
4059 /* Emit a library call comparison between floating point X and Y.
4060 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4063 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4064 rtx
*ptest
, machine_mode
*pmode
)
4066 enum rtx_code swapped
= swap_condition (comparison
);
4067 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4068 machine_mode orig_mode
= GET_MODE (x
);
4069 machine_mode mode
, cmp_mode
;
4070 rtx true_rtx
, false_rtx
;
4071 rtx value
, target
, equiv
;
4074 bool reversed_p
= false;
4075 cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4077 for (mode
= orig_mode
;
4079 mode
= GET_MODE_WIDER_MODE (mode
))
4081 if (code_to_optab (comparison
)
4082 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4085 if (code_to_optab (swapped
)
4086 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4089 comparison
= swapped
;
4093 if (code_to_optab (reversed
)
4094 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4096 comparison
= reversed
;
4102 gcc_assert (mode
!= VOIDmode
);
4104 if (mode
!= orig_mode
)
4106 x
= convert_to_mode (mode
, x
, 0);
4107 y
= convert_to_mode (mode
, y
, 0);
4110 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4111 the RTL. The allows the RTL optimizers to delete the libcall if the
4112 condition can be determined at compile-time. */
4113 if (comparison
== UNORDERED
4114 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4116 true_rtx
= const_true_rtx
;
4117 false_rtx
= const0_rtx
;
4124 true_rtx
= const0_rtx
;
4125 false_rtx
= const_true_rtx
;
4129 true_rtx
= const_true_rtx
;
4130 false_rtx
= const0_rtx
;
4134 true_rtx
= const1_rtx
;
4135 false_rtx
= const0_rtx
;
4139 true_rtx
= const0_rtx
;
4140 false_rtx
= constm1_rtx
;
4144 true_rtx
= constm1_rtx
;
4145 false_rtx
= const0_rtx
;
4149 true_rtx
= const0_rtx
;
4150 false_rtx
= const1_rtx
;
4158 if (comparison
== UNORDERED
)
4160 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4161 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4162 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4163 temp
, const_true_rtx
, equiv
);
4167 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4168 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4169 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4170 equiv
, true_rtx
, false_rtx
);
4174 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4175 cmp_mode
, 2, x
, mode
, y
, mode
);
4176 insns
= get_insns ();
4179 target
= gen_reg_rtx (cmp_mode
);
4180 emit_libcall_block (insns
, target
, value
, equiv
);
4182 if (comparison
== UNORDERED
4183 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
4185 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
4187 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
4192 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4195 emit_indirect_jump (rtx loc
)
4197 if (!targetm
.have_indirect_jump ())
4198 sorry ("indirect jumps are not available on this target");
4201 struct expand_operand ops
[1];
4202 create_address_operand (&ops
[0], loc
);
4203 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
4209 /* Emit a conditional move instruction if the machine supports one for that
4210 condition and machine mode.
4212 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4213 the mode to use should they be constants. If it is VOIDmode, they cannot
4216 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4217 should be stored there. MODE is the mode to use should they be constants.
4218 If it is VOIDmode, they cannot both be constants.
4220 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4221 is not supported. */
4224 emit_conditional_move (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4225 machine_mode cmode
, rtx op2
, rtx op3
,
4226 machine_mode mode
, int unsignedp
)
4230 enum insn_code icode
;
4231 enum rtx_code reversed
;
4233 /* If the two source operands are identical, that's just a move. */
4235 if (rtx_equal_p (op2
, op3
))
4238 target
= gen_reg_rtx (mode
);
4240 emit_move_insn (target
, op3
);
4244 /* If one operand is constant, make it the second one. Only do this
4245 if the other operand is not constant as well. */
4247 if (swap_commutative_operands_p (op0
, op1
))
4249 std::swap (op0
, op1
);
4250 code
= swap_condition (code
);
4253 /* get_condition will prefer to generate LT and GT even if the old
4254 comparison was against zero, so undo that canonicalization here since
4255 comparisons against zero are cheaper. */
4256 if (code
== LT
&& op1
== const1_rtx
)
4257 code
= LE
, op1
= const0_rtx
;
4258 else if (code
== GT
&& op1
== constm1_rtx
)
4259 code
= GE
, op1
= const0_rtx
;
4261 if (cmode
== VOIDmode
)
4262 cmode
= GET_MODE (op0
);
4264 enum rtx_code orig_code
= code
;
4265 bool swapped
= false;
4266 if (swap_commutative_operands_p (op2
, op3
)
4267 && ((reversed
= reversed_comparison_code_parts (code
, op0
, op1
, NULL
))
4270 std::swap (op2
, op3
);
4275 if (mode
== VOIDmode
)
4276 mode
= GET_MODE (op2
);
4278 icode
= direct_optab_handler (movcc_optab
, mode
);
4280 if (icode
== CODE_FOR_nothing
)
4284 target
= gen_reg_rtx (mode
);
4286 for (int pass
= 0; ; pass
++)
4288 code
= unsignedp
? unsigned_condition (code
) : code
;
4289 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4291 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4292 punt and let the caller figure out how best to deal with this
4294 if (COMPARISON_P (comparison
))
4296 saved_pending_stack_adjust save
;
4297 save_pending_stack_adjust (&save
);
4298 last
= get_last_insn ();
4299 do_pending_stack_adjust ();
4300 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4301 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
4302 OPTAB_WIDEN
, &comparison
, &cmode
);
4305 struct expand_operand ops
[4];
4307 create_output_operand (&ops
[0], target
, mode
);
4308 create_fixed_operand (&ops
[1], comparison
);
4309 create_input_operand (&ops
[2], op2
, mode
);
4310 create_input_operand (&ops
[3], op3
, mode
);
4311 if (maybe_expand_insn (icode
, 4, ops
))
4313 if (ops
[0].value
!= target
)
4314 convert_move (target
, ops
[0].value
, false);
4318 delete_insns_since (last
);
4319 restore_pending_stack_adjust (&save
);
4325 /* If the preferred op2/op3 order is not usable, retry with other
4326 operand order, perhaps it will expand successfully. */
4329 else if ((reversed
= reversed_comparison_code_parts (orig_code
, op0
, op1
,
4335 std::swap (op2
, op3
);
4340 /* Emit a conditional negate or bitwise complement using the
4341 negcc or notcc optabs if available. Return NULL_RTX if such operations
4342 are not available. Otherwise return the RTX holding the result.
4343 TARGET is the desired destination of the result. COMP is the comparison
4344 on which to negate. If COND is true move into TARGET the negation
4345 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4346 CODE is either NEG or NOT. MODE is the machine mode in which the
4347 operation is performed. */
4350 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
4351 machine_mode mode
, rtx cond
, rtx op1
,
4354 optab op
= unknown_optab
;
4357 else if (code
== NOT
)
4362 insn_code icode
= direct_optab_handler (op
, mode
);
4364 if (icode
== CODE_FOR_nothing
)
4368 target
= gen_reg_rtx (mode
);
4370 rtx_insn
*last
= get_last_insn ();
4371 struct expand_operand ops
[4];
4373 create_output_operand (&ops
[0], target
, mode
);
4374 create_fixed_operand (&ops
[1], cond
);
4375 create_input_operand (&ops
[2], op1
, mode
);
4376 create_input_operand (&ops
[3], op2
, mode
);
4378 if (maybe_expand_insn (icode
, 4, ops
))
4380 if (ops
[0].value
!= target
)
4381 convert_move (target
, ops
[0].value
, false);
4385 delete_insns_since (last
);
4389 /* Emit a conditional addition instruction if the machine supports one for that
4390 condition and machine mode.
4392 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4393 the mode to use should they be constants. If it is VOIDmode, they cannot
4396 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4397 should be stored there. MODE is the mode to use should they be constants.
4398 If it is VOIDmode, they cannot both be constants.
4400 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4401 is not supported. */
4404 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4405 machine_mode cmode
, rtx op2
, rtx op3
,
4406 machine_mode mode
, int unsignedp
)
4410 enum insn_code icode
;
4412 /* If one operand is constant, make it the second one. Only do this
4413 if the other operand is not constant as well. */
4415 if (swap_commutative_operands_p (op0
, op1
))
4417 std::swap (op0
, op1
);
4418 code
= swap_condition (code
);
4421 /* get_condition will prefer to generate LT and GT even if the old
4422 comparison was against zero, so undo that canonicalization here since
4423 comparisons against zero are cheaper. */
4424 if (code
== LT
&& op1
== const1_rtx
)
4425 code
= LE
, op1
= const0_rtx
;
4426 else if (code
== GT
&& op1
== constm1_rtx
)
4427 code
= GE
, op1
= const0_rtx
;
4429 if (cmode
== VOIDmode
)
4430 cmode
= GET_MODE (op0
);
4432 if (mode
== VOIDmode
)
4433 mode
= GET_MODE (op2
);
4435 icode
= optab_handler (addcc_optab
, mode
);
4437 if (icode
== CODE_FOR_nothing
)
4441 target
= gen_reg_rtx (mode
);
4443 code
= unsignedp
? unsigned_condition (code
) : code
;
4444 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4446 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4447 return NULL and let the caller figure out how best to deal with this
4449 if (!COMPARISON_P (comparison
))
4452 do_pending_stack_adjust ();
4453 last
= get_last_insn ();
4454 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4455 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4456 &comparison
, &cmode
);
4459 struct expand_operand ops
[4];
4461 create_output_operand (&ops
[0], target
, mode
);
4462 create_fixed_operand (&ops
[1], comparison
);
4463 create_input_operand (&ops
[2], op2
, mode
);
4464 create_input_operand (&ops
[3], op3
, mode
);
4465 if (maybe_expand_insn (icode
, 4, ops
))
4467 if (ops
[0].value
!= target
)
4468 convert_move (target
, ops
[0].value
, false);
4472 delete_insns_since (last
);
4476 /* These functions attempt to generate an insn body, rather than
4477 emitting the insn, but if the gen function already emits them, we
4478 make no attempt to turn them back into naked patterns. */
4480 /* Generate and return an insn body to add Y to X. */
4483 gen_add2_insn (rtx x
, rtx y
)
4485 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
4487 gcc_assert (insn_operand_matches (icode
, 0, x
));
4488 gcc_assert (insn_operand_matches (icode
, 1, x
));
4489 gcc_assert (insn_operand_matches (icode
, 2, y
));
4491 return GEN_FCN (icode
) (x
, x
, y
);
4494 /* Generate and return an insn body to add r1 and c,
4495 storing the result in r0. */
4498 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
4500 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
4502 if (icode
== CODE_FOR_nothing
4503 || !insn_operand_matches (icode
, 0, r0
)
4504 || !insn_operand_matches (icode
, 1, r1
)
4505 || !insn_operand_matches (icode
, 2, c
))
4508 return GEN_FCN (icode
) (r0
, r1
, c
);
4512 have_add2_insn (rtx x
, rtx y
)
4514 enum insn_code icode
;
4516 gcc_assert (GET_MODE (x
) != VOIDmode
);
4518 icode
= optab_handler (add_optab
, GET_MODE (x
));
4520 if (icode
== CODE_FOR_nothing
)
4523 if (!insn_operand_matches (icode
, 0, x
)
4524 || !insn_operand_matches (icode
, 1, x
)
4525 || !insn_operand_matches (icode
, 2, y
))
4531 /* Generate and return an insn body to add Y to X. */
4534 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
4536 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4538 gcc_assert (insn_operand_matches (icode
, 0, x
));
4539 gcc_assert (insn_operand_matches (icode
, 1, y
));
4540 gcc_assert (insn_operand_matches (icode
, 2, z
));
4542 return GEN_FCN (icode
) (x
, y
, z
);
4545 /* Return true if the target implements an addptr pattern and X, Y,
4546 and Z are valid for the pattern predicates. */
4549 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
4551 enum insn_code icode
;
4553 gcc_assert (GET_MODE (x
) != VOIDmode
);
4555 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4557 if (icode
== CODE_FOR_nothing
)
4560 if (!insn_operand_matches (icode
, 0, x
)
4561 || !insn_operand_matches (icode
, 1, y
)
4562 || !insn_operand_matches (icode
, 2, z
))
4568 /* Generate and return an insn body to subtract Y from X. */
4571 gen_sub2_insn (rtx x
, rtx y
)
4573 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
4575 gcc_assert (insn_operand_matches (icode
, 0, x
));
4576 gcc_assert (insn_operand_matches (icode
, 1, x
));
4577 gcc_assert (insn_operand_matches (icode
, 2, y
));
4579 return GEN_FCN (icode
) (x
, x
, y
);
4582 /* Generate and return an insn body to subtract r1 and c,
4583 storing the result in r0. */
4586 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
4588 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
4590 if (icode
== CODE_FOR_nothing
4591 || !insn_operand_matches (icode
, 0, r0
)
4592 || !insn_operand_matches (icode
, 1, r1
)
4593 || !insn_operand_matches (icode
, 2, c
))
4596 return GEN_FCN (icode
) (r0
, r1
, c
);
4600 have_sub2_insn (rtx x
, rtx y
)
4602 enum insn_code icode
;
4604 gcc_assert (GET_MODE (x
) != VOIDmode
);
4606 icode
= optab_handler (sub_optab
, GET_MODE (x
));
4608 if (icode
== CODE_FOR_nothing
)
4611 if (!insn_operand_matches (icode
, 0, x
)
4612 || !insn_operand_matches (icode
, 1, x
)
4613 || !insn_operand_matches (icode
, 2, y
))
4619 /* Generate the body of an insn to extend Y (with mode MFROM)
4620 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4623 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
4624 machine_mode mfrom
, int unsignedp
)
4626 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
4627 return GEN_FCN (icode
) (x
, y
);
4630 /* Generate code to convert FROM to floating point
4631 and store in TO. FROM must be fixed point and not VOIDmode.
4632 UNSIGNEDP nonzero means regard FROM as unsigned.
4633 Normally this is done by correcting the final value
4634 if it is negative. */
4637 expand_float (rtx to
, rtx from
, int unsignedp
)
4639 enum insn_code icode
;
4641 machine_mode fmode
, imode
;
4642 bool can_do_signed
= false;
4644 /* Crash now, because we won't be able to decide which mode to use. */
4645 gcc_assert (GET_MODE (from
) != VOIDmode
);
4647 /* Look for an insn to do the conversion. Do it in the specified
4648 modes if possible; otherwise convert either input, output or both to
4649 wider mode. If the integer mode is wider than the mode of FROM,
4650 we can do the conversion signed even if the input is unsigned. */
4652 for (fmode
= GET_MODE (to
); fmode
!= VOIDmode
;
4653 fmode
= GET_MODE_WIDER_MODE (fmode
))
4654 for (imode
= GET_MODE (from
); imode
!= VOIDmode
;
4655 imode
= GET_MODE_WIDER_MODE (imode
))
4657 int doing_unsigned
= unsignedp
;
4659 if (fmode
!= GET_MODE (to
)
4660 && significand_size (fmode
) < GET_MODE_PRECISION (GET_MODE (from
)))
4663 icode
= can_float_p (fmode
, imode
, unsignedp
);
4664 if (icode
== CODE_FOR_nothing
&& unsignedp
)
4666 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
4667 if (scode
!= CODE_FOR_nothing
)
4668 can_do_signed
= true;
4669 if (imode
!= GET_MODE (from
))
4670 icode
= scode
, doing_unsigned
= 0;
4673 if (icode
!= CODE_FOR_nothing
)
4675 if (imode
!= GET_MODE (from
))
4676 from
= convert_to_mode (imode
, from
, unsignedp
);
4678 if (fmode
!= GET_MODE (to
))
4679 target
= gen_reg_rtx (fmode
);
4681 emit_unop_insn (icode
, target
, from
,
4682 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
4685 convert_move (to
, target
, 0);
4690 /* Unsigned integer, and no way to convert directly. Convert as signed,
4691 then unconditionally adjust the result. */
4692 if (unsignedp
&& can_do_signed
)
4694 rtx_code_label
*label
= gen_label_rtx ();
4696 REAL_VALUE_TYPE offset
;
4698 /* Look for a usable floating mode FMODE wider than the source and at
4699 least as wide as the target. Using FMODE will avoid rounding woes
4700 with unsigned values greater than the signed maximum value. */
4702 for (fmode
= GET_MODE (to
); fmode
!= VOIDmode
;
4703 fmode
= GET_MODE_WIDER_MODE (fmode
))
4704 if (GET_MODE_PRECISION (GET_MODE (from
)) < GET_MODE_BITSIZE (fmode
)
4705 && can_float_p (fmode
, GET_MODE (from
), 0) != CODE_FOR_nothing
)
4708 if (fmode
== VOIDmode
)
4710 /* There is no such mode. Pretend the target is wide enough. */
4711 fmode
= GET_MODE (to
);
4713 /* Avoid double-rounding when TO is narrower than FROM. */
4714 if ((significand_size (fmode
) + 1)
4715 < GET_MODE_PRECISION (GET_MODE (from
)))
4718 rtx_code_label
*neglabel
= gen_label_rtx ();
4720 /* Don't use TARGET if it isn't a register, is a hard register,
4721 or is the wrong mode. */
4723 || REGNO (target
) < FIRST_PSEUDO_REGISTER
4724 || GET_MODE (target
) != fmode
)
4725 target
= gen_reg_rtx (fmode
);
4727 imode
= GET_MODE (from
);
4728 do_pending_stack_adjust ();
4730 /* Test whether the sign bit is set. */
4731 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
4734 /* The sign bit is not set. Convert as signed. */
4735 expand_float (target
, from
, 0);
4736 emit_jump_insn (targetm
.gen_jump (label
));
4739 /* The sign bit is set.
4740 Convert to a usable (positive signed) value by shifting right
4741 one bit, while remembering if a nonzero bit was shifted
4742 out; i.e., compute (from & 1) | (from >> 1). */
4744 emit_label (neglabel
);
4745 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
4746 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4747 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
4748 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
4750 expand_float (target
, temp
, 0);
4752 /* Multiply by 2 to undo the shift above. */
4753 temp
= expand_binop (fmode
, add_optab
, target
, target
,
4754 target
, 0, OPTAB_LIB_WIDEN
);
4756 emit_move_insn (target
, temp
);
4758 do_pending_stack_adjust ();
4764 /* If we are about to do some arithmetic to correct for an
4765 unsigned operand, do it in a pseudo-register. */
4767 if (GET_MODE (to
) != fmode
4768 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
4769 target
= gen_reg_rtx (fmode
);
4771 /* Convert as signed integer to floating. */
4772 expand_float (target
, from
, 0);
4774 /* If FROM is negative (and therefore TO is negative),
4775 correct its value by 2**bitwidth. */
4777 do_pending_stack_adjust ();
4778 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, GET_MODE (from
),
4782 real_2expN (&offset
, GET_MODE_PRECISION (GET_MODE (from
)), fmode
);
4783 temp
= expand_binop (fmode
, add_optab
, target
,
4784 const_double_from_real_value (offset
, fmode
),
4785 target
, 0, OPTAB_LIB_WIDEN
);
4787 emit_move_insn (target
, temp
);
4789 do_pending_stack_adjust ();
4794 /* No hardware instruction available; call a library routine. */
4799 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
4801 if (GET_MODE_PRECISION (GET_MODE (from
)) < GET_MODE_PRECISION (SImode
))
4802 from
= convert_to_mode (SImode
, from
, unsignedp
);
4804 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
4805 gcc_assert (libfunc
);
4809 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4810 GET_MODE (to
), 1, from
,
4812 insns
= get_insns ();
4815 emit_libcall_block (insns
, target
, value
,
4816 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
4817 GET_MODE (to
), from
));
4822 /* Copy result to requested destination
4823 if we have been computing in a temp location. */
4827 if (GET_MODE (target
) == GET_MODE (to
))
4828 emit_move_insn (to
, target
);
4830 convert_move (to
, target
, 0);
4834 /* Generate code to convert FROM to fixed point and store in TO. FROM
4835 must be floating point. */
4838 expand_fix (rtx to
, rtx from
, int unsignedp
)
4840 enum insn_code icode
;
4842 machine_mode fmode
, imode
;
4843 bool must_trunc
= false;
4845 /* We first try to find a pair of modes, one real and one integer, at
4846 least as wide as FROM and TO, respectively, in which we can open-code
4847 this conversion. If the integer mode is wider than the mode of TO,
4848 we can do the conversion either signed or unsigned. */
4850 for (fmode
= GET_MODE (from
); fmode
!= VOIDmode
;
4851 fmode
= GET_MODE_WIDER_MODE (fmode
))
4852 for (imode
= GET_MODE (to
); imode
!= VOIDmode
;
4853 imode
= GET_MODE_WIDER_MODE (imode
))
4855 int doing_unsigned
= unsignedp
;
4857 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
4858 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
4859 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
4861 if (icode
!= CODE_FOR_nothing
)
4863 rtx_insn
*last
= get_last_insn ();
4864 if (fmode
!= GET_MODE (from
))
4865 from
= convert_to_mode (fmode
, from
, 0);
4869 rtx temp
= gen_reg_rtx (GET_MODE (from
));
4870 from
= expand_unop (GET_MODE (from
), ftrunc_optab
, from
,
4874 if (imode
!= GET_MODE (to
))
4875 target
= gen_reg_rtx (imode
);
4877 if (maybe_emit_unop_insn (icode
, target
, from
,
4878 doing_unsigned
? UNSIGNED_FIX
: FIX
))
4881 convert_move (to
, target
, unsignedp
);
4884 delete_insns_since (last
);
4888 /* For an unsigned conversion, there is one more way to do it.
4889 If we have a signed conversion, we generate code that compares
4890 the real value to the largest representable positive number. If if
4891 is smaller, the conversion is done normally. Otherwise, subtract
4892 one plus the highest signed number, convert, and add it back.
4894 We only need to check all real modes, since we know we didn't find
4895 anything with a wider integer mode.
4897 This code used to extend FP value into mode wider than the destination.
4898 This is needed for decimal float modes which cannot accurately
4899 represent one plus the highest signed number of the same size, but
4900 not for binary modes. Consider, for instance conversion from SFmode
4903 The hot path through the code is dealing with inputs smaller than 2^63
4904 and doing just the conversion, so there is no bits to lose.
4906 In the other path we know the value is positive in the range 2^63..2^64-1
4907 inclusive. (as for other input overflow happens and result is undefined)
4908 So we know that the most important bit set in mantissa corresponds to
4909 2^63. The subtraction of 2^63 should not generate any rounding as it
4910 simply clears out that bit. The rest is trivial. */
4912 if (unsignedp
&& GET_MODE_PRECISION (GET_MODE (to
)) <= HOST_BITS_PER_WIDE_INT
)
4913 for (fmode
= GET_MODE (from
); fmode
!= VOIDmode
;
4914 fmode
= GET_MODE_WIDER_MODE (fmode
))
4915 if (CODE_FOR_nothing
!= can_fix_p (GET_MODE (to
), fmode
, 0, &must_trunc
)
4916 && (!DECIMAL_FLOAT_MODE_P (fmode
)
4917 || GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (GET_MODE (to
))))
4920 REAL_VALUE_TYPE offset
;
4922 rtx_code_label
*lab1
, *lab2
;
4925 bitsize
= GET_MODE_PRECISION (GET_MODE (to
));
4926 real_2expN (&offset
, bitsize
- 1, fmode
);
4927 limit
= const_double_from_real_value (offset
, fmode
);
4928 lab1
= gen_label_rtx ();
4929 lab2
= gen_label_rtx ();
4931 if (fmode
!= GET_MODE (from
))
4932 from
= convert_to_mode (fmode
, from
, 0);
4934 /* See if we need to do the subtraction. */
4935 do_pending_stack_adjust ();
4936 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
, GET_MODE (from
),
4939 /* If not, do the signed "fix" and branch around fixup code. */
4940 expand_fix (to
, from
, 0);
4941 emit_jump_insn (targetm
.gen_jump (lab2
));
4944 /* Otherwise, subtract 2**(N-1), convert to signed number,
4945 then add 2**(N-1). Do the addition using XOR since this
4946 will often generate better code. */
4948 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
4949 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
4950 expand_fix (to
, target
, 0);
4951 target
= expand_binop (GET_MODE (to
), xor_optab
, to
,
4953 (HOST_WIDE_INT_1
<< (bitsize
- 1),
4955 to
, 1, OPTAB_LIB_WIDEN
);
4958 emit_move_insn (to
, target
);
4962 if (optab_handler (mov_optab
, GET_MODE (to
)) != CODE_FOR_nothing
)
4964 /* Make a place for a REG_NOTE and add it. */
4965 insn
= emit_move_insn (to
, to
);
4966 set_dst_reg_note (insn
, REG_EQUAL
,
4967 gen_rtx_fmt_e (UNSIGNED_FIX
, GET_MODE (to
),
4975 /* We can't do it with an insn, so use a library call. But first ensure
4976 that the mode of TO is at least as wide as SImode, since those are the
4977 only library calls we know about. */
4979 if (GET_MODE_PRECISION (GET_MODE (to
)) < GET_MODE_PRECISION (SImode
))
4981 target
= gen_reg_rtx (SImode
);
4983 expand_fix (target
, from
, unsignedp
);
4991 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
4992 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
4993 gcc_assert (libfunc
);
4997 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4998 GET_MODE (to
), 1, from
,
5000 insns
= get_insns ();
5003 emit_libcall_block (insns
, target
, value
,
5004 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5005 GET_MODE (to
), from
));
5010 if (GET_MODE (to
) == GET_MODE (target
))
5011 emit_move_insn (to
, target
);
5013 convert_move (to
, target
, 0);
5018 /* Promote integer arguments for a libcall if necessary.
5019 emit_library_call_value cannot do the promotion because it does not
5020 know if it should do a signed or unsigned promotion. This is because
5021 there are no tree types defined for libcalls. */
5024 prepare_libcall_arg (rtx arg
, int uintp
)
5026 machine_mode mode
= GET_MODE (arg
);
5027 machine_mode arg_mode
;
5028 if (SCALAR_INT_MODE_P (mode
))
5030 /* If we need to promote the integer function argument we need to do
5031 it here instead of inside emit_library_call_value because in
5032 emit_library_call_value we don't know if we should do a signed or
5033 unsigned promotion. */
5036 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
5037 &unsigned_p
, NULL_TREE
, 0);
5038 if (arg_mode
!= mode
)
5039 return convert_to_mode (arg_mode
, arg
, uintp
);
5044 /* Generate code to convert FROM or TO a fixed-point.
5045 If UINTP is true, either TO or FROM is an unsigned integer.
5046 If SATP is true, we need to saturate the result. */
5049 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
5051 machine_mode to_mode
= GET_MODE (to
);
5052 machine_mode from_mode
= GET_MODE (from
);
5054 enum rtx_code this_code
;
5055 enum insn_code code
;
5060 if (to_mode
== from_mode
)
5062 emit_move_insn (to
, from
);
5068 tab
= satp
? satfractuns_optab
: fractuns_optab
;
5069 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
5073 tab
= satp
? satfract_optab
: fract_optab
;
5074 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
5076 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
5077 if (code
!= CODE_FOR_nothing
)
5079 emit_unop_insn (code
, to
, from
, this_code
);
5083 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
5084 gcc_assert (libfunc
);
5086 from
= prepare_libcall_arg (from
, uintp
);
5087 from_mode
= GET_MODE (from
);
5090 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
5091 1, from
, from_mode
);
5092 insns
= get_insns ();
5095 emit_libcall_block (insns
, to
, value
,
5096 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
5099 /* Generate code to convert FROM to fixed point and store in TO. FROM
5100 must be floating point, TO must be signed. Use the conversion optab
5101 TAB to do the conversion. */
5104 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
5106 enum insn_code icode
;
5108 machine_mode fmode
, imode
;
5110 /* We first try to find a pair of modes, one real and one integer, at
5111 least as wide as FROM and TO, respectively, in which we can open-code
5112 this conversion. If the integer mode is wider than the mode of TO,
5113 we can do the conversion either signed or unsigned. */
5115 for (fmode
= GET_MODE (from
); fmode
!= VOIDmode
;
5116 fmode
= GET_MODE_WIDER_MODE (fmode
))
5117 for (imode
= GET_MODE (to
); imode
!= VOIDmode
;
5118 imode
= GET_MODE_WIDER_MODE (imode
))
5120 icode
= convert_optab_handler (tab
, imode
, fmode
);
5121 if (icode
!= CODE_FOR_nothing
)
5123 rtx_insn
*last
= get_last_insn ();
5124 if (fmode
!= GET_MODE (from
))
5125 from
= convert_to_mode (fmode
, from
, 0);
5127 if (imode
!= GET_MODE (to
))
5128 target
= gen_reg_rtx (imode
);
5130 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
5132 delete_insns_since (last
);
5136 convert_move (to
, target
, 0);
5144 /* Report whether we have an instruction to perform the operation
5145 specified by CODE on operands of mode MODE. */
5147 have_insn_for (enum rtx_code code
, machine_mode mode
)
5149 return (code_to_optab (code
)
5150 && (optab_handler (code_to_optab (code
), mode
)
5151 != CODE_FOR_nothing
));
5154 /* Print information about the current contents of the optabs on
5158 debug_optab_libfuncs (void)
5162 /* Dump the arithmetic optabs. */
5163 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
5164 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5166 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
5169 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5170 fprintf (stderr
, "%s\t%s:\t%s\n",
5171 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5177 /* Dump the conversion optabs. */
5178 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
5179 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5180 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
5182 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
5186 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5187 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
5188 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5196 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5197 CODE. Return 0 on failure. */
5200 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
5202 machine_mode mode
= GET_MODE (op1
);
5203 enum insn_code icode
;
5207 if (mode
== VOIDmode
)
5210 icode
= optab_handler (ctrap_optab
, mode
);
5211 if (icode
== CODE_FOR_nothing
)
5214 /* Some targets only accept a zero trap code. */
5215 if (!insn_operand_matches (icode
, 3, tcode
))
5218 do_pending_stack_adjust ();
5220 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
5225 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
5228 /* If that failed, then give up. */
5236 insn
= get_insns ();
5241 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5242 or unsigned operation code. */
5245 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
5257 code
= unsignedp
? LTU
: LT
;
5260 code
= unsignedp
? LEU
: LE
;
5263 code
= unsignedp
? GTU
: GT
;
5266 code
= unsignedp
? GEU
: GE
;
5269 case UNORDERED_EXPR
:
5308 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5309 select signed or unsigned operators. OPNO holds the index of the
5310 first comparison operand for insn ICODE. Do not generate the
5311 compare instruction itself. */
5314 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
5315 tree t_op0
, tree t_op1
, bool unsignedp
,
5316 enum insn_code icode
, unsigned int opno
)
5318 struct expand_operand ops
[2];
5319 rtx rtx_op0
, rtx_op1
;
5320 machine_mode m0
, m1
;
5321 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
5323 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
5325 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5326 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5327 cases, use the original mode. */
5328 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
5330 m0
= GET_MODE (rtx_op0
);
5332 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
5334 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
5336 m1
= GET_MODE (rtx_op1
);
5338 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
5340 create_input_operand (&ops
[0], rtx_op0
, m0
);
5341 create_input_operand (&ops
[1], rtx_op1
, m1
);
5342 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
5344 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
5347 /* Checks if vec_perm mask SEL is a constant equivalent to a shift of the first
5348 vec_perm operand, assuming the second operand is a constant vector of zeroes.
5349 Return the shift distance in bits if so, or NULL_RTX if the vec_perm is not a
5352 shift_amt_for_vec_perm_mask (rtx sel
)
5354 unsigned int i
, first
, nelt
= GET_MODE_NUNITS (GET_MODE (sel
));
5355 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (GET_MODE (sel
));
5357 if (GET_CODE (sel
) != CONST_VECTOR
)
5360 first
= INTVAL (CONST_VECTOR_ELT (sel
, 0));
5363 for (i
= 1; i
< nelt
; i
++)
5365 int idx
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
5366 unsigned int expected
= i
+ first
;
5367 /* Indices into the second vector are all equivalent. */
5368 if (idx
< 0 || (MIN (nelt
, (unsigned) idx
) != MIN (nelt
, expected
)))
5372 return GEN_INT (first
* bitsize
);
5375 /* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
5378 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
5379 rtx v0
, rtx v1
, rtx sel
)
5381 machine_mode tmode
= GET_MODE (target
);
5382 machine_mode smode
= GET_MODE (sel
);
5383 struct expand_operand ops
[4];
5385 create_output_operand (&ops
[0], target
, tmode
);
5386 create_input_operand (&ops
[3], sel
, smode
);
5388 /* Make an effort to preserve v0 == v1. The target expander is able to
5389 rely on this to determine if we're permuting a single input operand. */
5390 if (rtx_equal_p (v0
, v1
))
5392 if (!insn_operand_matches (icode
, 1, v0
))
5393 v0
= force_reg (tmode
, v0
);
5394 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
5395 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
5397 create_fixed_operand (&ops
[1], v0
);
5398 create_fixed_operand (&ops
[2], v0
);
5402 create_input_operand (&ops
[1], v0
, tmode
);
5403 create_input_operand (&ops
[2], v1
, tmode
);
5406 if (maybe_expand_insn (icode
, 4, ops
))
5407 return ops
[0].value
;
5411 /* Generate instructions for vec_perm optab given its mode
5412 and three operands. */
5415 expand_vec_perm (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
5417 enum insn_code icode
;
5418 machine_mode qimode
;
5419 unsigned int i
, w
, e
, u
;
5420 rtx tmp
, sel_qi
= NULL
;
5423 if (!target
|| GET_MODE (target
) != mode
)
5424 target
= gen_reg_rtx (mode
);
5426 w
= GET_MODE_SIZE (mode
);
5427 e
= GET_MODE_NUNITS (mode
);
5428 u
= GET_MODE_UNIT_SIZE (mode
);
5430 /* Set QIMODE to a different vector mode with byte elements.
5431 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5433 if (GET_MODE_INNER (mode
) != QImode
)
5435 qimode
= mode_for_vector (QImode
, w
);
5436 if (!VECTOR_MODE_P (qimode
))
5440 /* If the input is a constant, expand it specially. */
5441 gcc_assert (GET_MODE_CLASS (GET_MODE (sel
)) == MODE_VECTOR_INT
);
5442 if (GET_CODE (sel
) == CONST_VECTOR
)
5444 /* See if this can be handled with a vec_shr. We only do this if the
5445 second vector is all zeroes. */
5446 enum insn_code shift_code
= optab_handler (vec_shr_optab
, mode
);
5447 enum insn_code shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
5448 ? optab_handler (vec_shr_optab
, qimode
)
5449 : CODE_FOR_nothing
);
5450 rtx shift_amt
= NULL_RTX
;
5451 if (v1
== CONST0_RTX (GET_MODE (v1
))
5452 && (shift_code
!= CODE_FOR_nothing
5453 || shift_code_qi
!= CODE_FOR_nothing
))
5455 shift_amt
= shift_amt_for_vec_perm_mask (sel
);
5458 struct expand_operand ops
[3];
5459 if (shift_code
!= CODE_FOR_nothing
)
5461 create_output_operand (&ops
[0], target
, mode
);
5462 create_input_operand (&ops
[1], v0
, mode
);
5463 create_convert_operand_from_type (&ops
[2], shift_amt
,
5465 if (maybe_expand_insn (shift_code
, 3, ops
))
5466 return ops
[0].value
;
5468 if (shift_code_qi
!= CODE_FOR_nothing
)
5470 tmp
= gen_reg_rtx (qimode
);
5471 create_output_operand (&ops
[0], tmp
, qimode
);
5472 create_input_operand (&ops
[1], gen_lowpart (qimode
, v0
),
5474 create_convert_operand_from_type (&ops
[2], shift_amt
,
5476 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
5477 return gen_lowpart (mode
, ops
[0].value
);
5482 icode
= direct_optab_handler (vec_perm_const_optab
, mode
);
5483 if (icode
!= CODE_FOR_nothing
)
5485 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5490 /* Fall back to a constant byte-based permutation. */
5491 if (qimode
!= VOIDmode
)
5493 vec
= rtvec_alloc (w
);
5494 for (i
= 0; i
< e
; ++i
)
5496 unsigned int j
, this_e
;
5498 this_e
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
5499 this_e
&= 2 * e
- 1;
5502 for (j
= 0; j
< u
; ++j
)
5503 RTVEC_ELT (vec
, i
* u
+ j
) = GEN_INT (this_e
+ j
);
5505 sel_qi
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5507 icode
= direct_optab_handler (vec_perm_const_optab
, qimode
);
5508 if (icode
!= CODE_FOR_nothing
)
5510 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5511 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5512 gen_lowpart (qimode
, v1
), sel_qi
);
5514 return gen_lowpart (mode
, tmp
);
5519 /* Otherwise expand as a fully variable permuation. */
5520 icode
= direct_optab_handler (vec_perm_optab
, mode
);
5521 if (icode
!= CODE_FOR_nothing
)
5523 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5528 /* As a special case to aid several targets, lower the element-based
5529 permutation to a byte-based permutation and try again. */
5530 if (qimode
== VOIDmode
)
5532 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
5533 if (icode
== CODE_FOR_nothing
)
5538 /* Multiply each element by its byte size. */
5539 machine_mode selmode
= GET_MODE (sel
);
5541 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
5542 NULL
, 0, OPTAB_DIRECT
);
5544 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
5545 GEN_INT (exact_log2 (u
)),
5546 NULL
, 0, OPTAB_DIRECT
);
5547 gcc_assert (sel
!= NULL
);
5549 /* Broadcast the low byte each element into each of its bytes. */
5550 vec
= rtvec_alloc (w
);
5551 for (i
= 0; i
< w
; ++i
)
5553 int this_e
= i
/ u
* u
;
5554 if (BYTES_BIG_ENDIAN
)
5556 RTVEC_ELT (vec
, i
) = GEN_INT (this_e
);
5558 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5559 sel
= gen_lowpart (qimode
, sel
);
5560 sel
= expand_vec_perm (qimode
, sel
, sel
, tmp
, NULL
);
5561 gcc_assert (sel
!= NULL
);
5563 /* Add the byte offset to each byte element. */
5564 /* Note that the definition of the indicies here is memory ordering,
5565 so there should be no difference between big and little endian. */
5566 vec
= rtvec_alloc (w
);
5567 for (i
= 0; i
< w
; ++i
)
5568 RTVEC_ELT (vec
, i
) = GEN_INT (i
% u
);
5569 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5570 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
5571 sel
, 0, OPTAB_DIRECT
);
5572 gcc_assert (sel_qi
!= NULL
);
5575 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5576 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5577 gen_lowpart (qimode
, v1
), sel_qi
);
5579 tmp
= gen_lowpart (mode
, tmp
);
5583 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5587 expand_vec_cond_mask_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5590 struct expand_operand ops
[4];
5591 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5592 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
5593 enum insn_code icode
= get_vcond_mask_icode (mode
, mask_mode
);
5594 rtx mask
, rtx_op1
, rtx_op2
;
5596 if (icode
== CODE_FOR_nothing
)
5599 mask
= expand_normal (op0
);
5600 rtx_op1
= expand_normal (op1
);
5601 rtx_op2
= expand_normal (op2
);
5603 mask
= force_reg (mask_mode
, mask
);
5604 rtx_op1
= force_reg (GET_MODE (rtx_op1
), rtx_op1
);
5606 create_output_operand (&ops
[0], target
, mode
);
5607 create_input_operand (&ops
[1], rtx_op1
, mode
);
5608 create_input_operand (&ops
[2], rtx_op2
, mode
);
5609 create_input_operand (&ops
[3], mask
, mask_mode
);
5610 expand_insn (icode
, 4, ops
);
5612 return ops
[0].value
;
5615 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5619 expand_vec_cond_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5622 struct expand_operand ops
[6];
5623 enum insn_code icode
;
5624 rtx comparison
, rtx_op1
, rtx_op2
;
5625 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5626 machine_mode cmp_op_mode
;
5629 enum tree_code tcode
;
5631 if (COMPARISON_CLASS_P (op0
))
5633 op0a
= TREE_OPERAND (op0
, 0);
5634 op0b
= TREE_OPERAND (op0
, 1);
5635 tcode
= TREE_CODE (op0
);
5639 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0
)));
5640 if (get_vcond_mask_icode (mode
, TYPE_MODE (TREE_TYPE (op0
)))
5641 != CODE_FOR_nothing
)
5642 return expand_vec_cond_mask_expr (vec_cond_type
, op0
, op1
,
5647 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0
)))
5648 == MODE_VECTOR_INT
);
5650 op0b
= build_zero_cst (TREE_TYPE (op0
));
5654 cmp_op_mode
= TYPE_MODE (TREE_TYPE (op0a
));
5655 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5658 gcc_assert (GET_MODE_SIZE (mode
) == GET_MODE_SIZE (cmp_op_mode
)
5659 && GET_MODE_NUNITS (mode
) == GET_MODE_NUNITS (cmp_op_mode
));
5661 icode
= get_vcond_icode (mode
, cmp_op_mode
, unsignedp
);
5662 if (icode
== CODE_FOR_nothing
)
5664 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5665 icode
= get_vcond_eq_icode (mode
, cmp_op_mode
);
5666 if (icode
== CODE_FOR_nothing
)
5670 comparison
= vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
,
5672 rtx_op1
= expand_normal (op1
);
5673 rtx_op2
= expand_normal (op2
);
5675 create_output_operand (&ops
[0], target
, mode
);
5676 create_input_operand (&ops
[1], rtx_op1
, mode
);
5677 create_input_operand (&ops
[2], rtx_op2
, mode
);
5678 create_fixed_operand (&ops
[3], comparison
);
5679 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
5680 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
5681 expand_insn (icode
, 6, ops
);
5682 return ops
[0].value
;
5685 /* Generate insns for a vector comparison into a mask. */
5688 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
5690 struct expand_operand ops
[4];
5691 enum insn_code icode
;
5693 machine_mode mask_mode
= TYPE_MODE (type
);
5697 enum tree_code tcode
;
5699 op0a
= TREE_OPERAND (exp
, 0);
5700 op0b
= TREE_OPERAND (exp
, 1);
5701 tcode
= TREE_CODE (exp
);
5703 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5704 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
5706 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
5707 if (icode
== CODE_FOR_nothing
)
5709 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5710 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
5711 if (icode
== CODE_FOR_nothing
)
5715 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
5716 unsignedp
, icode
, 2);
5717 create_output_operand (&ops
[0], target
, mask_mode
);
5718 create_fixed_operand (&ops
[1], comparison
);
5719 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
5720 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
5721 expand_insn (icode
, 4, ops
);
5722 return ops
[0].value
;
5725 /* Expand a highpart multiply. */
5728 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
5729 rtx target
, bool uns_p
)
5731 struct expand_operand eops
[3];
5732 enum insn_code icode
;
5733 int method
, i
, nunits
;
5739 method
= can_mult_highpart_p (mode
, uns_p
);
5745 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
5746 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
5749 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
5750 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
5753 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
5754 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
5755 if (BYTES_BIG_ENDIAN
)
5756 std::swap (tab1
, tab2
);
5762 icode
= optab_handler (tab1
, mode
);
5763 nunits
= GET_MODE_NUNITS (mode
);
5764 wmode
= insn_data
[icode
].operand
[0].mode
;
5765 gcc_checking_assert (2 * GET_MODE_NUNITS (wmode
) == nunits
);
5766 gcc_checking_assert (GET_MODE_SIZE (wmode
) == GET_MODE_SIZE (mode
));
5768 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5769 create_input_operand (&eops
[1], op0
, mode
);
5770 create_input_operand (&eops
[2], op1
, mode
);
5771 expand_insn (icode
, 3, eops
);
5772 m1
= gen_lowpart (mode
, eops
[0].value
);
5774 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5775 create_input_operand (&eops
[1], op0
, mode
);
5776 create_input_operand (&eops
[2], op1
, mode
);
5777 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
5778 m2
= gen_lowpart (mode
, eops
[0].value
);
5780 v
= rtvec_alloc (nunits
);
5783 for (i
= 0; i
< nunits
; ++i
)
5784 RTVEC_ELT (v
, i
) = GEN_INT (!BYTES_BIG_ENDIAN
+ (i
& ~1)
5785 + ((i
& 1) ? nunits
: 0));
5789 for (i
= 0; i
< nunits
; ++i
)
5790 RTVEC_ELT (v
, i
) = GEN_INT (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
5792 perm
= gen_rtx_CONST_VECTOR (mode
, v
);
5794 return expand_vec_perm (mode
, m1
, m2
, perm
, target
);
5797 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
5801 find_cc_set (rtx x
, const_rtx pat
, void *data
)
5803 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
5804 && GET_CODE (pat
) == SET
)
5806 rtx
*p_cc_reg
= (rtx
*) data
;
5807 gcc_assert (!*p_cc_reg
);
5812 /* This is a helper function for the other atomic operations. This function
5813 emits a loop that contains SEQ that iterates until a compare-and-swap
5814 operation at the end succeeds. MEM is the memory to be modified. SEQ is
5815 a set of instructions that takes a value from OLD_REG as an input and
5816 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
5817 set to the current contents of MEM. After SEQ, a compare-and-swap will
5818 attempt to update MEM with NEW_REG. The function returns true when the
5819 loop was generated successfully. */
5822 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
5824 machine_mode mode
= GET_MODE (mem
);
5825 rtx_code_label
*label
;
5826 rtx cmp_reg
, success
, oldval
;
5828 /* The loop we want to generate looks like
5834 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
5838 Note that we only do the plain load from memory once. Subsequent
5839 iterations use the value loaded by the compare-and-swap pattern. */
5841 label
= gen_label_rtx ();
5842 cmp_reg
= gen_reg_rtx (mode
);
5844 emit_move_insn (cmp_reg
, mem
);
5846 emit_move_insn (old_reg
, cmp_reg
);
5852 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
5853 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
5857 if (oldval
!= cmp_reg
)
5858 emit_move_insn (cmp_reg
, oldval
);
5860 /* Mark this jump predicted not taken. */
5861 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
5862 GET_MODE (success
), 1, label
,
5863 profile_probability::guessed_never ());
5868 /* This function tries to emit an atomic_exchange intruction. VAL is written
5869 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
5870 using TARGET if possible. */
5873 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
5875 machine_mode mode
= GET_MODE (mem
);
5876 enum insn_code icode
;
5878 /* If the target supports the exchange directly, great. */
5879 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
5880 if (icode
!= CODE_FOR_nothing
)
5882 struct expand_operand ops
[4];
5884 create_output_operand (&ops
[0], target
, mode
);
5885 create_fixed_operand (&ops
[1], mem
);
5886 create_input_operand (&ops
[2], val
, mode
);
5887 create_integer_operand (&ops
[3], model
);
5888 if (maybe_expand_insn (icode
, 4, ops
))
5889 return ops
[0].value
;
5895 /* This function tries to implement an atomic exchange operation using
5896 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
5897 The previous contents of *MEM are returned, using TARGET if possible.
5898 Since this instructionn is an acquire barrier only, stronger memory
5899 models may require additional barriers to be emitted. */
5902 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
5903 enum memmodel model
)
5905 machine_mode mode
= GET_MODE (mem
);
5906 enum insn_code icode
;
5907 rtx_insn
*last_insn
= get_last_insn ();
5909 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
5911 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
5912 exists, and the memory model is stronger than acquire, add a release
5913 barrier before the instruction. */
5915 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
5916 expand_mem_thread_fence (model
);
5918 if (icode
!= CODE_FOR_nothing
)
5920 struct expand_operand ops
[3];
5921 create_output_operand (&ops
[0], target
, mode
);
5922 create_fixed_operand (&ops
[1], mem
);
5923 create_input_operand (&ops
[2], val
, mode
);
5924 if (maybe_expand_insn (icode
, 3, ops
))
5925 return ops
[0].value
;
5928 /* If an external test-and-set libcall is provided, use that instead of
5929 any external compare-and-swap that we might get from the compare-and-
5930 swap-loop expansion later. */
5931 if (!can_compare_and_swap_p (mode
, false))
5933 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
5934 if (libfunc
!= NULL
)
5938 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
5939 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
5940 mode
, 2, addr
, ptr_mode
,
5945 /* If the test_and_set can't be emitted, eliminate any barrier that might
5946 have been emitted. */
5947 delete_insns_since (last_insn
);
5951 /* This function tries to implement an atomic exchange operation using a
5952 compare_and_swap loop. VAL is written to *MEM. The previous contents of
5953 *MEM are returned, using TARGET if possible. No memory model is required
5954 since a compare_and_swap loop is seq-cst. */
5957 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
5959 machine_mode mode
= GET_MODE (mem
);
5961 if (can_compare_and_swap_p (mode
, true))
5963 if (!target
|| !register_operand (target
, mode
))
5964 target
= gen_reg_rtx (mode
);
5965 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
5972 /* This function tries to implement an atomic test-and-set operation
5973 using the atomic_test_and_set instruction pattern. A boolean value
5974 is returned from the operation, using TARGET if possible. */
5977 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
5979 machine_mode pat_bool_mode
;
5980 struct expand_operand ops
[3];
5982 if (!targetm
.have_atomic_test_and_set ())
5985 /* While we always get QImode from __atomic_test_and_set, we get
5986 other memory modes from __sync_lock_test_and_set. Note that we
5987 use no endian adjustment here. This matches the 4.6 behavior
5988 in the Sparc backend. */
5989 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
5990 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
5991 if (GET_MODE (mem
) != QImode
)
5992 mem
= adjust_address_nv (mem
, QImode
, 0);
5994 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
5995 create_output_operand (&ops
[0], target
, pat_bool_mode
);
5996 create_fixed_operand (&ops
[1], mem
);
5997 create_integer_operand (&ops
[2], model
);
5999 if (maybe_expand_insn (icode
, 3, ops
))
6000 return ops
[0].value
;
6004 /* This function expands the legacy _sync_lock test_and_set operation which is
6005 generally an atomic exchange. Some limited targets only allow the
6006 constant 1 to be stored. This is an ACQUIRE operation.
6008 TARGET is an optional place to stick the return value.
6009 MEM is where VAL is stored. */
6012 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
6016 /* Try an atomic_exchange first. */
6017 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
6021 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
6022 MEMMODEL_SYNC_ACQUIRE
);
6026 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6030 /* If there are no other options, try atomic_test_and_set if the value
6031 being stored is 1. */
6032 if (val
== const1_rtx
)
6033 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
6038 /* This function expands the atomic test_and_set operation:
6039 atomically store a boolean TRUE into MEM and return the previous value.
6041 MEMMODEL is the memory model variant to use.
6042 TARGET is an optional place to stick the return value. */
6045 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6047 machine_mode mode
= GET_MODE (mem
);
6048 rtx ret
, trueval
, subtarget
;
6050 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
6054 /* Be binary compatible with non-default settings of trueval, and different
6055 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6056 another only has atomic-exchange. */
6057 if (targetm
.atomic_test_and_set_trueval
== 1)
6059 trueval
= const1_rtx
;
6060 subtarget
= target
? target
: gen_reg_rtx (mode
);
6064 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
6065 subtarget
= gen_reg_rtx (mode
);
6068 /* Try the atomic-exchange optab... */
6069 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
6071 /* ... then an atomic-compare-and-swap loop ... */
6073 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
6075 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6077 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
6079 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6080 things with the value 1. Thus we try again without trueval. */
6081 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
6082 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
6084 /* Failing all else, assume a single threaded environment and simply
6085 perform the operation. */
6088 /* If the result is ignored skip the move to target. */
6089 if (subtarget
!= const0_rtx
)
6090 emit_move_insn (subtarget
, mem
);
6092 emit_move_insn (mem
, trueval
);
6096 /* Recall that have to return a boolean value; rectify if trueval
6097 is not exactly one. */
6098 if (targetm
.atomic_test_and_set_trueval
!= 1)
6099 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
6104 /* This function expands the atomic exchange operation:
6105 atomically store VAL in MEM and return the previous value in MEM.
6107 MEMMODEL is the memory model variant to use.
6108 TARGET is an optional place to stick the return value. */
6111 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6113 machine_mode mode
= GET_MODE (mem
);
6116 /* If loads are not atomic for the required size and we are not called to
6117 provide a __sync builtin, do not do anything so that we stay consistent
6118 with atomic loads of the same size. */
6119 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6122 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6124 /* Next try a compare-and-swap loop for the exchange. */
6126 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6131 /* This function expands the atomic compare exchange operation:
6133 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6134 *PTARGET_OVAL is an optional place to store the old value from memory.
6135 Both target parameters may be NULL or const0_rtx to indicate that we do
6136 not care about that return value. Both target parameters are updated on
6137 success to the actual location of the corresponding result.
6139 MEMMODEL is the memory model variant to use.
6141 The return value of the function is true for success. */
6144 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
6145 rtx mem
, rtx expected
, rtx desired
,
6146 bool is_weak
, enum memmodel succ_model
,
6147 enum memmodel fail_model
)
6149 machine_mode mode
= GET_MODE (mem
);
6150 struct expand_operand ops
[8];
6151 enum insn_code icode
;
6152 rtx target_oval
, target_bool
= NULL_RTX
;
6155 /* If loads are not atomic for the required size and we are not called to
6156 provide a __sync builtin, do not do anything so that we stay consistent
6157 with atomic loads of the same size. */
6158 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
6161 /* Load expected into a register for the compare and swap. */
6162 if (MEM_P (expected
))
6163 expected
= copy_to_reg (expected
);
6165 /* Make sure we always have some place to put the return oldval.
6166 Further, make sure that place is distinct from the input expected,
6167 just in case we need that path down below. */
6168 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
6169 ptarget_oval
= NULL
;
6171 if (ptarget_oval
== NULL
6172 || (target_oval
= *ptarget_oval
) == NULL
6173 || reg_overlap_mentioned_p (expected
, target_oval
))
6174 target_oval
= gen_reg_rtx (mode
);
6176 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
6177 if (icode
!= CODE_FOR_nothing
)
6179 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
6181 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
6182 ptarget_bool
= NULL
;
6184 /* Make sure we always have a place for the bool operand. */
6185 if (ptarget_bool
== NULL
6186 || (target_bool
= *ptarget_bool
) == NULL
6187 || GET_MODE (target_bool
) != bool_mode
)
6188 target_bool
= gen_reg_rtx (bool_mode
);
6190 /* Emit the compare_and_swap. */
6191 create_output_operand (&ops
[0], target_bool
, bool_mode
);
6192 create_output_operand (&ops
[1], target_oval
, mode
);
6193 create_fixed_operand (&ops
[2], mem
);
6194 create_input_operand (&ops
[3], expected
, mode
);
6195 create_input_operand (&ops
[4], desired
, mode
);
6196 create_integer_operand (&ops
[5], is_weak
);
6197 create_integer_operand (&ops
[6], succ_model
);
6198 create_integer_operand (&ops
[7], fail_model
);
6199 if (maybe_expand_insn (icode
, 8, ops
))
6201 /* Return success/failure. */
6202 target_bool
= ops
[0].value
;
6203 target_oval
= ops
[1].value
;
6208 /* Otherwise fall back to the original __sync_val_compare_and_swap
6209 which is always seq-cst. */
6210 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
6211 if (icode
!= CODE_FOR_nothing
)
6215 create_output_operand (&ops
[0], target_oval
, mode
);
6216 create_fixed_operand (&ops
[1], mem
);
6217 create_input_operand (&ops
[2], expected
, mode
);
6218 create_input_operand (&ops
[3], desired
, mode
);
6219 if (!maybe_expand_insn (icode
, 4, ops
))
6222 target_oval
= ops
[0].value
;
6224 /* If the caller isn't interested in the boolean return value,
6225 skip the computation of it. */
6226 if (ptarget_bool
== NULL
)
6229 /* Otherwise, work out if the compare-and-swap succeeded. */
6231 if (have_insn_for (COMPARE
, CCmode
))
6232 note_stores (PATTERN (get_last_insn ()), find_cc_set
, &cc_reg
);
6235 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
6236 const0_rtx
, VOIDmode
, 0, 1);
6239 goto success_bool_from_val
;
6242 /* Also check for library support for __sync_val_compare_and_swap. */
6243 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
6244 if (libfunc
!= NULL
)
6246 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6247 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6248 mode
, 3, addr
, ptr_mode
,
6249 expected
, mode
, desired
, mode
);
6250 emit_move_insn (target_oval
, target
);
6252 /* Compute the boolean return value only if requested. */
6254 goto success_bool_from_val
;
6262 success_bool_from_val
:
6263 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
6264 expected
, VOIDmode
, 1, 1);
6266 /* Make sure that the oval output winds up where the caller asked. */
6268 *ptarget_oval
= target_oval
;
6270 *ptarget_bool
= target_bool
;
6274 /* Generate asm volatile("" : : : "memory") as the memory barrier. */
6277 expand_asm_memory_barrier (void)
6281 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
6282 rtvec_alloc (0), rtvec_alloc (0),
6283 rtvec_alloc (0), UNKNOWN_LOCATION
);
6284 MEM_VOLATILE_P (asm_op
) = 1;
6286 clob
= gen_rtx_SCRATCH (VOIDmode
);
6287 clob
= gen_rtx_MEM (BLKmode
, clob
);
6288 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
6290 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
6293 /* This routine will either emit the mem_thread_fence pattern or issue a
6294 sync_synchronize to generate a fence for memory model MEMMODEL. */
6297 expand_mem_thread_fence (enum memmodel model
)
6299 if (targetm
.have_mem_thread_fence ())
6300 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
6301 else if (!is_mm_relaxed (model
))
6303 if (targetm
.have_memory_barrier ())
6304 emit_insn (targetm
.gen_memory_barrier ());
6305 else if (synchronize_libfunc
!= NULL_RTX
)
6306 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
, 0);
6308 expand_asm_memory_barrier ();
6312 /* This routine will either emit the mem_signal_fence pattern or issue a
6313 sync_synchronize to generate a fence for memory model MEMMODEL. */
6316 expand_mem_signal_fence (enum memmodel model
)
6318 if (targetm
.have_mem_signal_fence ())
6319 emit_insn (targetm
.gen_mem_signal_fence (GEN_INT (model
)));
6320 else if (!is_mm_relaxed (model
))
6322 /* By default targets are coherent between a thread and the signal
6323 handler running on the same thread. Thus this really becomes a
6324 compiler barrier, in that stores must not be sunk past
6325 (or raised above) a given point. */
6326 expand_asm_memory_barrier ();
6330 /* This function expands the atomic load operation:
6331 return the atomically loaded value in MEM.
6333 MEMMODEL is the memory model variant to use.
6334 TARGET is an option place to stick the return value. */
6337 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
6339 machine_mode mode
= GET_MODE (mem
);
6340 enum insn_code icode
;
6342 /* If the target supports the load directly, great. */
6343 icode
= direct_optab_handler (atomic_load_optab
, mode
);
6344 if (icode
!= CODE_FOR_nothing
)
6346 struct expand_operand ops
[3];
6348 create_output_operand (&ops
[0], target
, mode
);
6349 create_fixed_operand (&ops
[1], mem
);
6350 create_integer_operand (&ops
[2], model
);
6351 if (maybe_expand_insn (icode
, 3, ops
))
6352 return ops
[0].value
;
6355 /* If the size of the object is greater than word size on this target,
6356 then we assume that a load will not be atomic. We could try to
6357 emulate a load with a compare-and-swap operation, but the store that
6358 doing this could result in would be incorrect if this is a volatile
6359 atomic load or targetting read-only-mapped memory. */
6360 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
6361 /* If there is no atomic load, leave the library call. */
6364 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6365 if (!target
|| target
== const0_rtx
)
6366 target
= gen_reg_rtx (mode
);
6368 /* For SEQ_CST, emit a barrier before the load. */
6369 if (is_mm_seq_cst (model
))
6370 expand_mem_thread_fence (model
);
6372 emit_move_insn (target
, mem
);
6374 /* Emit the appropriate barrier after the load. */
6375 expand_mem_thread_fence (model
);
6380 /* This function expands the atomic store operation:
6381 Atomically store VAL in MEM.
6382 MEMMODEL is the memory model variant to use.
6383 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6384 function returns const0_rtx if a pattern was emitted. */
6387 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
6389 machine_mode mode
= GET_MODE (mem
);
6390 enum insn_code icode
;
6391 struct expand_operand ops
[3];
6393 /* If the target supports the store directly, great. */
6394 icode
= direct_optab_handler (atomic_store_optab
, mode
);
6395 if (icode
!= CODE_FOR_nothing
)
6397 create_fixed_operand (&ops
[0], mem
);
6398 create_input_operand (&ops
[1], val
, mode
);
6399 create_integer_operand (&ops
[2], model
);
6400 if (maybe_expand_insn (icode
, 3, ops
))
6404 /* If using __sync_lock_release is a viable alternative, try it.
6405 Note that this will not be set to true if we are expanding a generic
6406 __atomic_store_n. */
6409 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
6410 if (icode
!= CODE_FOR_nothing
)
6412 create_fixed_operand (&ops
[0], mem
);
6413 create_input_operand (&ops
[1], const0_rtx
, mode
);
6414 if (maybe_expand_insn (icode
, 2, ops
))
6416 /* lock_release is only a release barrier. */
6417 if (is_mm_seq_cst (model
))
6418 expand_mem_thread_fence (model
);
6424 /* If the size of the object is greater than word size on this target,
6425 a default store will not be atomic. */
6426 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
6428 /* If loads are atomic or we are called to provide a __sync builtin,
6429 we can try a atomic_exchange and throw away the result. Otherwise,
6430 don't do anything so that we do not create an inconsistency between
6431 loads and stores. */
6432 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
6434 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
6436 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
6444 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6445 expand_mem_thread_fence (model
);
6447 emit_move_insn (mem
, val
);
6449 /* For SEQ_CST, also emit a barrier after the store. */
6450 if (is_mm_seq_cst (model
))
6451 expand_mem_thread_fence (model
);
6457 /* Structure containing the pointers and values required to process the
6458 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6460 struct atomic_op_functions
6462 direct_optab mem_fetch_before
;
6463 direct_optab mem_fetch_after
;
6464 direct_optab mem_no_result
;
6467 direct_optab no_result
;
6468 enum rtx_code reverse_code
;
6472 /* Fill in structure pointed to by OP with the various optab entries for an
6473 operation of type CODE. */
6476 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
6478 gcc_assert (op
!= NULL
);
6480 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6481 in the source code during compilation, and the optab entries are not
6482 computable until runtime. Fill in the values at runtime. */
6486 op
->mem_fetch_before
= atomic_fetch_add_optab
;
6487 op
->mem_fetch_after
= atomic_add_fetch_optab
;
6488 op
->mem_no_result
= atomic_add_optab
;
6489 op
->fetch_before
= sync_old_add_optab
;
6490 op
->fetch_after
= sync_new_add_optab
;
6491 op
->no_result
= sync_add_optab
;
6492 op
->reverse_code
= MINUS
;
6495 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
6496 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
6497 op
->mem_no_result
= atomic_sub_optab
;
6498 op
->fetch_before
= sync_old_sub_optab
;
6499 op
->fetch_after
= sync_new_sub_optab
;
6500 op
->no_result
= sync_sub_optab
;
6501 op
->reverse_code
= PLUS
;
6504 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
6505 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
6506 op
->mem_no_result
= atomic_xor_optab
;
6507 op
->fetch_before
= sync_old_xor_optab
;
6508 op
->fetch_after
= sync_new_xor_optab
;
6509 op
->no_result
= sync_xor_optab
;
6510 op
->reverse_code
= XOR
;
6513 op
->mem_fetch_before
= atomic_fetch_and_optab
;
6514 op
->mem_fetch_after
= atomic_and_fetch_optab
;
6515 op
->mem_no_result
= atomic_and_optab
;
6516 op
->fetch_before
= sync_old_and_optab
;
6517 op
->fetch_after
= sync_new_and_optab
;
6518 op
->no_result
= sync_and_optab
;
6519 op
->reverse_code
= UNKNOWN
;
6522 op
->mem_fetch_before
= atomic_fetch_or_optab
;
6523 op
->mem_fetch_after
= atomic_or_fetch_optab
;
6524 op
->mem_no_result
= atomic_or_optab
;
6525 op
->fetch_before
= sync_old_ior_optab
;
6526 op
->fetch_after
= sync_new_ior_optab
;
6527 op
->no_result
= sync_ior_optab
;
6528 op
->reverse_code
= UNKNOWN
;
6531 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
6532 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
6533 op
->mem_no_result
= atomic_nand_optab
;
6534 op
->fetch_before
= sync_old_nand_optab
;
6535 op
->fetch_after
= sync_new_nand_optab
;
6536 op
->no_result
= sync_nand_optab
;
6537 op
->reverse_code
= UNKNOWN
;
6544 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6545 using memory order MODEL. If AFTER is true the operation needs to return
6546 the value of *MEM after the operation, otherwise the previous value.
6547 TARGET is an optional place to place the result. The result is unused if
6549 Return the result if there is a better sequence, otherwise NULL_RTX. */
6552 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6553 enum memmodel model
, bool after
)
6555 /* If the value is prefetched, or not used, it may be possible to replace
6556 the sequence with a native exchange operation. */
6557 if (!after
|| target
== const0_rtx
)
6559 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
6560 if (code
== AND
&& val
== const0_rtx
)
6562 if (target
== const0_rtx
)
6563 target
= gen_reg_rtx (GET_MODE (mem
));
6564 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6567 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
6568 if (code
== IOR
&& val
== constm1_rtx
)
6570 if (target
== const0_rtx
)
6571 target
= gen_reg_rtx (GET_MODE (mem
));
6572 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6579 /* Try to emit an instruction for a specific operation varaition.
6580 OPTAB contains the OP functions.
6581 TARGET is an optional place to return the result. const0_rtx means unused.
6582 MEM is the memory location to operate on.
6583 VAL is the value to use in the operation.
6584 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6585 MODEL is the memory model, if used.
6586 AFTER is true if the returned result is the value after the operation. */
6589 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
6590 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
6592 machine_mode mode
= GET_MODE (mem
);
6593 struct expand_operand ops
[4];
6594 enum insn_code icode
;
6598 /* Check to see if there is a result returned. */
6599 if (target
== const0_rtx
)
6603 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
6604 create_integer_operand (&ops
[2], model
);
6609 icode
= direct_optab_handler (optab
->no_result
, mode
);
6613 /* Otherwise, we need to generate a result. */
6618 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
6619 : optab
->mem_fetch_before
, mode
);
6620 create_integer_operand (&ops
[3], model
);
6625 icode
= optab_handler (after
? optab
->fetch_after
6626 : optab
->fetch_before
, mode
);
6629 create_output_operand (&ops
[op_counter
++], target
, mode
);
6631 if (icode
== CODE_FOR_nothing
)
6634 create_fixed_operand (&ops
[op_counter
++], mem
);
6635 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6636 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
6638 if (maybe_expand_insn (icode
, num_ops
, ops
))
6639 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
6645 /* This function expands an atomic fetch_OP or OP_fetch operation:
6646 TARGET is an option place to stick the return value. const0_rtx indicates
6647 the result is unused.
6648 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6649 CODE is the operation being performed (OP)
6650 MEMMODEL is the memory model variant to use.
6651 AFTER is true to return the result of the operation (OP_fetch).
6652 AFTER is false to return the value before the operation (fetch_OP).
6654 This function will *only* generate instructions if there is a direct
6655 optab. No compare and swap loops or libcalls will be generated. */
6658 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
6659 enum rtx_code code
, enum memmodel model
,
6662 machine_mode mode
= GET_MODE (mem
);
6663 struct atomic_op_functions optab
;
6665 bool unused_result
= (target
== const0_rtx
);
6667 get_atomic_op_for_code (&optab
, code
);
6669 /* Check to see if there are any better instructions. */
6670 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
6674 /* Check for the case where the result isn't used and try those patterns. */
6677 /* Try the memory model variant first. */
6678 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
6682 /* Next try the old style withuot a memory model. */
6683 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
6687 /* There is no no-result pattern, so try patterns with a result. */
6691 /* Try the __atomic version. */
6692 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
6696 /* Try the older __sync version. */
6697 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
6701 /* If the fetch value can be calculated from the other variation of fetch,
6702 try that operation. */
6703 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
6705 /* Try the __atomic version, then the older __sync version. */
6706 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
6708 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
6712 /* If the result isn't used, no need to do compensation code. */
6716 /* Issue compensation code. Fetch_after == fetch_before OP val.
6717 Fetch_before == after REVERSE_OP val. */
6719 code
= optab
.reverse_code
;
6722 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
6723 true, OPTAB_LIB_WIDEN
);
6724 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
6727 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6728 true, OPTAB_LIB_WIDEN
);
6733 /* No direct opcode can be generated. */
6739 /* This function expands an atomic fetch_OP or OP_fetch operation:
6740 TARGET is an option place to stick the return value. const0_rtx indicates
6741 the result is unused.
6742 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6743 CODE is the operation being performed (OP)
6744 MEMMODEL is the memory model variant to use.
6745 AFTER is true to return the result of the operation (OP_fetch).
6746 AFTER is false to return the value before the operation (fetch_OP). */
6748 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6749 enum memmodel model
, bool after
)
6751 machine_mode mode
= GET_MODE (mem
);
6753 bool unused_result
= (target
== const0_rtx
);
6755 /* If loads are not atomic for the required size and we are not called to
6756 provide a __sync builtin, do not do anything so that we stay consistent
6757 with atomic loads of the same size. */
6758 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6761 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
6767 /* Add/sub can be implemented by doing the reverse operation with -(val). */
6768 if (code
== PLUS
|| code
== MINUS
)
6771 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
6774 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
6775 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
6779 /* PLUS worked so emit the insns and return. */
6786 /* PLUS did not work, so throw away the negation code and continue. */
6790 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
6791 if (!can_compare_and_swap_p (mode
, false))
6795 enum rtx_code orig_code
= code
;
6796 struct atomic_op_functions optab
;
6798 get_atomic_op_for_code (&optab
, code
);
6799 libfunc
= optab_libfunc (after
? optab
.fetch_after
6800 : optab
.fetch_before
, mode
);
6802 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
6806 code
= optab
.reverse_code
;
6807 libfunc
= optab_libfunc (after
? optab
.fetch_before
6808 : optab
.fetch_after
, mode
);
6810 if (libfunc
!= NULL
)
6812 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6813 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
6814 2, addr
, ptr_mode
, val
, mode
);
6816 if (!unused_result
&& fixup
)
6817 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6818 true, OPTAB_LIB_WIDEN
);
6822 /* We need the original code for any further attempts. */
6826 /* If nothing else has succeeded, default to a compare and swap loop. */
6827 if (can_compare_and_swap_p (mode
, true))
6830 rtx t0
= gen_reg_rtx (mode
), t1
;
6834 /* If the result is used, get a register for it. */
6837 if (!target
|| !register_operand (target
, mode
))
6838 target
= gen_reg_rtx (mode
);
6839 /* If fetch_before, copy the value now. */
6841 emit_move_insn (target
, t0
);
6844 target
= const0_rtx
;
6849 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
6850 true, OPTAB_LIB_WIDEN
);
6851 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
6854 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
6857 /* For after, copy the value now. */
6858 if (!unused_result
&& after
)
6859 emit_move_insn (target
, t1
);
6860 insn
= get_insns ();
6863 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
6870 /* Return true if OPERAND is suitable for operand number OPNO of
6871 instruction ICODE. */
6874 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
6876 return (!insn_data
[(int) icode
].operand
[opno
].predicate
6877 || (insn_data
[(int) icode
].operand
[opno
].predicate
6878 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
6881 /* TARGET is a target of a multiword operation that we are going to
6882 implement as a series of word-mode operations. Return true if
6883 TARGET is suitable for this purpose. */
6886 valid_multiword_target_p (rtx target
)
6891 mode
= GET_MODE (target
);
6892 for (i
= 0; i
< GET_MODE_SIZE (mode
); i
+= UNITS_PER_WORD
)
6893 if (!validate_subreg (word_mode
, mode
, target
, i
))
6898 /* Like maybe_legitimize_operand, but do not change the code of the
6899 current rtx value. */
6902 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
6903 struct expand_operand
*op
)
6905 /* See if the operand matches in its current form. */
6906 if (insn_operand_matches (icode
, opno
, op
->value
))
6909 /* If the operand is a memory whose address has no side effects,
6910 try forcing the address into a non-virtual pseudo register.
6911 The check for side effects is important because copy_to_mode_reg
6912 cannot handle things like auto-modified addresses. */
6913 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
6918 addr
= XEXP (mem
, 0);
6919 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
6920 && !side_effects_p (addr
))
6925 last
= get_last_insn ();
6926 mode
= get_address_mode (mem
);
6927 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
6928 if (insn_operand_matches (icode
, opno
, mem
))
6933 delete_insns_since (last
);
6940 /* Try to make OP match operand OPNO of instruction ICODE. Return true
6941 on success, storing the new operand value back in OP. */
6944 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
6945 struct expand_operand
*op
)
6947 machine_mode mode
, imode
;
6948 bool old_volatile_ok
, result
;
6954 old_volatile_ok
= volatile_ok
;
6956 result
= maybe_legitimize_operand_same_code (icode
, opno
, op
);
6957 volatile_ok
= old_volatile_ok
;
6961 gcc_assert (mode
!= VOIDmode
);
6963 && op
->value
!= const0_rtx
6964 && GET_MODE (op
->value
) == mode
6965 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
6968 op
->value
= gen_reg_rtx (mode
);
6974 gcc_assert (mode
!= VOIDmode
);
6975 gcc_assert (GET_MODE (op
->value
) == VOIDmode
6976 || GET_MODE (op
->value
) == mode
);
6977 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
6980 op
->value
= copy_to_mode_reg (mode
, op
->value
);
6983 case EXPAND_CONVERT_TO
:
6984 gcc_assert (mode
!= VOIDmode
);
6985 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
6988 case EXPAND_CONVERT_FROM
:
6989 if (GET_MODE (op
->value
) != VOIDmode
)
6990 mode
= GET_MODE (op
->value
);
6992 /* The caller must tell us what mode this value has. */
6993 gcc_assert (mode
!= VOIDmode
);
6995 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
6996 if (imode
!= VOIDmode
&& imode
!= mode
)
6998 op
->value
= convert_modes (imode
, mode
, op
->value
, op
->unsigned_p
);
7003 case EXPAND_ADDRESS
:
7004 gcc_assert (mode
!= VOIDmode
);
7005 op
->value
= convert_memory_address (mode
, op
->value
);
7008 case EXPAND_INTEGER
:
7009 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7010 if (mode
!= VOIDmode
&& const_int_operand (op
->value
, mode
))
7014 return insn_operand_matches (icode
, opno
, op
->value
);
7017 /* Make OP describe an input operand that should have the same value
7018 as VALUE, after any mode conversion that the target might request.
7019 TYPE is the type of VALUE. */
7022 create_convert_operand_from_type (struct expand_operand
*op
,
7023 rtx value
, tree type
)
7025 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
7026 TYPE_UNSIGNED (type
));
7029 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7030 of instruction ICODE. Return true on success, leaving the new operand
7031 values in the OPS themselves. Emit no code on failure. */
7034 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
7035 unsigned int nops
, struct expand_operand
*ops
)
7040 last
= get_last_insn ();
7041 for (i
= 0; i
< nops
; i
++)
7042 if (!maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
7044 delete_insns_since (last
);
7050 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7051 as its operands. Return the instruction pattern on success,
7052 and emit any necessary set-up code. Return null and emit no
7056 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
7057 struct expand_operand
*ops
)
7059 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
7060 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
7066 return GEN_FCN (icode
) (ops
[0].value
);
7068 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
7070 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
7072 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7075 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7076 ops
[3].value
, ops
[4].value
);
7078 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7079 ops
[3].value
, ops
[4].value
, ops
[5].value
);
7081 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7082 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7085 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7086 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7087 ops
[6].value
, ops
[7].value
);
7089 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7090 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7091 ops
[6].value
, ops
[7].value
, ops
[8].value
);
7096 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7097 as its operands. Return true on success and emit no code on failure. */
7100 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
7101 struct expand_operand
*ops
)
7103 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7112 /* Like maybe_expand_insn, but for jumps. */
7115 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7116 struct expand_operand
*ops
)
7118 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7121 emit_jump_insn (pat
);
7127 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7131 expand_insn (enum insn_code icode
, unsigned int nops
,
7132 struct expand_operand
*ops
)
7134 if (!maybe_expand_insn (icode
, nops
, ops
))
7138 /* Like expand_insn, but for jumps. */
7141 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7142 struct expand_operand
*ops
)
7144 if (!maybe_expand_jump_insn (icode
, nops
, ops
))