1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2017 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
37 /* Include insn-config.h before expr.h so that HAVE_conditional_move
38 is properly defined. */
39 #include "stor-layout.h"
44 #include "optabs-tree.h"
47 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
49 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
50 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
52 /* Debug facility for use in GDB. */
53 void debug_optab_libfuncs (void);
55 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
56 the result of operation CODE applied to OP0 (and OP1 if it is a binary
59 If the last insn does not set TARGET, don't do anything, but return 1.
61 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
62 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
63 try again, ensuring that TARGET is not one of the operands. */
66 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
, rtx op1
)
72 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
74 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
75 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
76 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
77 && GET_RTX_CLASS (code
) != RTX_COMPARE
78 && GET_RTX_CLASS (code
) != RTX_UNARY
)
81 if (GET_CODE (target
) == ZERO_EXTRACT
)
84 for (last_insn
= insns
;
85 NEXT_INSN (last_insn
) != NULL_RTX
;
86 last_insn
= NEXT_INSN (last_insn
))
89 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
90 a value changing in the insn, so the note would be invalid for CSE. */
91 if (reg_overlap_mentioned_p (target
, op0
)
92 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
95 && (rtx_equal_p (target
, op0
)
96 || (op1
&& rtx_equal_p (target
, op1
))))
98 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
99 over expanding it as temp = MEM op X, MEM = temp. If the target
100 supports MEM = MEM op X instructions, it is sometimes too hard
101 to reconstruct that form later, especially if X is also a memory,
102 and due to multiple occurrences of addresses the address might
103 be forced into register unnecessarily.
104 Note that not emitting the REG_EQUIV note might inhibit
105 CSE in some cases. */
106 set
= single_set (last_insn
);
108 && GET_CODE (SET_SRC (set
)) == code
109 && MEM_P (SET_DEST (set
))
110 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
111 || (op1
&& rtx_equal_p (SET_DEST (set
),
112 XEXP (SET_SRC (set
), 1)))))
118 set
= set_for_reg_notes (last_insn
);
122 if (! rtx_equal_p (SET_DEST (set
), target
)
123 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
124 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
125 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
128 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
138 if (GET_MODE (op0
) != VOIDmode
&& GET_MODE (target
) != GET_MODE (op0
))
140 note
= gen_rtx_fmt_e (code
, GET_MODE (op0
), copy_rtx (op0
));
141 if (GET_MODE_SIZE (GET_MODE (op0
))
142 > GET_MODE_SIZE (GET_MODE (target
)))
143 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
144 note
, GET_MODE (op0
));
146 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
147 note
, GET_MODE (op0
));
152 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
156 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
158 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
163 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
164 for a widening operation would be. In most cases this would be OP0, but if
165 that's a constant it'll be VOIDmode, which isn't useful. */
168 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
170 machine_mode m0
= GET_MODE (op0
);
171 machine_mode m1
= GET_MODE (op1
);
174 if (m0
== VOIDmode
&& m1
== VOIDmode
)
176 else if (m0
== VOIDmode
|| GET_MODE_SIZE (m0
) < GET_MODE_SIZE (m1
))
181 if (GET_MODE_SIZE (result
) > GET_MODE_SIZE (to_mode
))
187 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
188 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
189 not actually do a sign-extend or zero-extend, but can leave the
190 higher-order bits of the result rtx undefined, for example, in the case
191 of logical operations, but not right shifts. */
194 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
195 int unsignedp
, int no_extend
)
199 /* If we don't have to extend and this is a constant, return it. */
200 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
203 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
204 extend since it will be more efficient to do so unless the signedness of
205 a promoted object differs from our extension. */
207 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
208 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
209 return convert_modes (mode
, oldmode
, op
, unsignedp
);
211 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
213 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
214 return gen_lowpart (mode
, force_reg (GET_MODE (op
), op
));
216 /* Otherwise, get an object of MODE, clobber it, and set the low-order
219 result
= gen_reg_rtx (mode
);
220 emit_clobber (result
);
221 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
225 /* Expand vector widening operations.
227 There are two different classes of operations handled here:
228 1) Operations whose result is wider than all the arguments to the operation.
229 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
230 In this case OP0 and optionally OP1 would be initialized,
231 but WIDE_OP wouldn't (not relevant for this case).
232 2) Operations whose result is of the same size as the last argument to the
233 operation, but wider than all the other arguments to the operation.
234 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
235 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
237 E.g, when called to expand the following operations, this is how
238 the arguments will be initialized:
240 widening-sum 2 oprnd0 - oprnd1
241 widening-dot-product 3 oprnd0 oprnd1 oprnd2
242 widening-mult 2 oprnd0 oprnd1 -
243 type-promotion (vec-unpack) 1 oprnd0 - - */
246 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
247 rtx target
, int unsignedp
)
249 struct expand_operand eops
[4];
250 tree oprnd0
, oprnd1
, oprnd2
;
251 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
252 optab widen_pattern_optab
;
253 enum insn_code icode
;
254 int nops
= TREE_CODE_LENGTH (ops
->code
);
258 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
259 widen_pattern_optab
=
260 optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
261 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
262 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
263 icode
= find_widening_optab_handler (widen_pattern_optab
,
264 TYPE_MODE (TREE_TYPE (ops
->op2
)),
267 icode
= optab_handler (widen_pattern_optab
, tmode0
);
268 gcc_assert (icode
!= CODE_FOR_nothing
);
273 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
276 /* The last operand is of a wider mode than the rest of the operands. */
281 gcc_assert (tmode1
== tmode0
);
284 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
288 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
289 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
291 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
293 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
294 expand_insn (icode
, op
, eops
);
295 return eops
[0].value
;
298 /* Generate code to perform an operation specified by TERNARY_OPTAB
299 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
301 UNSIGNEDP is for the case where we have to widen the operands
302 to perform the operation. It says to use zero-extension.
304 If TARGET is nonzero, the value
305 is generated there, if it is convenient to do so.
306 In all cases an rtx is returned for the locus of the value;
307 this may or may not be TARGET. */
310 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
311 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
313 struct expand_operand ops
[4];
314 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
316 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
318 create_output_operand (&ops
[0], target
, mode
);
319 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
320 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
321 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
322 expand_insn (icode
, 4, ops
);
327 /* Like expand_binop, but return a constant rtx if the result can be
328 calculated at compile time. The arguments and return value are
329 otherwise the same as for expand_binop. */
332 simplify_expand_binop (machine_mode mode
, optab binoptab
,
333 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
334 enum optab_methods methods
)
336 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
338 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
344 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
347 /* Like simplify_expand_binop, but always put the result in TARGET.
348 Return true if the expansion succeeded. */
351 force_expand_binop (machine_mode mode
, optab binoptab
,
352 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
353 enum optab_methods methods
)
355 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
356 target
, unsignedp
, methods
);
360 emit_move_insn (target
, x
);
364 /* Create a new vector value in VMODE with all elements set to OP. The
365 mode of OP must be the element mode of VMODE. If OP is a constant,
366 then the return value will be a constant. */
369 expand_vector_broadcast (machine_mode vmode
, rtx op
)
371 enum insn_code icode
;
376 gcc_checking_assert (VECTOR_MODE_P (vmode
));
378 n
= GET_MODE_NUNITS (vmode
);
379 vec
= rtvec_alloc (n
);
380 for (i
= 0; i
< n
; ++i
)
381 RTVEC_ELT (vec
, i
) = op
;
384 return gen_rtx_CONST_VECTOR (vmode
, vec
);
386 /* ??? If the target doesn't have a vec_init, then we have no easy way
387 of performing this operation. Most of this sort of generic support
388 is hidden away in the vector lowering support in gimple. */
389 icode
= convert_optab_handler (vec_init_optab
, vmode
,
390 GET_MODE_INNER (vmode
));
391 if (icode
== CODE_FOR_nothing
)
394 ret
= gen_reg_rtx (vmode
);
395 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
400 /* This subroutine of expand_doubleword_shift handles the cases in which
401 the effective shift value is >= BITS_PER_WORD. The arguments and return
402 value are the same as for the parent routine, except that SUPERWORD_OP1
403 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
404 INTO_TARGET may be null if the caller has decided to calculate it. */
407 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
408 rtx outof_target
, rtx into_target
,
409 int unsignedp
, enum optab_methods methods
)
411 if (into_target
!= 0)
412 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
413 into_target
, unsignedp
, methods
))
416 if (outof_target
!= 0)
418 /* For a signed right shift, we must fill OUTOF_TARGET with copies
419 of the sign bit, otherwise we must fill it with zeros. */
420 if (binoptab
!= ashr_optab
)
421 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
423 if (!force_expand_binop (word_mode
, binoptab
,
424 outof_input
, GEN_INT (BITS_PER_WORD
- 1),
425 outof_target
, unsignedp
, methods
))
431 /* This subroutine of expand_doubleword_shift handles the cases in which
432 the effective shift value is < BITS_PER_WORD. The arguments and return
433 value are the same as for the parent routine. */
436 expand_subword_shift (machine_mode op1_mode
, optab binoptab
,
437 rtx outof_input
, rtx into_input
, rtx op1
,
438 rtx outof_target
, rtx into_target
,
439 int unsignedp
, enum optab_methods methods
,
440 unsigned HOST_WIDE_INT shift_mask
)
442 optab reverse_unsigned_shift
, unsigned_shift
;
445 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
446 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
448 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
449 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
450 the opposite direction to BINOPTAB. */
451 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
453 carries
= outof_input
;
454 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
455 op1_mode
), op1_mode
);
456 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
461 /* We must avoid shifting by BITS_PER_WORD bits since that is either
462 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
463 has unknown behavior. Do a single shift first, then shift by the
464 remainder. It's OK to use ~OP1 as the remainder if shift counts
465 are truncated to the mode size. */
466 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
467 outof_input
, const1_rtx
, 0, unsignedp
, methods
);
468 if (shift_mask
== BITS_PER_WORD
- 1)
470 tmp
= immed_wide_int_const
471 (wi::minus_one (GET_MODE_PRECISION (op1_mode
)), op1_mode
);
472 tmp
= simplify_expand_binop (op1_mode
, xor_optab
, op1
, tmp
,
477 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
478 op1_mode
), op1_mode
);
479 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
483 if (tmp
== 0 || carries
== 0)
485 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
486 carries
, tmp
, 0, unsignedp
, methods
);
490 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
491 so the result can go directly into INTO_TARGET if convenient. */
492 tmp
= expand_binop (word_mode
, unsigned_shift
, into_input
, op1
,
493 into_target
, unsignedp
, methods
);
497 /* Now OR in the bits carried over from OUTOF_INPUT. */
498 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
499 into_target
, unsignedp
, methods
))
502 /* Use a standard word_mode shift for the out-of half. */
503 if (outof_target
!= 0)
504 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
505 outof_target
, unsignedp
, methods
))
512 /* Try implementing expand_doubleword_shift using conditional moves.
513 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
514 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
515 are the shift counts to use in the former and latter case. All other
516 arguments are the same as the parent routine. */
519 expand_doubleword_shift_condmove (machine_mode op1_mode
, optab binoptab
,
520 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
521 rtx outof_input
, rtx into_input
,
522 rtx subword_op1
, rtx superword_op1
,
523 rtx outof_target
, rtx into_target
,
524 int unsignedp
, enum optab_methods methods
,
525 unsigned HOST_WIDE_INT shift_mask
)
527 rtx outof_superword
, into_superword
;
529 /* Put the superword version of the output into OUTOF_SUPERWORD and
531 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
532 if (outof_target
!= 0 && subword_op1
== superword_op1
)
534 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
535 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
536 into_superword
= outof_target
;
537 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
538 outof_superword
, 0, unsignedp
, methods
))
543 into_superword
= gen_reg_rtx (word_mode
);
544 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
545 outof_superword
, into_superword
,
550 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
551 if (!expand_subword_shift (op1_mode
, binoptab
,
552 outof_input
, into_input
, subword_op1
,
553 outof_target
, into_target
,
554 unsignedp
, methods
, shift_mask
))
557 /* Select between them. Do the INTO half first because INTO_SUPERWORD
558 might be the current value of OUTOF_TARGET. */
559 if (!emit_conditional_move (into_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
560 into_target
, into_superword
, word_mode
, false))
563 if (outof_target
!= 0)
564 if (!emit_conditional_move (outof_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
565 outof_target
, outof_superword
,
572 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
573 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
574 input operand; the shift moves bits in the direction OUTOF_INPUT->
575 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
576 of the target. OP1 is the shift count and OP1_MODE is its mode.
577 If OP1 is constant, it will have been truncated as appropriate
578 and is known to be nonzero.
580 If SHIFT_MASK is zero, the result of word shifts is undefined when the
581 shift count is outside the range [0, BITS_PER_WORD). This routine must
582 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
584 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
585 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
586 fill with zeros or sign bits as appropriate.
588 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
589 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
590 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
591 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
594 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
595 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
596 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
597 function wants to calculate it itself.
599 Return true if the shift could be successfully synthesized. */
602 expand_doubleword_shift (machine_mode op1_mode
, optab binoptab
,
603 rtx outof_input
, rtx into_input
, rtx op1
,
604 rtx outof_target
, rtx into_target
,
605 int unsignedp
, enum optab_methods methods
,
606 unsigned HOST_WIDE_INT shift_mask
)
608 rtx superword_op1
, tmp
, cmp1
, cmp2
;
609 enum rtx_code cmp_code
;
611 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
612 fill the result with sign or zero bits as appropriate. If so, the value
613 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
614 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
615 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
617 This isn't worthwhile for constant shifts since the optimizers will
618 cope better with in-range shift counts. */
619 if (shift_mask
>= BITS_PER_WORD
621 && !CONSTANT_P (op1
))
623 if (!expand_doubleword_shift (op1_mode
, binoptab
,
624 outof_input
, into_input
, op1
,
626 unsignedp
, methods
, shift_mask
))
628 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
629 outof_target
, unsignedp
, methods
))
634 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
635 is true when the effective shift value is less than BITS_PER_WORD.
636 Set SUPERWORD_OP1 to the shift count that should be used to shift
637 OUTOF_INPUT into INTO_TARGET when the condition is false. */
638 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
639 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
641 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
642 is a subword shift count. */
643 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
645 cmp2
= CONST0_RTX (op1_mode
);
651 /* Set CMP1 to OP1 - BITS_PER_WORD. */
652 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
654 cmp2
= CONST0_RTX (op1_mode
);
656 superword_op1
= cmp1
;
661 /* If we can compute the condition at compile time, pick the
662 appropriate subroutine. */
663 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
664 if (tmp
!= 0 && CONST_INT_P (tmp
))
666 if (tmp
== const0_rtx
)
667 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
668 outof_target
, into_target
,
671 return expand_subword_shift (op1_mode
, binoptab
,
672 outof_input
, into_input
, op1
,
673 outof_target
, into_target
,
674 unsignedp
, methods
, shift_mask
);
677 /* Try using conditional moves to generate straight-line code. */
678 if (HAVE_conditional_move
)
680 rtx_insn
*start
= get_last_insn ();
681 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
682 cmp_code
, cmp1
, cmp2
,
683 outof_input
, into_input
,
685 outof_target
, into_target
,
686 unsignedp
, methods
, shift_mask
))
688 delete_insns_since (start
);
691 /* As a last resort, use branches to select the correct alternative. */
692 rtx_code_label
*subword_label
= gen_label_rtx ();
693 rtx_code_label
*done_label
= gen_label_rtx ();
696 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
698 profile_probability::uninitialized ());
701 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
702 outof_target
, into_target
,
706 emit_jump_insn (targetm
.gen_jump (done_label
));
708 emit_label (subword_label
);
710 if (!expand_subword_shift (op1_mode
, binoptab
,
711 outof_input
, into_input
, op1
,
712 outof_target
, into_target
,
713 unsignedp
, methods
, shift_mask
))
716 emit_label (done_label
);
720 /* Subroutine of expand_binop. Perform a double word multiplication of
721 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
722 as the target's word_mode. This function return NULL_RTX if anything
723 goes wrong, in which case it may have already emitted instructions
724 which need to be deleted.
726 If we want to multiply two two-word values and have normal and widening
727 multiplies of single-word values, we can do this with three smaller
730 The multiplication proceeds as follows:
731 _______________________
732 [__op0_high_|__op0_low__]
733 _______________________
734 * [__op1_high_|__op1_low__]
735 _______________________________________________
736 _______________________
737 (1) [__op0_low__*__op1_low__]
738 _______________________
739 (2a) [__op0_low__*__op1_high_]
740 _______________________
741 (2b) [__op0_high_*__op1_low__]
742 _______________________
743 (3) [__op0_high_*__op1_high_]
746 This gives a 4-word result. Since we are only interested in the
747 lower 2 words, partial result (3) and the upper words of (2a) and
748 (2b) don't need to be calculated. Hence (2a) and (2b) can be
749 calculated using non-widening multiplication.
751 (1), however, needs to be calculated with an unsigned widening
752 multiplication. If this operation is not directly supported we
753 try using a signed widening multiplication and adjust the result.
754 This adjustment works as follows:
756 If both operands are positive then no adjustment is needed.
758 If the operands have different signs, for example op0_low < 0 and
759 op1_low >= 0, the instruction treats the most significant bit of
760 op0_low as a sign bit instead of a bit with significance
761 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
762 with 2**BITS_PER_WORD - op0_low, and two's complements the
763 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
766 Similarly, if both operands are negative, we need to add
767 (op0_low + op1_low) * 2**BITS_PER_WORD.
769 We use a trick to adjust quickly. We logically shift op0_low right
770 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
771 op0_high (op1_high) before it is used to calculate 2b (2a). If no
772 logical shift exists, we do an arithmetic right shift and subtract
776 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
777 bool umulp
, enum optab_methods methods
)
779 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
780 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
781 rtx wordm1
= umulp
? NULL_RTX
: GEN_INT (BITS_PER_WORD
- 1);
782 rtx product
, adjust
, product_high
, temp
;
784 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
785 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
786 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
787 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
789 /* If we're using an unsigned multiply to directly compute the product
790 of the low-order words of the operands and perform any required
791 adjustments of the operands, we begin by trying two more multiplications
792 and then computing the appropriate sum.
794 We have checked above that the required addition is provided.
795 Full-word addition will normally always succeed, especially if
796 it is provided at all, so we don't worry about its failure. The
797 multiplication may well fail, however, so we do handle that. */
801 /* ??? This could be done with emit_store_flag where available. */
802 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
803 NULL_RTX
, 1, methods
);
805 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
806 NULL_RTX
, 0, OPTAB_DIRECT
);
809 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
810 NULL_RTX
, 0, methods
);
813 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
814 NULL_RTX
, 0, OPTAB_DIRECT
);
821 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
822 NULL_RTX
, 0, OPTAB_DIRECT
);
826 /* OP0_HIGH should now be dead. */
830 /* ??? This could be done with emit_store_flag where available. */
831 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
832 NULL_RTX
, 1, methods
);
834 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
835 NULL_RTX
, 0, OPTAB_DIRECT
);
838 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
839 NULL_RTX
, 0, methods
);
842 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
843 NULL_RTX
, 0, OPTAB_DIRECT
);
850 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
851 NULL_RTX
, 0, OPTAB_DIRECT
);
855 /* OP1_HIGH should now be dead. */
857 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
858 NULL_RTX
, 0, OPTAB_DIRECT
);
860 if (target
&& !REG_P (target
))
864 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
865 target
, 1, OPTAB_DIRECT
);
867 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
868 target
, 1, OPTAB_DIRECT
);
873 product_high
= operand_subword (product
, high
, 1, mode
);
874 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
875 NULL_RTX
, 0, OPTAB_DIRECT
);
876 emit_move_insn (product_high
, adjust
);
880 /* Wrapper around expand_binop which takes an rtx code to specify
881 the operation to perform, not an optab pointer. All other
882 arguments are the same. */
884 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
885 rtx op1
, rtx target
, int unsignedp
,
886 enum optab_methods methods
)
888 optab binop
= code_to_optab (code
);
891 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
894 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
895 binop. Order them according to commutative_operand_precedence and, if
896 possible, try to put TARGET or a pseudo first. */
898 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
900 int op0_prec
= commutative_operand_precedence (op0
);
901 int op1_prec
= commutative_operand_precedence (op1
);
903 if (op0_prec
< op1_prec
)
906 if (op0_prec
> op1_prec
)
909 /* With equal precedence, both orders are ok, but it is better if the
910 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
911 if (target
== 0 || REG_P (target
))
912 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
914 return rtx_equal_p (op1
, target
);
917 /* Return true if BINOPTAB implements a shift operation. */
920 shift_optab_p (optab binoptab
)
922 switch (optab_to_code (binoptab
))
938 /* Return true if BINOPTAB implements a commutative binary operation. */
941 commutative_optab_p (optab binoptab
)
943 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
944 || binoptab
== smul_widen_optab
945 || binoptab
== umul_widen_optab
946 || binoptab
== smul_highpart_optab
947 || binoptab
== umul_highpart_optab
);
950 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
951 optimizing, and if the operand is a constant that costs more than
952 1 instruction, force the constant into a register and return that
953 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
956 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
957 int opn
, rtx x
, bool unsignedp
)
959 bool speed
= optimize_insn_for_speed_p ();
964 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
965 > set_src_cost (x
, mode
, speed
)))
969 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
970 if (intval
!= INTVAL (x
))
971 x
= GEN_INT (intval
);
974 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
975 x
= force_reg (mode
, x
);
980 /* Helper function for expand_binop: handle the case where there
981 is an insn that directly implements the indicated operation.
982 Returns null if this is not possible. */
984 expand_binop_directly (machine_mode mode
, optab binoptab
,
986 rtx target
, int unsignedp
, enum optab_methods methods
,
989 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
990 enum insn_code icode
= find_widening_optab_handler (binoptab
, mode
,
992 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
993 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
994 machine_mode mode0
, mode1
, tmp_mode
;
995 struct expand_operand ops
[3];
998 rtx xop0
= op0
, xop1
= op1
;
999 bool canonicalize_op1
= false;
1001 /* If it is a commutative operator and the modes would match
1002 if we would swap the operands, we can save the conversions. */
1003 commutative_p
= commutative_optab_p (binoptab
);
1005 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1006 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode1
)
1007 std::swap (xop0
, xop1
);
1009 /* If we are optimizing, force expensive constants into a register. */
1010 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1011 if (!shift_optab_p (binoptab
))
1012 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1014 /* Shifts and rotates often use a different mode for op1 from op0;
1015 for VOIDmode constants we don't know the mode, so force it
1016 to be canonicalized using convert_modes. */
1017 canonicalize_op1
= true;
1019 /* In case the insn wants input operands in modes different from
1020 those of the actual operands, convert the operands. It would
1021 seem that we don't need to convert CONST_INTs, but we do, so
1022 that they're properly zero-extended, sign-extended or truncated
1025 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1026 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1028 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1032 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1033 ? GET_MODE (xop1
) : mode
);
1034 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1036 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1040 /* If operation is commutative,
1041 try to make the first operand a register.
1042 Even better, try to make it the same as the target.
1043 Also try to make the last operand a constant. */
1045 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1046 std::swap (xop0
, xop1
);
1048 /* Now, if insn's predicates don't allow our operands, put them into
1051 if (binoptab
== vec_pack_trunc_optab
1052 || binoptab
== vec_pack_usat_optab
1053 || binoptab
== vec_pack_ssat_optab
1054 || binoptab
== vec_pack_ufix_trunc_optab
1055 || binoptab
== vec_pack_sfix_trunc_optab
)
1057 /* The mode of the result is different then the mode of the
1059 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1060 if (VECTOR_MODE_P (mode
)
1061 && GET_MODE_NUNITS (tmp_mode
) != 2 * GET_MODE_NUNITS (mode
))
1063 delete_insns_since (last
);
1070 create_output_operand (&ops
[0], target
, tmp_mode
);
1071 create_input_operand (&ops
[1], xop0
, mode0
);
1072 create_input_operand (&ops
[2], xop1
, mode1
);
1073 pat
= maybe_gen_insn (icode
, 3, ops
);
1076 /* If PAT is composed of more than one insn, try to add an appropriate
1077 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1078 operand, call expand_binop again, this time without a target. */
1079 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1080 && ! add_equal_note (pat
, ops
[0].value
,
1081 optab_to_code (binoptab
),
1082 ops
[1].value
, ops
[2].value
))
1084 delete_insns_since (last
);
1085 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1086 unsignedp
, methods
);
1090 return ops
[0].value
;
1092 delete_insns_since (last
);
1096 /* Generate code to perform an operation specified by BINOPTAB
1097 on operands OP0 and OP1, with result having machine-mode MODE.
1099 UNSIGNEDP is for the case where we have to widen the operands
1100 to perform the operation. It says to use zero-extension.
1102 If TARGET is nonzero, the value
1103 is generated there, if it is convenient to do so.
1104 In all cases an rtx is returned for the locus of the value;
1105 this may or may not be TARGET. */
1108 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1109 rtx target
, int unsignedp
, enum optab_methods methods
)
1111 enum optab_methods next_methods
1112 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1113 ? OPTAB_WIDEN
: methods
);
1114 enum mode_class mclass
;
1115 machine_mode wider_mode
;
1118 rtx_insn
*entry_last
= get_last_insn ();
1121 mclass
= GET_MODE_CLASS (mode
);
1123 /* If subtracting an integer constant, convert this into an addition of
1124 the negated constant. */
1126 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1128 op1
= negate_rtx (mode
, op1
);
1129 binoptab
= add_optab
;
1131 /* For shifts, constant invalid op1 might be expanded from different
1132 mode than MODE. As those are invalid, force them to a register
1133 to avoid further problems during expansion. */
1134 else if (CONST_INT_P (op1
)
1135 && shift_optab_p (binoptab
)
1136 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1138 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1139 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1142 /* Record where to delete back to if we backtrack. */
1143 last
= get_last_insn ();
1145 /* If we can do it with a three-operand insn, do so. */
1147 if (methods
!= OPTAB_MUST_WIDEN
1148 && find_widening_optab_handler (binoptab
, mode
,
1149 widened_mode (mode
, op0
, op1
), 1)
1150 != CODE_FOR_nothing
)
1152 temp
= expand_binop_directly (mode
, binoptab
, op0
, op1
, target
,
1153 unsignedp
, methods
, last
);
1158 /* If we were trying to rotate, and that didn't work, try rotating
1159 the other direction before falling back to shifts and bitwise-or. */
1160 if (((binoptab
== rotl_optab
1161 && optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
1162 || (binoptab
== rotr_optab
1163 && optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
))
1164 && mclass
== MODE_INT
)
1166 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1168 unsigned int bits
= GET_MODE_PRECISION (mode
);
1170 if (CONST_INT_P (op1
))
1171 newop1
= GEN_INT (bits
- INTVAL (op1
));
1172 else if (targetm
.shift_truncation_mask (mode
) == bits
- 1)
1173 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1175 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1176 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1177 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1179 temp
= expand_binop_directly (mode
, otheroptab
, op0
, newop1
,
1180 target
, unsignedp
, methods
, last
);
1185 /* If this is a multiply, see if we can do a widening operation that
1186 takes operands of this mode and makes a wider mode. */
1188 if (binoptab
== smul_optab
1189 && GET_MODE_2XWIDER_MODE (mode
).exists (&wider_mode
)
1190 && (convert_optab_handler ((unsignedp
1192 : smul_widen_optab
),
1193 wider_mode
, mode
) != CODE_FOR_nothing
))
1195 temp
= expand_binop (wider_mode
,
1196 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1197 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1201 if (GET_MODE_CLASS (mode
) == MODE_INT
1202 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1203 return gen_lowpart (mode
, temp
);
1205 return convert_to_mode (mode
, temp
, unsignedp
);
1209 /* If this is a vector shift by a scalar, see if we can do a vector
1210 shift by a vector. If so, broadcast the scalar into a vector. */
1211 if (mclass
== MODE_VECTOR_INT
)
1213 optab otheroptab
= unknown_optab
;
1215 if (binoptab
== ashl_optab
)
1216 otheroptab
= vashl_optab
;
1217 else if (binoptab
== ashr_optab
)
1218 otheroptab
= vashr_optab
;
1219 else if (binoptab
== lshr_optab
)
1220 otheroptab
= vlshr_optab
;
1221 else if (binoptab
== rotl_optab
)
1222 otheroptab
= vrotl_optab
;
1223 else if (binoptab
== rotr_optab
)
1224 otheroptab
= vrotr_optab
;
1226 if (otheroptab
&& optab_handler (otheroptab
, mode
) != CODE_FOR_nothing
)
1228 /* The scalar may have been extended to be too wide. Truncate
1229 it back to the proper size to fit in the broadcast vector. */
1230 machine_mode inner_mode
= GET_MODE_INNER (mode
);
1231 if (!CONST_INT_P (op1
)
1232 && (GET_MODE_BITSIZE (inner_mode
)
1233 < GET_MODE_BITSIZE (GET_MODE (op1
))))
1234 op1
= force_reg (inner_mode
,
1235 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1237 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1240 temp
= expand_binop_directly (mode
, otheroptab
, op0
, vop1
,
1241 target
, unsignedp
, methods
, last
);
1248 /* Look for a wider mode of the same class for which we think we
1249 can open-code the operation. Check for a widening multiply at the
1250 wider mode as well. */
1252 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1253 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1254 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1256 machine_mode next_mode
;
1257 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1258 || (binoptab
== smul_optab
1259 && GET_MODE_WIDER_MODE (wider_mode
).exists (&next_mode
)
1260 && (find_widening_optab_handler ((unsignedp
1262 : smul_widen_optab
),
1264 != CODE_FOR_nothing
)))
1266 rtx xop0
= op0
, xop1
= op1
;
1269 /* For certain integer operations, we need not actually extend
1270 the narrow operands, as long as we will truncate
1271 the results to the same narrowness. */
1273 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1274 || binoptab
== xor_optab
1275 || binoptab
== add_optab
|| binoptab
== sub_optab
1276 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1277 && mclass
== MODE_INT
)
1280 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1282 if (binoptab
!= ashl_optab
)
1283 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1287 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1289 /* The second operand of a shift must always be extended. */
1290 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1291 no_extend
&& binoptab
!= ashl_optab
);
1293 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1294 unsignedp
, OPTAB_DIRECT
);
1297 if (mclass
!= MODE_INT
1298 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1301 target
= gen_reg_rtx (mode
);
1302 convert_move (target
, temp
, 0);
1306 return gen_lowpart (mode
, temp
);
1309 delete_insns_since (last
);
1313 /* If operation is commutative,
1314 try to make the first operand a register.
1315 Even better, try to make it the same as the target.
1316 Also try to make the last operand a constant. */
1317 if (commutative_optab_p (binoptab
)
1318 && swap_commutative_operands_with_target (target
, op0
, op1
))
1319 std::swap (op0
, op1
);
1321 /* These can be done a word at a time. */
1322 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1323 && mclass
== MODE_INT
1324 && GET_MODE_SIZE (mode
) > UNITS_PER_WORD
1325 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1330 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1331 won't be accurate, so use a new target. */
1335 || !valid_multiword_target_p (target
))
1336 target
= gen_reg_rtx (mode
);
1340 /* Do the actual arithmetic. */
1341 for (i
= 0; i
< GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
; i
++)
1343 rtx target_piece
= operand_subword (target
, i
, 1, mode
);
1344 rtx x
= expand_binop (word_mode
, binoptab
,
1345 operand_subword_force (op0
, i
, mode
),
1346 operand_subword_force (op1
, i
, mode
),
1347 target_piece
, unsignedp
, next_methods
);
1352 if (target_piece
!= x
)
1353 emit_move_insn (target_piece
, x
);
1356 insns
= get_insns ();
1359 if (i
== GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
)
1366 /* Synthesize double word shifts from single word shifts. */
1367 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1368 || binoptab
== ashr_optab
)
1369 && mclass
== MODE_INT
1370 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1371 && GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
1372 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
)
1373 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1374 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1375 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1377 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1378 machine_mode op1_mode
;
1380 double_shift_mask
= targetm
.shift_truncation_mask (mode
);
1381 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1382 op1_mode
= GET_MODE (op1
) != VOIDmode
? GET_MODE (op1
) : word_mode
;
1384 /* Apply the truncation to constant shifts. */
1385 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1386 op1
= GEN_INT (INTVAL (op1
) & double_shift_mask
);
1388 if (op1
== CONST0_RTX (op1_mode
))
1391 /* Make sure that this is a combination that expand_doubleword_shift
1392 can handle. See the comments there for details. */
1393 if (double_shift_mask
== 0
1394 || (shift_mask
== BITS_PER_WORD
- 1
1395 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1398 rtx into_target
, outof_target
;
1399 rtx into_input
, outof_input
;
1400 int left_shift
, outof_word
;
1402 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1403 won't be accurate, so use a new target. */
1407 || !valid_multiword_target_p (target
))
1408 target
= gen_reg_rtx (mode
);
1412 /* OUTOF_* is the word we are shifting bits away from, and
1413 INTO_* is the word that we are shifting bits towards, thus
1414 they differ depending on the direction of the shift and
1415 WORDS_BIG_ENDIAN. */
1417 left_shift
= binoptab
== ashl_optab
;
1418 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1420 outof_target
= operand_subword (target
, outof_word
, 1, mode
);
1421 into_target
= operand_subword (target
, 1 - outof_word
, 1, mode
);
1423 outof_input
= operand_subword_force (op0
, outof_word
, mode
);
1424 into_input
= operand_subword_force (op0
, 1 - outof_word
, mode
);
1426 if (expand_doubleword_shift (op1_mode
, binoptab
,
1427 outof_input
, into_input
, op1
,
1428 outof_target
, into_target
,
1429 unsignedp
, next_methods
, shift_mask
))
1431 insns
= get_insns ();
1441 /* Synthesize double word rotates from single word shifts. */
1442 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1443 && mclass
== MODE_INT
1444 && CONST_INT_P (op1
)
1445 && GET_MODE_PRECISION (mode
) == 2 * BITS_PER_WORD
1446 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1447 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1450 rtx into_target
, outof_target
;
1451 rtx into_input
, outof_input
;
1453 int shift_count
, left_shift
, outof_word
;
1455 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1456 won't be accurate, so use a new target. Do this also if target is not
1457 a REG, first because having a register instead may open optimization
1458 opportunities, and second because if target and op0 happen to be MEMs
1459 designating the same location, we would risk clobbering it too early
1460 in the code sequence we generate below. */
1465 || !valid_multiword_target_p (target
))
1466 target
= gen_reg_rtx (mode
);
1470 shift_count
= INTVAL (op1
);
1472 /* OUTOF_* is the word we are shifting bits away from, and
1473 INTO_* is the word that we are shifting bits towards, thus
1474 they differ depending on the direction of the shift and
1475 WORDS_BIG_ENDIAN. */
1477 left_shift
= (binoptab
== rotl_optab
);
1478 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1480 outof_target
= operand_subword (target
, outof_word
, 1, mode
);
1481 into_target
= operand_subword (target
, 1 - outof_word
, 1, mode
);
1483 outof_input
= operand_subword_force (op0
, outof_word
, mode
);
1484 into_input
= operand_subword_force (op0
, 1 - outof_word
, mode
);
1486 if (shift_count
== BITS_PER_WORD
)
1488 /* This is just a word swap. */
1489 emit_move_insn (outof_target
, into_input
);
1490 emit_move_insn (into_target
, outof_input
);
1495 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1496 rtx first_shift_count
, second_shift_count
;
1497 optab reverse_unsigned_shift
, unsigned_shift
;
1499 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1500 ? lshr_optab
: ashl_optab
);
1502 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1503 ? ashl_optab
: lshr_optab
);
1505 if (shift_count
> BITS_PER_WORD
)
1507 first_shift_count
= GEN_INT (shift_count
- BITS_PER_WORD
);
1508 second_shift_count
= GEN_INT (2 * BITS_PER_WORD
- shift_count
);
1512 first_shift_count
= GEN_INT (BITS_PER_WORD
- shift_count
);
1513 second_shift_count
= GEN_INT (shift_count
);
1516 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1517 outof_input
, first_shift_count
,
1518 NULL_RTX
, unsignedp
, next_methods
);
1519 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1520 into_input
, second_shift_count
,
1521 NULL_RTX
, unsignedp
, next_methods
);
1523 if (into_temp1
!= 0 && into_temp2
!= 0)
1524 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1525 into_target
, unsignedp
, next_methods
);
1529 if (inter
!= 0 && inter
!= into_target
)
1530 emit_move_insn (into_target
, inter
);
1532 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1533 into_input
, first_shift_count
,
1534 NULL_RTX
, unsignedp
, next_methods
);
1535 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1536 outof_input
, second_shift_count
,
1537 NULL_RTX
, unsignedp
, next_methods
);
1539 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1540 inter
= expand_binop (word_mode
, ior_optab
,
1541 outof_temp1
, outof_temp2
,
1542 outof_target
, unsignedp
, next_methods
);
1544 if (inter
!= 0 && inter
!= outof_target
)
1545 emit_move_insn (outof_target
, inter
);
1548 insns
= get_insns ();
1558 /* These can be done a word at a time by propagating carries. */
1559 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1560 && mclass
== MODE_INT
1561 && GET_MODE_SIZE (mode
) >= 2 * UNITS_PER_WORD
1562 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1565 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1566 const unsigned int nwords
= GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
;
1567 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1568 rtx xop0
, xop1
, xtarget
;
1570 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1571 value is one of those, use it. Otherwise, use 1 since it is the
1572 one easiest to get. */
1573 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1574 int normalizep
= STORE_FLAG_VALUE
;
1579 /* Prepare the operands. */
1580 xop0
= force_reg (mode
, op0
);
1581 xop1
= force_reg (mode
, op1
);
1583 xtarget
= gen_reg_rtx (mode
);
1585 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1588 /* Indicate for flow that the entire target reg is being set. */
1590 emit_clobber (xtarget
);
1592 /* Do the actual arithmetic. */
1593 for (i
= 0; i
< nwords
; i
++)
1595 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
1596 rtx target_piece
= operand_subword (xtarget
, index
, 1, mode
);
1597 rtx op0_piece
= operand_subword_force (xop0
, index
, mode
);
1598 rtx op1_piece
= operand_subword_force (xop1
, index
, mode
);
1601 /* Main add/subtract of the input operands. */
1602 x
= expand_binop (word_mode
, binoptab
,
1603 op0_piece
, op1_piece
,
1604 target_piece
, unsignedp
, next_methods
);
1610 /* Store carry from main add/subtract. */
1611 carry_out
= gen_reg_rtx (word_mode
);
1612 carry_out
= emit_store_flag_force (carry_out
,
1613 (binoptab
== add_optab
1616 word_mode
, 1, normalizep
);
1623 /* Add/subtract previous carry to main result. */
1624 newx
= expand_binop (word_mode
,
1625 normalizep
== 1 ? binoptab
: otheroptab
,
1627 NULL_RTX
, 1, next_methods
);
1631 /* Get out carry from adding/subtracting carry in. */
1632 rtx carry_tmp
= gen_reg_rtx (word_mode
);
1633 carry_tmp
= emit_store_flag_force (carry_tmp
,
1634 (binoptab
== add_optab
1637 word_mode
, 1, normalizep
);
1639 /* Logical-ior the two poss. carry together. */
1640 carry_out
= expand_binop (word_mode
, ior_optab
,
1641 carry_out
, carry_tmp
,
1642 carry_out
, 0, next_methods
);
1646 emit_move_insn (target_piece
, newx
);
1650 if (x
!= target_piece
)
1651 emit_move_insn (target_piece
, x
);
1654 carry_in
= carry_out
;
1657 if (i
== GET_MODE_BITSIZE (mode
) / (unsigned) BITS_PER_WORD
)
1659 if (optab_handler (mov_optab
, mode
) != CODE_FOR_nothing
1660 || ! rtx_equal_p (target
, xtarget
))
1662 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
1664 set_dst_reg_note (temp
, REG_EQUAL
,
1665 gen_rtx_fmt_ee (optab_to_code (binoptab
),
1666 mode
, copy_rtx (xop0
),
1677 delete_insns_since (last
);
1680 /* Attempt to synthesize double word multiplies using a sequence of word
1681 mode multiplications. We first attempt to generate a sequence using a
1682 more efficient unsigned widening multiply, and if that fails we then
1683 try using a signed widening multiply. */
1685 if (binoptab
== smul_optab
1686 && mclass
== MODE_INT
1687 && GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
1688 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
1689 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
1691 rtx product
= NULL_RTX
;
1692 if (widening_optab_handler (umul_widen_optab
, mode
, word_mode
)
1693 != CODE_FOR_nothing
)
1695 product
= expand_doubleword_mult (mode
, op0
, op1
, target
,
1698 delete_insns_since (last
);
1701 if (product
== NULL_RTX
1702 && widening_optab_handler (smul_widen_optab
, mode
, word_mode
)
1703 != CODE_FOR_nothing
)
1705 product
= expand_doubleword_mult (mode
, op0
, op1
, target
,
1708 delete_insns_since (last
);
1711 if (product
!= NULL_RTX
)
1713 if (optab_handler (mov_optab
, mode
) != CODE_FOR_nothing
)
1715 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
1717 set_dst_reg_note (move
,
1719 gen_rtx_fmt_ee (MULT
, mode
,
1722 target
? target
: product
);
1728 /* It can't be open-coded in this mode.
1729 Use a library call if one is available and caller says that's ok. */
1731 libfunc
= optab_libfunc (binoptab
, mode
);
1733 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
1737 machine_mode op1_mode
= mode
;
1742 if (shift_optab_p (binoptab
))
1744 op1_mode
= targetm
.libgcc_shift_count_mode ();
1745 /* Specify unsigned here,
1746 since negative shift counts are meaningless. */
1747 op1x
= convert_to_mode (op1_mode
, op1
, 1);
1750 if (GET_MODE (op0
) != VOIDmode
1751 && GET_MODE (op0
) != mode
)
1752 op0
= convert_to_mode (mode
, op0
, unsignedp
);
1754 /* Pass 1 for NO_QUEUE so we don't lose any increments
1755 if the libcall is cse'd or moved. */
1756 value
= emit_library_call_value (libfunc
,
1757 NULL_RTX
, LCT_CONST
, mode
, 2,
1758 op0
, mode
, op1x
, op1_mode
);
1760 insns
= get_insns ();
1763 bool trapv
= trapv_binoptab_p (binoptab
);
1764 target
= gen_reg_rtx (mode
);
1765 emit_libcall_block_1 (insns
, target
, value
,
1767 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
1768 mode
, op0
, op1
), trapv
);
1773 delete_insns_since (last
);
1775 /* It can't be done in this mode. Can we do it in a wider mode? */
1777 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
1778 || methods
== OPTAB_MUST_WIDEN
))
1780 /* Caller says, don't even try. */
1781 delete_insns_since (entry_last
);
1785 /* Compute the value of METHODS to pass to recursive calls.
1786 Don't allow widening to be tried recursively. */
1788 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
1790 /* Look for a wider mode of the same class for which it appears we can do
1793 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1795 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1797 if (find_widening_optab_handler (binoptab
, wider_mode
, mode
, 1)
1799 || (methods
== OPTAB_LIB
1800 && optab_libfunc (binoptab
, wider_mode
)))
1802 rtx xop0
= op0
, xop1
= op1
;
1805 /* For certain integer operations, we need not actually extend
1806 the narrow operands, as long as we will truncate
1807 the results to the same narrowness. */
1809 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1810 || binoptab
== xor_optab
1811 || binoptab
== add_optab
|| binoptab
== sub_optab
1812 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1813 && mclass
== MODE_INT
)
1816 xop0
= widen_operand (xop0
, wider_mode
, mode
,
1817 unsignedp
, no_extend
);
1819 /* The second operand of a shift must always be extended. */
1820 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1821 no_extend
&& binoptab
!= ashl_optab
);
1823 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1824 unsignedp
, methods
);
1827 if (mclass
!= MODE_INT
1828 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1831 target
= gen_reg_rtx (mode
);
1832 convert_move (target
, temp
, 0);
1836 return gen_lowpart (mode
, temp
);
1839 delete_insns_since (last
);
1844 delete_insns_since (entry_last
);
1848 /* Expand a binary operator which has both signed and unsigned forms.
1849 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1852 If we widen unsigned operands, we may use a signed wider operation instead
1853 of an unsigned wider operation, since the result would be the same. */
1856 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
1857 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
1858 enum optab_methods methods
)
1861 optab direct_optab
= unsignedp
? uoptab
: soptab
;
1864 /* Do it without widening, if possible. */
1865 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1866 unsignedp
, OPTAB_DIRECT
);
1867 if (temp
|| methods
== OPTAB_DIRECT
)
1870 /* Try widening to a signed int. Disable any direct use of any
1871 signed insn in the current mode. */
1872 save_enable
= swap_optab_enable (soptab
, mode
, false);
1874 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1875 unsignedp
, OPTAB_WIDEN
);
1877 /* For unsigned operands, try widening to an unsigned int. */
1878 if (!temp
&& unsignedp
)
1879 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1880 unsignedp
, OPTAB_WIDEN
);
1881 if (temp
|| methods
== OPTAB_WIDEN
)
1884 /* Use the right width libcall if that exists. */
1885 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1886 unsignedp
, OPTAB_LIB
);
1887 if (temp
|| methods
== OPTAB_LIB
)
1890 /* Must widen and use a libcall, use either signed or unsigned. */
1891 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1892 unsignedp
, methods
);
1893 if (!temp
&& unsignedp
)
1894 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1895 unsignedp
, methods
);
1898 /* Undo the fiddling above. */
1900 swap_optab_enable (soptab
, mode
, true);
1904 /* Generate code to perform an operation specified by UNOPPTAB
1905 on operand OP0, with two results to TARG0 and TARG1.
1906 We assume that the order of the operands for the instruction
1907 is TARG0, TARG1, OP0.
1909 Either TARG0 or TARG1 may be zero, but what that means is that
1910 the result is not actually wanted. We will generate it into
1911 a dummy pseudo-reg and discard it. They may not both be zero.
1913 Returns 1 if this operation can be performed; 0 if not. */
1916 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
1919 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
1920 enum mode_class mclass
;
1921 machine_mode wider_mode
;
1922 rtx_insn
*entry_last
= get_last_insn ();
1925 mclass
= GET_MODE_CLASS (mode
);
1928 targ0
= gen_reg_rtx (mode
);
1930 targ1
= gen_reg_rtx (mode
);
1932 /* Record where to go back to if we fail. */
1933 last
= get_last_insn ();
1935 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
1937 struct expand_operand ops
[3];
1938 enum insn_code icode
= optab_handler (unoptab
, mode
);
1940 create_fixed_operand (&ops
[0], targ0
);
1941 create_fixed_operand (&ops
[1], targ1
);
1942 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
1943 if (maybe_expand_insn (icode
, 3, ops
))
1947 /* It can't be done in this mode. Can we do it in a wider mode? */
1949 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1951 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1953 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
1955 rtx t0
= gen_reg_rtx (wider_mode
);
1956 rtx t1
= gen_reg_rtx (wider_mode
);
1957 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
1959 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
1961 convert_move (targ0
, t0
, unsignedp
);
1962 convert_move (targ1
, t1
, unsignedp
);
1966 delete_insns_since (last
);
1971 delete_insns_since (entry_last
);
1975 /* Generate code to perform an operation specified by BINOPTAB
1976 on operands OP0 and OP1, with two results to TARG1 and TARG2.
1977 We assume that the order of the operands for the instruction
1978 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
1979 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
1981 Either TARG0 or TARG1 may be zero, but what that means is that
1982 the result is not actually wanted. We will generate it into
1983 a dummy pseudo-reg and discard it. They may not both be zero.
1985 Returns 1 if this operation can be performed; 0 if not. */
1988 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
1991 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
1992 enum mode_class mclass
;
1993 machine_mode wider_mode
;
1994 rtx_insn
*entry_last
= get_last_insn ();
1997 mclass
= GET_MODE_CLASS (mode
);
2000 targ0
= gen_reg_rtx (mode
);
2002 targ1
= gen_reg_rtx (mode
);
2004 /* Record where to go back to if we fail. */
2005 last
= get_last_insn ();
2007 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2009 struct expand_operand ops
[4];
2010 enum insn_code icode
= optab_handler (binoptab
, mode
);
2011 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2012 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2013 rtx xop0
= op0
, xop1
= op1
;
2015 /* If we are optimizing, force expensive constants into a register. */
2016 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2017 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2019 create_fixed_operand (&ops
[0], targ0
);
2020 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2021 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
2022 create_fixed_operand (&ops
[3], targ1
);
2023 if (maybe_expand_insn (icode
, 4, ops
))
2025 delete_insns_since (last
);
2028 /* It can't be done in this mode. Can we do it in a wider mode? */
2030 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2032 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2034 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2036 rtx t0
= gen_reg_rtx (wider_mode
);
2037 rtx t1
= gen_reg_rtx (wider_mode
);
2038 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2039 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2041 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2044 convert_move (targ0
, t0
, unsignedp
);
2045 convert_move (targ1
, t1
, unsignedp
);
2049 delete_insns_since (last
);
2054 delete_insns_since (entry_last
);
2058 /* Expand the two-valued library call indicated by BINOPTAB, but
2059 preserve only one of the values. If TARG0 is non-NULL, the first
2060 value is placed into TARG0; otherwise the second value is placed
2061 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2062 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2063 This routine assumes that the value returned by the library call is
2064 as if the return value was of an integral mode twice as wide as the
2065 mode of OP0. Returns 1 if the call was successful. */
2068 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2069 rtx targ0
, rtx targ1
, enum rtx_code code
)
2072 machine_mode libval_mode
;
2077 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2078 gcc_assert (!targ0
!= !targ1
);
2080 mode
= GET_MODE (op0
);
2081 libfunc
= optab_libfunc (binoptab
, mode
);
2085 /* The value returned by the library function will have twice as
2086 many bits as the nominal MODE. */
2087 libval_mode
= smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode
));
2089 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2093 /* Get the part of VAL containing the value that we want. */
2094 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2095 targ0
? 0 : GET_MODE_SIZE (mode
));
2096 insns
= get_insns ();
2098 /* Move the into the desired location. */
2099 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2100 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2106 /* Wrapper around expand_unop which takes an rtx code to specify
2107 the operation to perform, not an optab pointer. All other
2108 arguments are the same. */
2110 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2111 rtx target
, int unsignedp
)
2113 optab unop
= code_to_optab (code
);
2116 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2122 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2124 A similar operation can be used for clrsb. UNOPTAB says which operation
2125 we are trying to expand. */
2127 widen_leading (machine_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2129 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2130 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2132 machine_mode wider_mode
;
2133 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2135 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2140 last
= get_last_insn ();
2143 target
= gen_reg_rtx (mode
);
2144 xop0
= widen_operand (op0
, wider_mode
, mode
,
2145 unoptab
!= clrsb_optab
, false);
2146 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2147 unoptab
!= clrsb_optab
);
2150 (wider_mode
, sub_optab
, temp
,
2151 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2152 - GET_MODE_PRECISION (mode
),
2154 target
, true, OPTAB_DIRECT
);
2156 delete_insns_since (last
);
2165 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2166 quantities, choosing which based on whether the high word is nonzero. */
2168 expand_doubleword_clz (machine_mode mode
, rtx op0
, rtx target
)
2170 rtx xop0
= force_reg (mode
, op0
);
2171 rtx subhi
= gen_highpart (word_mode
, xop0
);
2172 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2173 rtx_code_label
*hi0_label
= gen_label_rtx ();
2174 rtx_code_label
*after_label
= gen_label_rtx ();
2178 /* If we were not given a target, use a word_mode register, not a
2179 'mode' register. The result will fit, and nobody is expecting
2180 anything bigger (the return type of __builtin_clz* is int). */
2182 target
= gen_reg_rtx (word_mode
);
2184 /* In any case, write to a word_mode scratch in both branches of the
2185 conditional, so we can ensure there is a single move insn setting
2186 'target' to tag a REG_EQUAL note on. */
2187 result
= gen_reg_rtx (word_mode
);
2191 /* If the high word is not equal to zero,
2192 then clz of the full value is clz of the high word. */
2193 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2194 word_mode
, true, hi0_label
);
2196 temp
= expand_unop_direct (word_mode
, clz_optab
, subhi
, result
, true);
2201 convert_move (result
, temp
, true);
2203 emit_jump_insn (targetm
.gen_jump (after_label
));
2206 /* Else clz of the full value is clz of the low word plus the number
2207 of bits in the high word. */
2208 emit_label (hi0_label
);
2210 temp
= expand_unop_direct (word_mode
, clz_optab
, sublo
, 0, true);
2213 temp
= expand_binop (word_mode
, add_optab
, temp
,
2214 gen_int_mode (GET_MODE_BITSIZE (word_mode
), word_mode
),
2215 result
, true, OPTAB_DIRECT
);
2219 convert_move (result
, temp
, true);
2221 emit_label (after_label
);
2222 convert_move (target
, result
, true);
2227 add_equal_note (seq
, target
, CLZ
, xop0
, 0);
2236 /* Try calculating popcount of a double-word quantity as two popcount's of
2237 word-sized quantities and summing up the results. */
2239 expand_doubleword_popcount (machine_mode mode
, rtx op0
, rtx target
)
2246 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2247 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2249 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2250 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2258 /* If we were not given a target, use a word_mode register, not a
2259 'mode' register. The result will fit, and nobody is expecting
2260 anything bigger (the return type of __builtin_popcount* is int). */
2262 target
= gen_reg_rtx (word_mode
);
2264 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2269 add_equal_note (seq
, t
, POPCOUNT
, op0
, 0);
2277 (parity:narrow (low (x) ^ high (x))) */
2279 expand_doubleword_parity (machine_mode mode
, rtx op0
, rtx target
)
2281 rtx t
= expand_binop (word_mode
, xor_optab
,
2282 operand_subword_force (op0
, 0, mode
),
2283 operand_subword_force (op0
, 1, mode
),
2284 NULL_RTX
, 0, OPTAB_DIRECT
);
2285 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2291 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2293 widen_bswap (machine_mode mode
, rtx op0
, rtx target
)
2295 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2296 machine_mode wider_mode
;
2300 if (!CLASS_HAS_WIDER_MODES_P (mclass
))
2303 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2304 if (optab_handler (bswap_optab
, wider_mode
) != CODE_FOR_nothing
)
2309 last
= get_last_insn ();
2311 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2312 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2314 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2315 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2317 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2318 GET_MODE_BITSIZE (wider_mode
)
2319 - GET_MODE_BITSIZE (mode
),
2325 target
= gen_reg_rtx (mode
);
2326 emit_move_insn (target
, gen_lowpart (mode
, x
));
2329 delete_insns_since (last
);
2334 /* Try calculating bswap as two bswaps of two word-sized operands. */
2337 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2341 t1
= expand_unop (word_mode
, bswap_optab
,
2342 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2343 t0
= expand_unop (word_mode
, bswap_optab
,
2344 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2346 if (target
== 0 || !valid_multiword_target_p (target
))
2347 target
= gen_reg_rtx (mode
);
2349 emit_clobber (target
);
2350 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2351 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2356 /* Try calculating (parity x) as (and (popcount x) 1), where
2357 popcount can also be done in a wider mode. */
2359 expand_parity (machine_mode mode
, rtx op0
, rtx target
)
2361 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2362 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2364 machine_mode wider_mode
;
2365 FOR_EACH_MODE_FROM (wider_mode
, mode
)
2367 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2372 last
= get_last_insn ();
2374 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2375 target
= gen_reg_rtx (wider_mode
);
2377 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2378 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2381 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2382 target
, true, OPTAB_DIRECT
);
2386 if (mclass
!= MODE_INT
2387 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2388 return convert_to_mode (mode
, temp
, 0);
2390 return gen_lowpart (mode
, temp
);
2393 delete_insns_since (last
);
2400 /* Try calculating ctz(x) as K - clz(x & -x) ,
2401 where K is GET_MODE_PRECISION(mode) - 1.
2403 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2404 don't have to worry about what the hardware does in that case. (If
2405 the clz instruction produces the usual value at 0, which is K, the
2406 result of this code sequence will be -1; expand_ffs, below, relies
2407 on this. It might be nice to have it be K instead, for consistency
2408 with the (very few) processors that provide a ctz with a defined
2409 value, but that would take one more instruction, and it would be
2410 less convenient for expand_ffs anyway. */
2413 expand_ctz (machine_mode mode
, rtx op0
, rtx target
)
2418 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2423 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2425 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2426 true, OPTAB_DIRECT
);
2428 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2430 temp
= expand_binop (mode
, sub_optab
,
2431 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
2433 true, OPTAB_DIRECT
);
2443 add_equal_note (seq
, temp
, CTZ
, op0
, 0);
2449 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2450 else with the sequence used by expand_clz.
2452 The ffs builtin promises to return zero for a zero value and ctz/clz
2453 may have an undefined value in that case. If they do not give us a
2454 convenient value, we have to generate a test and branch. */
2456 expand_ffs (machine_mode mode
, rtx op0
, rtx target
)
2458 HOST_WIDE_INT val
= 0;
2459 bool defined_at_zero
= false;
2463 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
2467 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
2471 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
2473 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
2476 temp
= expand_ctz (mode
, op0
, 0);
2480 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
2482 defined_at_zero
= true;
2483 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
2489 if (defined_at_zero
&& val
== -1)
2490 /* No correction needed at zero. */;
2493 /* We don't try to do anything clever with the situation found
2494 on some processors (eg Alpha) where ctz(0:mode) ==
2495 bitsize(mode). If someone can think of a way to send N to -1
2496 and leave alone all values in the range 0..N-1 (where N is a
2497 power of two), cheaper than this test-and-branch, please add it.
2499 The test-and-branch is done after the operation itself, in case
2500 the operation sets condition codes that can be recycled for this.
2501 (This is true on i386, for instance.) */
2503 rtx_code_label
*nonzero_label
= gen_label_rtx ();
2504 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
2505 mode
, true, nonzero_label
);
2507 convert_move (temp
, GEN_INT (-1), false);
2508 emit_label (nonzero_label
);
2511 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2512 to produce a value in the range 0..bitsize. */
2513 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
2514 target
, false, OPTAB_DIRECT
);
2521 add_equal_note (seq
, temp
, FFS
, op0
, 0);
2530 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2531 conditions, VAL may already be a SUBREG against which we cannot generate
2532 a further SUBREG. In this case, we expect forcing the value into a
2533 register will work around the situation. */
2536 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
2540 ret
= lowpart_subreg (omode
, val
, imode
);
2543 val
= force_reg (imode
, val
);
2544 ret
= lowpart_subreg (omode
, val
, imode
);
2545 gcc_assert (ret
!= NULL
);
2550 /* Expand a floating point absolute value or negation operation via a
2551 logical operation on the sign bit. */
2554 expand_absneg_bit (enum rtx_code code
, scalar_float_mode mode
,
2555 rtx op0
, rtx target
)
2557 const struct real_format
*fmt
;
2558 int bitpos
, word
, nwords
, i
;
2563 /* The format has to have a simple sign bit. */
2564 fmt
= REAL_MODE_FORMAT (mode
);
2568 bitpos
= fmt
->signbit_rw
;
2572 /* Don't create negative zeros if the format doesn't support them. */
2573 if (code
== NEG
&& !fmt
->has_signed_zero
)
2576 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
2578 if (!int_mode_for_mode (mode
).exists (&imode
))
2587 if (FLOAT_WORDS_BIG_ENDIAN
)
2588 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
2590 word
= bitpos
/ BITS_PER_WORD
;
2591 bitpos
= bitpos
% BITS_PER_WORD
;
2592 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
2595 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
2601 || (nwords
> 1 && !valid_multiword_target_p (target
)))
2602 target
= gen_reg_rtx (mode
);
2608 for (i
= 0; i
< nwords
; ++i
)
2610 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
2611 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
2615 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2617 immed_wide_int_const (mask
, imode
),
2618 targ_piece
, 1, OPTAB_LIB_WIDEN
);
2619 if (temp
!= targ_piece
)
2620 emit_move_insn (targ_piece
, temp
);
2623 emit_move_insn (targ_piece
, op0_piece
);
2626 insns
= get_insns ();
2633 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2634 gen_lowpart (imode
, op0
),
2635 immed_wide_int_const (mask
, imode
),
2636 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
2637 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
2639 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
2640 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
2647 /* As expand_unop, but will fail rather than attempt the operation in a
2648 different mode or with a libcall. */
2650 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2653 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2655 struct expand_operand ops
[2];
2656 enum insn_code icode
= optab_handler (unoptab
, mode
);
2657 rtx_insn
*last
= get_last_insn ();
2660 create_output_operand (&ops
[0], target
, mode
);
2661 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2662 pat
= maybe_gen_insn (icode
, 2, ops
);
2665 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
2666 && ! add_equal_note (pat
, ops
[0].value
,
2667 optab_to_code (unoptab
),
2668 ops
[1].value
, NULL_RTX
))
2670 delete_insns_since (last
);
2671 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
2676 return ops
[0].value
;
2682 /* Generate code to perform an operation specified by UNOPTAB
2683 on operand OP0, with result having machine-mode MODE.
2685 UNSIGNEDP is for the case where we have to widen the operands
2686 to perform the operation. It says to use zero-extension.
2688 If TARGET is nonzero, the value
2689 is generated there, if it is convenient to do so.
2690 In all cases an rtx is returned for the locus of the value;
2691 this may or may not be TARGET. */
2694 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2697 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2698 machine_mode wider_mode
;
2699 scalar_float_mode float_mode
;
2703 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
2707 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2709 /* Widening (or narrowing) clz needs special treatment. */
2710 if (unoptab
== clz_optab
)
2712 temp
= widen_leading (mode
, op0
, target
, unoptab
);
2716 if (GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
2717 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2719 temp
= expand_doubleword_clz (mode
, op0
, target
);
2727 if (unoptab
== clrsb_optab
)
2729 temp
= widen_leading (mode
, op0
, target
, unoptab
);
2735 if (unoptab
== popcount_optab
2736 && GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
2737 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2738 && optimize_insn_for_speed_p ())
2740 temp
= expand_doubleword_popcount (mode
, op0
, target
);
2745 if (unoptab
== parity_optab
2746 && GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
2747 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2748 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
2749 && optimize_insn_for_speed_p ())
2751 temp
= expand_doubleword_parity (mode
, op0
, target
);
2756 /* Widening (or narrowing) bswap needs special treatment. */
2757 if (unoptab
== bswap_optab
)
2759 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2760 or ROTATERT. First try these directly; if this fails, then try the
2761 obvious pair of shifts with allowed widening, as this will probably
2762 be always more efficient than the other fallback methods. */
2768 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
2770 temp
= expand_binop (mode
, rotl_optab
, op0
, GEN_INT (8), target
,
2771 unsignedp
, OPTAB_DIRECT
);
2776 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
2778 temp
= expand_binop (mode
, rotr_optab
, op0
, GEN_INT (8), target
,
2779 unsignedp
, OPTAB_DIRECT
);
2784 last
= get_last_insn ();
2786 temp1
= expand_binop (mode
, ashl_optab
, op0
, GEN_INT (8), NULL_RTX
,
2787 unsignedp
, OPTAB_WIDEN
);
2788 temp2
= expand_binop (mode
, lshr_optab
, op0
, GEN_INT (8), NULL_RTX
,
2789 unsignedp
, OPTAB_WIDEN
);
2792 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
2793 unsignedp
, OPTAB_WIDEN
);
2798 delete_insns_since (last
);
2801 temp
= widen_bswap (mode
, op0
, target
);
2805 if (GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
2806 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2808 temp
= expand_doubleword_bswap (mode
, op0
, target
);
2816 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2817 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2819 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2822 rtx_insn
*last
= get_last_insn ();
2824 /* For certain operations, we need not actually extend
2825 the narrow operand, as long as we will truncate the
2826 results to the same narrowness. */
2828 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
2829 (unoptab
== neg_optab
2830 || unoptab
== one_cmpl_optab
)
2831 && mclass
== MODE_INT
);
2833 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2838 if (mclass
!= MODE_INT
2839 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2842 target
= gen_reg_rtx (mode
);
2843 convert_move (target
, temp
, 0);
2847 return gen_lowpart (mode
, temp
);
2850 delete_insns_since (last
);
2854 /* These can be done a word at a time. */
2855 if (unoptab
== one_cmpl_optab
2856 && mclass
== MODE_INT
2857 && GET_MODE_SIZE (mode
) > UNITS_PER_WORD
2858 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2863 if (target
== 0 || target
== op0
|| !valid_multiword_target_p (target
))
2864 target
= gen_reg_rtx (mode
);
2868 /* Do the actual arithmetic. */
2869 for (i
= 0; i
< GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
; i
++)
2871 rtx target_piece
= operand_subword (target
, i
, 1, mode
);
2872 rtx x
= expand_unop (word_mode
, unoptab
,
2873 operand_subword_force (op0
, i
, mode
),
2874 target_piece
, unsignedp
);
2876 if (target_piece
!= x
)
2877 emit_move_insn (target_piece
, x
);
2880 insns
= get_insns ();
2887 if (optab_to_code (unoptab
) == NEG
)
2889 /* Try negating floating point values by flipping the sign bit. */
2890 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
2892 temp
= expand_absneg_bit (NEG
, float_mode
, op0
, target
);
2897 /* If there is no negation pattern, and we have no negative zero,
2898 try subtracting from zero. */
2899 if (!HONOR_SIGNED_ZEROS (mode
))
2901 temp
= expand_binop (mode
, (unoptab
== negv_optab
2902 ? subv_optab
: sub_optab
),
2903 CONST0_RTX (mode
), op0
, target
,
2904 unsignedp
, OPTAB_DIRECT
);
2910 /* Try calculating parity (x) as popcount (x) % 2. */
2911 if (unoptab
== parity_optab
)
2913 temp
= expand_parity (mode
, op0
, target
);
2918 /* Try implementing ffs (x) in terms of clz (x). */
2919 if (unoptab
== ffs_optab
)
2921 temp
= expand_ffs (mode
, op0
, target
);
2926 /* Try implementing ctz (x) in terms of clz (x). */
2927 if (unoptab
== ctz_optab
)
2929 temp
= expand_ctz (mode
, op0
, target
);
2935 /* Now try a library call in this mode. */
2936 libfunc
= optab_libfunc (unoptab
, mode
);
2942 machine_mode outmode
= mode
;
2944 /* All of these functions return small values. Thus we choose to
2945 have them return something that isn't a double-word. */
2946 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
2947 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
2948 || unoptab
== parity_optab
)
2950 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
2951 optab_libfunc (unoptab
, mode
)));
2955 /* Pass 1 for NO_QUEUE so we don't lose any increments
2956 if the libcall is cse'd or moved. */
2957 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
2959 insns
= get_insns ();
2962 target
= gen_reg_rtx (outmode
);
2963 bool trapv
= trapv_unoptab_p (unoptab
);
2965 eq_value
= NULL_RTX
;
2968 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
2969 if (GET_MODE_SIZE (outmode
) < GET_MODE_SIZE (mode
))
2970 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
2971 else if (GET_MODE_SIZE (outmode
) > GET_MODE_SIZE (mode
))
2972 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
2973 outmode
, eq_value
, mode
);
2975 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
2980 /* It can't be done in this mode. Can we do it in a wider mode? */
2982 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2984 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2986 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
2987 || optab_libfunc (unoptab
, wider_mode
))
2990 rtx_insn
*last
= get_last_insn ();
2992 /* For certain operations, we need not actually extend
2993 the narrow operand, as long as we will truncate the
2994 results to the same narrowness. */
2995 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
2996 (unoptab
== neg_optab
2997 || unoptab
== one_cmpl_optab
2998 || unoptab
== bswap_optab
)
2999 && mclass
== MODE_INT
);
3001 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3004 /* If we are generating clz using wider mode, adjust the
3005 result. Similarly for clrsb. */
3006 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3009 (wider_mode
, sub_optab
, temp
,
3010 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
3011 - GET_MODE_PRECISION (mode
),
3013 target
, true, OPTAB_DIRECT
);
3015 /* Likewise for bswap. */
3016 if (unoptab
== bswap_optab
&& temp
!= 0)
3018 gcc_assert (GET_MODE_PRECISION (wider_mode
)
3019 == GET_MODE_BITSIZE (wider_mode
)
3020 && GET_MODE_PRECISION (mode
)
3021 == GET_MODE_BITSIZE (mode
));
3023 temp
= expand_shift (RSHIFT_EXPR
, wider_mode
, temp
,
3024 GET_MODE_BITSIZE (wider_mode
)
3025 - GET_MODE_BITSIZE (mode
),
3031 if (mclass
!= MODE_INT
)
3034 target
= gen_reg_rtx (mode
);
3035 convert_move (target
, temp
, 0);
3039 return gen_lowpart (mode
, temp
);
3042 delete_insns_since (last
);
3047 /* One final attempt at implementing negation via subtraction,
3048 this time allowing widening of the operand. */
3049 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3052 temp
= expand_binop (mode
,
3053 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3054 CONST0_RTX (mode
), op0
,
3055 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3063 /* Emit code to compute the absolute value of OP0, with result to
3064 TARGET if convenient. (TARGET may be 0.) The return value says
3065 where the result actually is to be found.
3067 MODE is the mode of the operand; the mode of the result is
3068 different but can be deduced from MODE.
3073 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3074 int result_unsignedp
)
3078 if (GET_MODE_CLASS (mode
) != MODE_INT
3080 result_unsignedp
= 1;
3082 /* First try to do it with a special abs instruction. */
3083 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3088 /* For floating point modes, try clearing the sign bit. */
3089 scalar_float_mode float_mode
;
3090 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3092 temp
= expand_absneg_bit (ABS
, float_mode
, op0
, target
);
3097 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3098 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3099 && !HONOR_SIGNED_ZEROS (mode
))
3101 rtx_insn
*last
= get_last_insn ();
3103 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3106 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3112 delete_insns_since (last
);
3115 /* If this machine has expensive jumps, we can do integer absolute
3116 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3117 where W is the width of MODE. */
3119 if (GET_MODE_CLASS (mode
) == MODE_INT
3120 && BRANCH_COST (optimize_insn_for_speed_p (),
3123 rtx extended
= expand_shift (RSHIFT_EXPR
, mode
, op0
,
3124 GET_MODE_PRECISION (mode
) - 1,
3127 temp
= expand_binop (mode
, xor_optab
, extended
, op0
, target
, 0,
3130 temp
= expand_binop (mode
, result_unsignedp
? sub_optab
: subv_optab
,
3131 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3141 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3142 int result_unsignedp
, int safe
)
3145 rtx_code_label
*op1
;
3147 if (GET_MODE_CLASS (mode
) != MODE_INT
3149 result_unsignedp
= 1;
3151 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3155 /* If that does not win, use conditional jump and negate. */
3157 /* It is safe to use the target if it is the same
3158 as the source if this is also a pseudo register */
3159 if (op0
== target
&& REG_P (op0
)
3160 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3163 op1
= gen_label_rtx ();
3164 if (target
== 0 || ! safe
3165 || GET_MODE (target
) != mode
3166 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3168 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3169 target
= gen_reg_rtx (mode
);
3171 emit_move_insn (target
, op0
);
3174 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3175 NULL_RTX
, NULL
, op1
,
3176 profile_probability::uninitialized ());
3178 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3181 emit_move_insn (target
, op0
);
3187 /* Emit code to compute the one's complement absolute value of OP0
3188 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3189 (TARGET may be NULL_RTX.) The return value says where the result
3190 actually is to be found.
3192 MODE is the mode of the operand; the mode of the result is
3193 different but can be deduced from MODE. */
3196 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3200 /* Not applicable for floating point modes. */
3201 if (FLOAT_MODE_P (mode
))
3204 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3205 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3207 rtx_insn
*last
= get_last_insn ();
3209 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3211 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3217 delete_insns_since (last
);
3220 /* If this machine has expensive jumps, we can do one's complement
3221 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3223 if (GET_MODE_CLASS (mode
) == MODE_INT
3224 && BRANCH_COST (optimize_insn_for_speed_p (),
3227 rtx extended
= expand_shift (RSHIFT_EXPR
, mode
, op0
,
3228 GET_MODE_PRECISION (mode
) - 1,
3231 temp
= expand_binop (mode
, xor_optab
, extended
, op0
, target
, 0,
3241 /* A subroutine of expand_copysign, perform the copysign operation using the
3242 abs and neg primitives advertised to exist on the target. The assumption
3243 is that we have a split register file, and leaving op0 in fp registers,
3244 and not playing with subregs so much, will help the register allocator. */
3247 expand_copysign_absneg (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3248 int bitpos
, bool op0_is_abs
)
3251 enum insn_code icode
;
3253 rtx_code_label
*label
;
3258 /* Check if the back end provides an insn that handles signbit for the
3260 icode
= optab_handler (signbit_optab
, mode
);
3261 if (icode
!= CODE_FOR_nothing
)
3263 imode
= insn_data
[(int) icode
].operand
[0].mode
;
3264 sign
= gen_reg_rtx (imode
);
3265 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3269 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3271 if (!int_mode_for_mode (mode
).exists (&imode
))
3273 op1
= gen_lowpart (imode
, op1
);
3280 if (FLOAT_WORDS_BIG_ENDIAN
)
3281 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3283 word
= bitpos
/ BITS_PER_WORD
;
3284 bitpos
= bitpos
% BITS_PER_WORD
;
3285 op1
= operand_subword_force (op1
, word
, mode
);
3288 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3289 sign
= expand_binop (imode
, and_optab
, op1
,
3290 immed_wide_int_const (mask
, imode
),
3291 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3296 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3303 if (target
== NULL_RTX
)
3304 target
= copy_to_reg (op0
);
3306 emit_move_insn (target
, op0
);
3309 label
= gen_label_rtx ();
3310 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3312 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3313 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3315 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3317 emit_move_insn (target
, op0
);
3325 /* A subroutine of expand_copysign, perform the entire copysign operation
3326 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3327 is true if op0 is known to have its sign bit clear. */
3330 expand_copysign_bit (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3331 int bitpos
, bool op0_is_abs
)
3333 scalar_int_mode imode
;
3334 int word
, nwords
, i
;
3338 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3340 if (!int_mode_for_mode (mode
).exists (&imode
))
3349 if (FLOAT_WORDS_BIG_ENDIAN
)
3350 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3352 word
= bitpos
/ BITS_PER_WORD
;
3353 bitpos
= bitpos
% BITS_PER_WORD
;
3354 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3357 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3362 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3363 target
= gen_reg_rtx (mode
);
3369 for (i
= 0; i
< nwords
; ++i
)
3371 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3372 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3378 = expand_binop (imode
, and_optab
, op0_piece
,
3379 immed_wide_int_const (~mask
, imode
),
3380 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3381 op1
= expand_binop (imode
, and_optab
,
3382 operand_subword_force (op1
, i
, mode
),
3383 immed_wide_int_const (mask
, imode
),
3384 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3386 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
3387 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3388 if (temp
!= targ_piece
)
3389 emit_move_insn (targ_piece
, temp
);
3392 emit_move_insn (targ_piece
, op0_piece
);
3395 insns
= get_insns ();
3402 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
3403 immed_wide_int_const (mask
, imode
),
3404 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3406 op0
= gen_lowpart (imode
, op0
);
3408 op0
= expand_binop (imode
, and_optab
, op0
,
3409 immed_wide_int_const (~mask
, imode
),
3410 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3412 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
3413 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3414 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3420 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3421 scalar floating point mode. Return NULL if we do not know how to
3422 expand the operation inline. */
3425 expand_copysign (rtx op0
, rtx op1
, rtx target
)
3427 scalar_float_mode mode
;
3428 const struct real_format
*fmt
;
3432 mode
= as_a
<scalar_float_mode
> (GET_MODE (op0
));
3433 gcc_assert (GET_MODE (op1
) == mode
);
3435 /* First try to do it with a special instruction. */
3436 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
3437 target
, 0, OPTAB_DIRECT
);
3441 fmt
= REAL_MODE_FORMAT (mode
);
3442 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
3446 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3448 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
3449 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
3453 if (fmt
->signbit_ro
>= 0
3454 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
3455 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
3456 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
3458 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
3459 fmt
->signbit_ro
, op0_is_abs
);
3464 if (fmt
->signbit_rw
< 0)
3466 return expand_copysign_bit (mode
, op0
, op1
, target
,
3467 fmt
->signbit_rw
, op0_is_abs
);
3470 /* Generate an instruction whose insn-code is INSN_CODE,
3471 with two operands: an output TARGET and an input OP0.
3472 TARGET *must* be nonzero, and the output is always stored there.
3473 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3474 the value that is stored into TARGET.
3476 Return false if expansion failed. */
3479 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
3482 struct expand_operand ops
[2];
3485 create_output_operand (&ops
[0], target
, GET_MODE (target
));
3486 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
3487 pat
= maybe_gen_insn (icode
, 2, ops
);
3491 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3493 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
);
3497 if (ops
[0].value
!= target
)
3498 emit_move_insn (target
, ops
[0].value
);
3501 /* Generate an instruction whose insn-code is INSN_CODE,
3502 with two operands: an output TARGET and an input OP0.
3503 TARGET *must* be nonzero, and the output is always stored there.
3504 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3505 the value that is stored into TARGET. */
3508 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
3510 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
3514 struct no_conflict_data
3517 rtx_insn
*first
, *insn
;
3521 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3522 the currently examined clobber / store has to stay in the list of
3523 insns that constitute the actual libcall block. */
3525 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
3527 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
3529 /* If this inns directly contributes to setting the target, it must stay. */
3530 if (reg_overlap_mentioned_p (p
->target
, dest
))
3531 p
->must_stay
= true;
3532 /* If we haven't committed to keeping any other insns in the list yet,
3533 there is nothing more to check. */
3534 else if (p
->insn
== p
->first
)
3536 /* If this insn sets / clobbers a register that feeds one of the insns
3537 already in the list, this insn has to stay too. */
3538 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
3539 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
3540 || reg_used_between_p (dest
, p
->first
, p
->insn
)
3541 /* Likewise if this insn depends on a register set by a previous
3542 insn in the list, or if it sets a result (presumably a hard
3543 register) that is set or clobbered by a previous insn.
3544 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3545 SET_DEST perform the former check on the address, and the latter
3546 check on the MEM. */
3547 || (GET_CODE (set
) == SET
3548 && (modified_in_p (SET_SRC (set
), p
->first
)
3549 || modified_in_p (SET_DEST (set
), p
->first
)
3550 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
3551 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
3552 p
->must_stay
= true;
3556 /* Emit code to make a call to a constant function or a library call.
3558 INSNS is a list containing all insns emitted in the call.
3559 These insns leave the result in RESULT. Our block is to copy RESULT
3560 to TARGET, which is logically equivalent to EQUIV.
3562 We first emit any insns that set a pseudo on the assumption that these are
3563 loading constants into registers; doing so allows them to be safely cse'ed
3564 between blocks. Then we emit all the other insns in the block, followed by
3565 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3566 note with an operand of EQUIV. */
3569 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
3570 bool equiv_may_trap
)
3572 rtx final_dest
= target
;
3573 rtx_insn
*next
, *last
, *insn
;
3575 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3576 into a MEM later. Protect the libcall block from this change. */
3577 if (! REG_P (target
) || REG_USERVAR_P (target
))
3578 target
= gen_reg_rtx (GET_MODE (target
));
3580 /* If we're using non-call exceptions, a libcall corresponding to an
3581 operation that may trap may also trap. */
3582 /* ??? See the comment in front of make_reg_eh_region_note. */
3583 if (cfun
->can_throw_non_call_exceptions
3584 && (equiv_may_trap
|| may_trap_p (equiv
)))
3586 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3589 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
3592 int lp_nr
= INTVAL (XEXP (note
, 0));
3593 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
3594 remove_note (insn
, note
);
3600 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3601 reg note to indicate that this call cannot throw or execute a nonlocal
3602 goto (unless there is already a REG_EH_REGION note, in which case
3604 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3606 make_reg_eh_region_note_nothrow_nononlocal (insn
);
3609 /* First emit all insns that set pseudos. Remove them from the list as
3610 we go. Avoid insns that set pseudos which were referenced in previous
3611 insns. These can be generated by move_by_pieces, for example,
3612 to update an address. Similarly, avoid insns that reference things
3613 set in previous insns. */
3615 for (insn
= insns
; insn
; insn
= next
)
3617 rtx set
= single_set (insn
);
3619 next
= NEXT_INSN (insn
);
3621 if (set
!= 0 && REG_P (SET_DEST (set
))
3622 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
3624 struct no_conflict_data data
;
3626 data
.target
= const0_rtx
;
3630 note_stores (PATTERN (insn
), no_conflict_move_test
, &data
);
3631 if (! data
.must_stay
)
3633 if (PREV_INSN (insn
))
3634 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
3639 SET_PREV_INSN (next
) = PREV_INSN (insn
);
3645 /* Some ports use a loop to copy large arguments onto the stack.
3646 Don't move anything outside such a loop. */
3651 /* Write the remaining insns followed by the final copy. */
3652 for (insn
= insns
; insn
; insn
= next
)
3654 next
= NEXT_INSN (insn
);
3659 last
= emit_move_insn (target
, result
);
3661 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
3663 if (final_dest
!= target
)
3664 emit_move_insn (final_dest
, target
);
3668 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
3670 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
3673 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3674 PURPOSE describes how this comparison will be used. CODE is the rtx
3675 comparison code we will be using.
3677 ??? Actually, CODE is slightly weaker than that. A target is still
3678 required to implement all of the normal bcc operations, but not
3679 required to implement all (or any) of the unordered bcc operations. */
3682 can_compare_p (enum rtx_code code
, machine_mode mode
,
3683 enum can_compare_purpose purpose
)
3686 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
3689 enum insn_code icode
;
3691 if (purpose
== ccp_jump
3692 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
3693 && insn_operand_matches (icode
, 0, test
))
3695 if (purpose
== ccp_store_flag
3696 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
3697 && insn_operand_matches (icode
, 1, test
))
3699 if (purpose
== ccp_cmov
3700 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
3703 mode
= GET_MODE_WIDER_MODE (mode
).else_void ();
3704 PUT_MODE (test
, mode
);
3706 while (mode
!= VOIDmode
);
3711 /* This function is called when we are going to emit a compare instruction that
3712 compares the values found in X and Y, using the rtl operator COMPARISON.
3714 If they have mode BLKmode, then SIZE specifies the size of both operands.
3716 UNSIGNEDP nonzero says that the operands are unsigned;
3717 this matters if they need to be widened (as given by METHODS).
3719 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3720 if we failed to produce one.
3722 *PMODE is the mode of the inputs (in case they are const_int).
3724 This function performs all the setup necessary so that the caller only has
3725 to emit a single comparison insn. This setup can involve doing a BLKmode
3726 comparison or emitting a library call to perform the comparison if no insn
3727 is available to handle it.
3728 The values which are passed in through pointers can be modified; the caller
3729 should perform the comparison on the modified values. Constant
3730 comparisons must have already been folded. */
3733 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
3734 int unsignedp
, enum optab_methods methods
,
3735 rtx
*ptest
, machine_mode
*pmode
)
3737 machine_mode mode
= *pmode
;
3739 machine_mode cmp_mode
;
3740 enum mode_class mclass
;
3742 /* The other methods are not needed. */
3743 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
3744 || methods
== OPTAB_LIB_WIDEN
);
3746 /* If we are optimizing, force expensive constants into a register. */
3747 if (CONSTANT_P (x
) && optimize
3748 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
3749 > COSTS_N_INSNS (1)))
3750 x
= force_reg (mode
, x
);
3752 if (CONSTANT_P (y
) && optimize
3753 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
3754 > COSTS_N_INSNS (1)))
3755 y
= force_reg (mode
, y
);
3758 /* Make sure if we have a canonical comparison. The RTL
3759 documentation states that canonical comparisons are required only
3760 for targets which have cc0. */
3761 gcc_assert (!CONSTANT_P (x
) || CONSTANT_P (y
));
3764 /* Don't let both operands fail to indicate the mode. */
3765 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
3766 x
= force_reg (mode
, x
);
3767 if (mode
== VOIDmode
)
3768 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
3770 /* Handle all BLKmode compares. */
3772 if (mode
== BLKmode
)
3774 machine_mode result_mode
;
3775 enum insn_code cmp_code
;
3778 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
3782 /* Try to use a memory block compare insn - either cmpstr
3783 or cmpmem will do. */
3784 FOR_EACH_MODE_IN_CLASS (cmp_mode
, MODE_INT
)
3786 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
3787 if (cmp_code
== CODE_FOR_nothing
)
3788 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
3789 if (cmp_code
== CODE_FOR_nothing
)
3790 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
3791 if (cmp_code
== CODE_FOR_nothing
)
3794 /* Must make sure the size fits the insn's mode. */
3795 if ((CONST_INT_P (size
)
3796 && INTVAL (size
) >= (1 << GET_MODE_BITSIZE (cmp_mode
)))
3797 || (GET_MODE_BITSIZE (GET_MODE (size
))
3798 > GET_MODE_BITSIZE (cmp_mode
)))
3801 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
3802 result
= gen_reg_rtx (result_mode
);
3803 size
= convert_to_mode (cmp_mode
, size
, 1);
3804 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
3806 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
3807 *pmode
= result_mode
;
3811 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
3814 /* Otherwise call a library function. */
3815 result
= emit_block_comp_via_libcall (XEXP (x
, 0), XEXP (y
, 0), size
);
3819 mode
= TYPE_MODE (integer_type_node
);
3820 methods
= OPTAB_LIB_WIDEN
;
3824 /* Don't allow operands to the compare to trap, as that can put the
3825 compare and branch in different basic blocks. */
3826 if (cfun
->can_throw_non_call_exceptions
)
3829 x
= copy_to_reg (x
);
3831 y
= copy_to_reg (y
);
3834 if (GET_MODE_CLASS (mode
) == MODE_CC
)
3836 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
3837 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3838 gcc_assert (icode
!= CODE_FOR_nothing
3839 && insn_operand_matches (icode
, 0, test
));
3844 mclass
= GET_MODE_CLASS (mode
);
3845 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3846 FOR_EACH_MODE_FROM (cmp_mode
, mode
)
3848 enum insn_code icode
;
3849 icode
= optab_handler (cbranch_optab
, cmp_mode
);
3850 if (icode
!= CODE_FOR_nothing
3851 && insn_operand_matches (icode
, 0, test
))
3853 rtx_insn
*last
= get_last_insn ();
3854 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
3855 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
3857 && insn_operand_matches (icode
, 1, op0
)
3858 && insn_operand_matches (icode
, 2, op1
))
3860 XEXP (test
, 0) = op0
;
3861 XEXP (test
, 1) = op1
;
3866 delete_insns_since (last
);
3869 if (methods
== OPTAB_DIRECT
|| !CLASS_HAS_WIDER_MODES_P (mclass
))
3873 if (methods
!= OPTAB_LIB_WIDEN
)
3876 if (!SCALAR_FLOAT_MODE_P (mode
))
3879 machine_mode ret_mode
;
3881 /* Handle a libcall just for the mode we are using. */
3882 libfunc
= optab_libfunc (cmp_optab
, mode
);
3883 gcc_assert (libfunc
);
3885 /* If we want unsigned, and this mode has a distinct unsigned
3886 comparison routine, use that. */
3889 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
3894 ret_mode
= targetm
.libgcc_cmp_return_mode ();
3895 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
3896 ret_mode
, 2, x
, mode
, y
, mode
);
3898 /* There are two kinds of comparison routines. Biased routines
3899 return 0/1/2, and unbiased routines return -1/0/1. Other parts
3900 of gcc expect that the comparison operation is equivalent
3901 to the modified comparison. For signed comparisons compare the
3902 result against 1 in the biased case, and zero in the unbiased
3903 case. For unsigned comparisons always compare against 1 after
3904 biasing the unbiased result by adding 1. This gives us a way to
3906 The comparisons in the fixed-point helper library are always
3911 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
3914 x
= plus_constant (ret_mode
, result
, 1);
3920 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
3924 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
3932 /* Before emitting an insn with code ICODE, make sure that X, which is going
3933 to be used for operand OPNUM of the insn, is converted from mode MODE to
3934 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
3935 that it is accepted by the operand predicate. Return the new value. */
3938 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
3939 machine_mode wider_mode
, int unsignedp
)
3941 if (mode
!= wider_mode
)
3942 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
3944 if (!insn_operand_matches (icode
, opnum
, x
))
3946 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
3947 if (reload_completed
)
3949 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
3951 x
= copy_to_mode_reg (op_mode
, x
);
3957 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
3958 we can do the branch. */
3961 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
3962 profile_probability prob
)
3964 machine_mode optab_mode
;
3965 enum mode_class mclass
;
3966 enum insn_code icode
;
3969 mclass
= GET_MODE_CLASS (mode
);
3970 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
3971 icode
= optab_handler (cbranch_optab
, optab_mode
);
3973 gcc_assert (icode
!= CODE_FOR_nothing
);
3974 gcc_assert (insn_operand_matches (icode
, 0, test
));
3975 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
3976 XEXP (test
, 1), label
));
3977 if (prob
.initialized_p ()
3978 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
3981 && any_condjump_p (insn
)
3982 && !find_reg_note (insn
, REG_BR_PROB
, 0))
3983 add_reg_br_prob_note (insn
, prob
);
3986 /* Generate code to compare X with Y so that the condition codes are
3987 set and to jump to LABEL if the condition is true. If X is a
3988 constant and Y is not a constant, then the comparison is swapped to
3989 ensure that the comparison RTL has the canonical form.
3991 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
3992 need to be widened. UNSIGNEDP is also used to select the proper
3993 branch condition code.
3995 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
3997 MODE is the mode of the inputs (in case they are const_int).
3999 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4000 It will be potentially converted into an unsigned variant based on
4001 UNSIGNEDP to select a proper jump instruction.
4003 PROB is the probability of jumping to LABEL. */
4006 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4007 machine_mode mode
, int unsignedp
, rtx label
,
4008 profile_probability prob
)
4010 rtx op0
= x
, op1
= y
;
4013 /* Swap operands and condition to ensure canonical RTL. */
4014 if (swap_commutative_operands_p (x
, y
)
4015 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4018 comparison
= swap_condition (comparison
);
4021 /* If OP0 is still a constant, then both X and Y must be constants
4022 or the opposite comparison is not supported. Force X into a register
4023 to create canonical RTL. */
4024 if (CONSTANT_P (op0
))
4025 op0
= force_reg (mode
, op0
);
4028 comparison
= unsigned_condition (comparison
);
4030 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4032 emit_cmp_and_jump_insn_1 (test
, mode
, label
, prob
);
4036 /* Emit a library call comparison between floating point X and Y.
4037 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4040 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4041 rtx
*ptest
, machine_mode
*pmode
)
4043 enum rtx_code swapped
= swap_condition (comparison
);
4044 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4045 machine_mode orig_mode
= GET_MODE (x
);
4046 machine_mode mode
, cmp_mode
;
4047 rtx true_rtx
, false_rtx
;
4048 rtx value
, target
, equiv
;
4051 bool reversed_p
= false;
4052 cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4054 FOR_EACH_MODE_FROM (mode
, orig_mode
)
4056 if (code_to_optab (comparison
)
4057 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4060 if (code_to_optab (swapped
)
4061 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4064 comparison
= swapped
;
4068 if (code_to_optab (reversed
)
4069 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4071 comparison
= reversed
;
4077 gcc_assert (mode
!= VOIDmode
);
4079 if (mode
!= orig_mode
)
4081 x
= convert_to_mode (mode
, x
, 0);
4082 y
= convert_to_mode (mode
, y
, 0);
4085 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4086 the RTL. The allows the RTL optimizers to delete the libcall if the
4087 condition can be determined at compile-time. */
4088 if (comparison
== UNORDERED
4089 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4091 true_rtx
= const_true_rtx
;
4092 false_rtx
= const0_rtx
;
4099 true_rtx
= const0_rtx
;
4100 false_rtx
= const_true_rtx
;
4104 true_rtx
= const_true_rtx
;
4105 false_rtx
= const0_rtx
;
4109 true_rtx
= const1_rtx
;
4110 false_rtx
= const0_rtx
;
4114 true_rtx
= const0_rtx
;
4115 false_rtx
= constm1_rtx
;
4119 true_rtx
= constm1_rtx
;
4120 false_rtx
= const0_rtx
;
4124 true_rtx
= const0_rtx
;
4125 false_rtx
= const1_rtx
;
4133 if (comparison
== UNORDERED
)
4135 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4136 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4137 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4138 temp
, const_true_rtx
, equiv
);
4142 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4143 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4144 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4145 equiv
, true_rtx
, false_rtx
);
4149 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4150 cmp_mode
, 2, x
, mode
, y
, mode
);
4151 insns
= get_insns ();
4154 target
= gen_reg_rtx (cmp_mode
);
4155 emit_libcall_block (insns
, target
, value
, equiv
);
4157 if (comparison
== UNORDERED
4158 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
4160 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
4162 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
4167 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4170 emit_indirect_jump (rtx loc
)
4172 if (!targetm
.have_indirect_jump ())
4173 sorry ("indirect jumps are not available on this target");
4176 struct expand_operand ops
[1];
4177 create_address_operand (&ops
[0], loc
);
4178 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
4184 /* Emit a conditional move instruction if the machine supports one for that
4185 condition and machine mode.
4187 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4188 the mode to use should they be constants. If it is VOIDmode, they cannot
4191 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4192 should be stored there. MODE is the mode to use should they be constants.
4193 If it is VOIDmode, they cannot both be constants.
4195 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4196 is not supported. */
4199 emit_conditional_move (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4200 machine_mode cmode
, rtx op2
, rtx op3
,
4201 machine_mode mode
, int unsignedp
)
4205 enum insn_code icode
;
4206 enum rtx_code reversed
;
4208 /* If the two source operands are identical, that's just a move. */
4210 if (rtx_equal_p (op2
, op3
))
4213 target
= gen_reg_rtx (mode
);
4215 emit_move_insn (target
, op3
);
4219 /* If one operand is constant, make it the second one. Only do this
4220 if the other operand is not constant as well. */
4222 if (swap_commutative_operands_p (op0
, op1
))
4224 std::swap (op0
, op1
);
4225 code
= swap_condition (code
);
4228 /* get_condition will prefer to generate LT and GT even if the old
4229 comparison was against zero, so undo that canonicalization here since
4230 comparisons against zero are cheaper. */
4231 if (code
== LT
&& op1
== const1_rtx
)
4232 code
= LE
, op1
= const0_rtx
;
4233 else if (code
== GT
&& op1
== constm1_rtx
)
4234 code
= GE
, op1
= const0_rtx
;
4236 if (cmode
== VOIDmode
)
4237 cmode
= GET_MODE (op0
);
4239 enum rtx_code orig_code
= code
;
4240 bool swapped
= false;
4241 if (swap_commutative_operands_p (op2
, op3
)
4242 && ((reversed
= reversed_comparison_code_parts (code
, op0
, op1
, NULL
))
4245 std::swap (op2
, op3
);
4250 if (mode
== VOIDmode
)
4251 mode
= GET_MODE (op2
);
4253 icode
= direct_optab_handler (movcc_optab
, mode
);
4255 if (icode
== CODE_FOR_nothing
)
4259 target
= gen_reg_rtx (mode
);
4261 for (int pass
= 0; ; pass
++)
4263 code
= unsignedp
? unsigned_condition (code
) : code
;
4264 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4266 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4267 punt and let the caller figure out how best to deal with this
4269 if (COMPARISON_P (comparison
))
4271 saved_pending_stack_adjust save
;
4272 save_pending_stack_adjust (&save
);
4273 last
= get_last_insn ();
4274 do_pending_stack_adjust ();
4275 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4276 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
4277 OPTAB_WIDEN
, &comparison
, &cmode
);
4280 struct expand_operand ops
[4];
4282 create_output_operand (&ops
[0], target
, mode
);
4283 create_fixed_operand (&ops
[1], comparison
);
4284 create_input_operand (&ops
[2], op2
, mode
);
4285 create_input_operand (&ops
[3], op3
, mode
);
4286 if (maybe_expand_insn (icode
, 4, ops
))
4288 if (ops
[0].value
!= target
)
4289 convert_move (target
, ops
[0].value
, false);
4293 delete_insns_since (last
);
4294 restore_pending_stack_adjust (&save
);
4300 /* If the preferred op2/op3 order is not usable, retry with other
4301 operand order, perhaps it will expand successfully. */
4304 else if ((reversed
= reversed_comparison_code_parts (orig_code
, op0
, op1
,
4310 std::swap (op2
, op3
);
4315 /* Emit a conditional negate or bitwise complement using the
4316 negcc or notcc optabs if available. Return NULL_RTX if such operations
4317 are not available. Otherwise return the RTX holding the result.
4318 TARGET is the desired destination of the result. COMP is the comparison
4319 on which to negate. If COND is true move into TARGET the negation
4320 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4321 CODE is either NEG or NOT. MODE is the machine mode in which the
4322 operation is performed. */
4325 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
4326 machine_mode mode
, rtx cond
, rtx op1
,
4329 optab op
= unknown_optab
;
4332 else if (code
== NOT
)
4337 insn_code icode
= direct_optab_handler (op
, mode
);
4339 if (icode
== CODE_FOR_nothing
)
4343 target
= gen_reg_rtx (mode
);
4345 rtx_insn
*last
= get_last_insn ();
4346 struct expand_operand ops
[4];
4348 create_output_operand (&ops
[0], target
, mode
);
4349 create_fixed_operand (&ops
[1], cond
);
4350 create_input_operand (&ops
[2], op1
, mode
);
4351 create_input_operand (&ops
[3], op2
, mode
);
4353 if (maybe_expand_insn (icode
, 4, ops
))
4355 if (ops
[0].value
!= target
)
4356 convert_move (target
, ops
[0].value
, false);
4360 delete_insns_since (last
);
4364 /* Emit a conditional addition instruction if the machine supports one for that
4365 condition and machine mode.
4367 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4368 the mode to use should they be constants. If it is VOIDmode, they cannot
4371 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4372 should be stored there. MODE is the mode to use should they be constants.
4373 If it is VOIDmode, they cannot both be constants.
4375 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4376 is not supported. */
4379 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4380 machine_mode cmode
, rtx op2
, rtx op3
,
4381 machine_mode mode
, int unsignedp
)
4385 enum insn_code icode
;
4387 /* If one operand is constant, make it the second one. Only do this
4388 if the other operand is not constant as well. */
4390 if (swap_commutative_operands_p (op0
, op1
))
4392 std::swap (op0
, op1
);
4393 code
= swap_condition (code
);
4396 /* get_condition will prefer to generate LT and GT even if the old
4397 comparison was against zero, so undo that canonicalization here since
4398 comparisons against zero are cheaper. */
4399 if (code
== LT
&& op1
== const1_rtx
)
4400 code
= LE
, op1
= const0_rtx
;
4401 else if (code
== GT
&& op1
== constm1_rtx
)
4402 code
= GE
, op1
= const0_rtx
;
4404 if (cmode
== VOIDmode
)
4405 cmode
= GET_MODE (op0
);
4407 if (mode
== VOIDmode
)
4408 mode
= GET_MODE (op2
);
4410 icode
= optab_handler (addcc_optab
, mode
);
4412 if (icode
== CODE_FOR_nothing
)
4416 target
= gen_reg_rtx (mode
);
4418 code
= unsignedp
? unsigned_condition (code
) : code
;
4419 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4421 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4422 return NULL and let the caller figure out how best to deal with this
4424 if (!COMPARISON_P (comparison
))
4427 do_pending_stack_adjust ();
4428 last
= get_last_insn ();
4429 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4430 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4431 &comparison
, &cmode
);
4434 struct expand_operand ops
[4];
4436 create_output_operand (&ops
[0], target
, mode
);
4437 create_fixed_operand (&ops
[1], comparison
);
4438 create_input_operand (&ops
[2], op2
, mode
);
4439 create_input_operand (&ops
[3], op3
, mode
);
4440 if (maybe_expand_insn (icode
, 4, ops
))
4442 if (ops
[0].value
!= target
)
4443 convert_move (target
, ops
[0].value
, false);
4447 delete_insns_since (last
);
4451 /* These functions attempt to generate an insn body, rather than
4452 emitting the insn, but if the gen function already emits them, we
4453 make no attempt to turn them back into naked patterns. */
4455 /* Generate and return an insn body to add Y to X. */
4458 gen_add2_insn (rtx x
, rtx y
)
4460 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
4462 gcc_assert (insn_operand_matches (icode
, 0, x
));
4463 gcc_assert (insn_operand_matches (icode
, 1, x
));
4464 gcc_assert (insn_operand_matches (icode
, 2, y
));
4466 return GEN_FCN (icode
) (x
, x
, y
);
4469 /* Generate and return an insn body to add r1 and c,
4470 storing the result in r0. */
4473 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
4475 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
4477 if (icode
== CODE_FOR_nothing
4478 || !insn_operand_matches (icode
, 0, r0
)
4479 || !insn_operand_matches (icode
, 1, r1
)
4480 || !insn_operand_matches (icode
, 2, c
))
4483 return GEN_FCN (icode
) (r0
, r1
, c
);
4487 have_add2_insn (rtx x
, rtx y
)
4489 enum insn_code icode
;
4491 gcc_assert (GET_MODE (x
) != VOIDmode
);
4493 icode
= optab_handler (add_optab
, GET_MODE (x
));
4495 if (icode
== CODE_FOR_nothing
)
4498 if (!insn_operand_matches (icode
, 0, x
)
4499 || !insn_operand_matches (icode
, 1, x
)
4500 || !insn_operand_matches (icode
, 2, y
))
4506 /* Generate and return an insn body to add Y to X. */
4509 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
4511 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4513 gcc_assert (insn_operand_matches (icode
, 0, x
));
4514 gcc_assert (insn_operand_matches (icode
, 1, y
));
4515 gcc_assert (insn_operand_matches (icode
, 2, z
));
4517 return GEN_FCN (icode
) (x
, y
, z
);
4520 /* Return true if the target implements an addptr pattern and X, Y,
4521 and Z are valid for the pattern predicates. */
4524 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
4526 enum insn_code icode
;
4528 gcc_assert (GET_MODE (x
) != VOIDmode
);
4530 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4532 if (icode
== CODE_FOR_nothing
)
4535 if (!insn_operand_matches (icode
, 0, x
)
4536 || !insn_operand_matches (icode
, 1, y
)
4537 || !insn_operand_matches (icode
, 2, z
))
4543 /* Generate and return an insn body to subtract Y from X. */
4546 gen_sub2_insn (rtx x
, rtx y
)
4548 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
4550 gcc_assert (insn_operand_matches (icode
, 0, x
));
4551 gcc_assert (insn_operand_matches (icode
, 1, x
));
4552 gcc_assert (insn_operand_matches (icode
, 2, y
));
4554 return GEN_FCN (icode
) (x
, x
, y
);
4557 /* Generate and return an insn body to subtract r1 and c,
4558 storing the result in r0. */
4561 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
4563 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
4565 if (icode
== CODE_FOR_nothing
4566 || !insn_operand_matches (icode
, 0, r0
)
4567 || !insn_operand_matches (icode
, 1, r1
)
4568 || !insn_operand_matches (icode
, 2, c
))
4571 return GEN_FCN (icode
) (r0
, r1
, c
);
4575 have_sub2_insn (rtx x
, rtx y
)
4577 enum insn_code icode
;
4579 gcc_assert (GET_MODE (x
) != VOIDmode
);
4581 icode
= optab_handler (sub_optab
, GET_MODE (x
));
4583 if (icode
== CODE_FOR_nothing
)
4586 if (!insn_operand_matches (icode
, 0, x
)
4587 || !insn_operand_matches (icode
, 1, x
)
4588 || !insn_operand_matches (icode
, 2, y
))
4594 /* Generate the body of an insn to extend Y (with mode MFROM)
4595 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4598 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
4599 machine_mode mfrom
, int unsignedp
)
4601 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
4602 return GEN_FCN (icode
) (x
, y
);
4605 /* Generate code to convert FROM to floating point
4606 and store in TO. FROM must be fixed point and not VOIDmode.
4607 UNSIGNEDP nonzero means regard FROM as unsigned.
4608 Normally this is done by correcting the final value
4609 if it is negative. */
4612 expand_float (rtx to
, rtx from
, int unsignedp
)
4614 enum insn_code icode
;
4616 machine_mode fmode
, imode
;
4617 bool can_do_signed
= false;
4619 /* Crash now, because we won't be able to decide which mode to use. */
4620 gcc_assert (GET_MODE (from
) != VOIDmode
);
4622 /* Look for an insn to do the conversion. Do it in the specified
4623 modes if possible; otherwise convert either input, output or both to
4624 wider mode. If the integer mode is wider than the mode of FROM,
4625 we can do the conversion signed even if the input is unsigned. */
4627 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
4628 FOR_EACH_MODE_FROM (imode
, GET_MODE (from
))
4630 int doing_unsigned
= unsignedp
;
4632 if (fmode
!= GET_MODE (to
)
4633 && significand_size (fmode
) < GET_MODE_PRECISION (GET_MODE (from
)))
4636 icode
= can_float_p (fmode
, imode
, unsignedp
);
4637 if (icode
== CODE_FOR_nothing
&& unsignedp
)
4639 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
4640 if (scode
!= CODE_FOR_nothing
)
4641 can_do_signed
= true;
4642 if (imode
!= GET_MODE (from
))
4643 icode
= scode
, doing_unsigned
= 0;
4646 if (icode
!= CODE_FOR_nothing
)
4648 if (imode
!= GET_MODE (from
))
4649 from
= convert_to_mode (imode
, from
, unsignedp
);
4651 if (fmode
!= GET_MODE (to
))
4652 target
= gen_reg_rtx (fmode
);
4654 emit_unop_insn (icode
, target
, from
,
4655 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
4658 convert_move (to
, target
, 0);
4663 /* Unsigned integer, and no way to convert directly. Convert as signed,
4664 then unconditionally adjust the result. */
4665 if (unsignedp
&& can_do_signed
)
4667 rtx_code_label
*label
= gen_label_rtx ();
4669 REAL_VALUE_TYPE offset
;
4671 /* Look for a usable floating mode FMODE wider than the source and at
4672 least as wide as the target. Using FMODE will avoid rounding woes
4673 with unsigned values greater than the signed maximum value. */
4675 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
4676 if (GET_MODE_PRECISION (GET_MODE (from
)) < GET_MODE_BITSIZE (fmode
)
4677 && can_float_p (fmode
, GET_MODE (from
), 0) != CODE_FOR_nothing
)
4680 if (fmode
== VOIDmode
)
4682 /* There is no such mode. Pretend the target is wide enough. */
4683 fmode
= GET_MODE (to
);
4685 /* Avoid double-rounding when TO is narrower than FROM. */
4686 if ((significand_size (fmode
) + 1)
4687 < GET_MODE_PRECISION (GET_MODE (from
)))
4690 rtx_code_label
*neglabel
= gen_label_rtx ();
4692 /* Don't use TARGET if it isn't a register, is a hard register,
4693 or is the wrong mode. */
4695 || REGNO (target
) < FIRST_PSEUDO_REGISTER
4696 || GET_MODE (target
) != fmode
)
4697 target
= gen_reg_rtx (fmode
);
4699 imode
= GET_MODE (from
);
4700 do_pending_stack_adjust ();
4702 /* Test whether the sign bit is set. */
4703 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
4706 /* The sign bit is not set. Convert as signed. */
4707 expand_float (target
, from
, 0);
4708 emit_jump_insn (targetm
.gen_jump (label
));
4711 /* The sign bit is set.
4712 Convert to a usable (positive signed) value by shifting right
4713 one bit, while remembering if a nonzero bit was shifted
4714 out; i.e., compute (from & 1) | (from >> 1). */
4716 emit_label (neglabel
);
4717 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
4718 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4719 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
4720 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
4722 expand_float (target
, temp
, 0);
4724 /* Multiply by 2 to undo the shift above. */
4725 temp
= expand_binop (fmode
, add_optab
, target
, target
,
4726 target
, 0, OPTAB_LIB_WIDEN
);
4728 emit_move_insn (target
, temp
);
4730 do_pending_stack_adjust ();
4736 /* If we are about to do some arithmetic to correct for an
4737 unsigned operand, do it in a pseudo-register. */
4739 if (GET_MODE (to
) != fmode
4740 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
4741 target
= gen_reg_rtx (fmode
);
4743 /* Convert as signed integer to floating. */
4744 expand_float (target
, from
, 0);
4746 /* If FROM is negative (and therefore TO is negative),
4747 correct its value by 2**bitwidth. */
4749 do_pending_stack_adjust ();
4750 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, GET_MODE (from
),
4754 real_2expN (&offset
, GET_MODE_PRECISION (GET_MODE (from
)), fmode
);
4755 temp
= expand_binop (fmode
, add_optab
, target
,
4756 const_double_from_real_value (offset
, fmode
),
4757 target
, 0, OPTAB_LIB_WIDEN
);
4759 emit_move_insn (target
, temp
);
4761 do_pending_stack_adjust ();
4766 /* No hardware instruction available; call a library routine. */
4771 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
4773 if (GET_MODE_PRECISION (GET_MODE (from
)) < GET_MODE_PRECISION (SImode
))
4774 from
= convert_to_mode (SImode
, from
, unsignedp
);
4776 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
4777 gcc_assert (libfunc
);
4781 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4782 GET_MODE (to
), 1, from
,
4784 insns
= get_insns ();
4787 emit_libcall_block (insns
, target
, value
,
4788 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
4789 GET_MODE (to
), from
));
4794 /* Copy result to requested destination
4795 if we have been computing in a temp location. */
4799 if (GET_MODE (target
) == GET_MODE (to
))
4800 emit_move_insn (to
, target
);
4802 convert_move (to
, target
, 0);
4806 /* Generate code to convert FROM to fixed point and store in TO. FROM
4807 must be floating point. */
4810 expand_fix (rtx to
, rtx from
, int unsignedp
)
4812 enum insn_code icode
;
4814 machine_mode fmode
, imode
;
4815 bool must_trunc
= false;
4817 /* We first try to find a pair of modes, one real and one integer, at
4818 least as wide as FROM and TO, respectively, in which we can open-code
4819 this conversion. If the integer mode is wider than the mode of TO,
4820 we can do the conversion either signed or unsigned. */
4822 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
4823 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
4825 int doing_unsigned
= unsignedp
;
4827 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
4828 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
4829 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
4831 if (icode
!= CODE_FOR_nothing
)
4833 rtx_insn
*last
= get_last_insn ();
4834 if (fmode
!= GET_MODE (from
))
4835 from
= convert_to_mode (fmode
, from
, 0);
4839 rtx temp
= gen_reg_rtx (GET_MODE (from
));
4840 from
= expand_unop (GET_MODE (from
), ftrunc_optab
, from
,
4844 if (imode
!= GET_MODE (to
))
4845 target
= gen_reg_rtx (imode
);
4847 if (maybe_emit_unop_insn (icode
, target
, from
,
4848 doing_unsigned
? UNSIGNED_FIX
: FIX
))
4851 convert_move (to
, target
, unsignedp
);
4854 delete_insns_since (last
);
4858 /* For an unsigned conversion, there is one more way to do it.
4859 If we have a signed conversion, we generate code that compares
4860 the real value to the largest representable positive number. If if
4861 is smaller, the conversion is done normally. Otherwise, subtract
4862 one plus the highest signed number, convert, and add it back.
4864 We only need to check all real modes, since we know we didn't find
4865 anything with a wider integer mode.
4867 This code used to extend FP value into mode wider than the destination.
4868 This is needed for decimal float modes which cannot accurately
4869 represent one plus the highest signed number of the same size, but
4870 not for binary modes. Consider, for instance conversion from SFmode
4873 The hot path through the code is dealing with inputs smaller than 2^63
4874 and doing just the conversion, so there is no bits to lose.
4876 In the other path we know the value is positive in the range 2^63..2^64-1
4877 inclusive. (as for other input overflow happens and result is undefined)
4878 So we know that the most important bit set in mantissa corresponds to
4879 2^63. The subtraction of 2^63 should not generate any rounding as it
4880 simply clears out that bit. The rest is trivial. */
4882 if (unsignedp
&& GET_MODE_PRECISION (GET_MODE (to
)) <= HOST_BITS_PER_WIDE_INT
)
4883 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
4884 if (CODE_FOR_nothing
!= can_fix_p (GET_MODE (to
), fmode
, 0, &must_trunc
)
4885 && (!DECIMAL_FLOAT_MODE_P (fmode
)
4886 || GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (GET_MODE (to
))))
4889 REAL_VALUE_TYPE offset
;
4891 rtx_code_label
*lab1
, *lab2
;
4894 bitsize
= GET_MODE_PRECISION (GET_MODE (to
));
4895 real_2expN (&offset
, bitsize
- 1, fmode
);
4896 limit
= const_double_from_real_value (offset
, fmode
);
4897 lab1
= gen_label_rtx ();
4898 lab2
= gen_label_rtx ();
4900 if (fmode
!= GET_MODE (from
))
4901 from
= convert_to_mode (fmode
, from
, 0);
4903 /* See if we need to do the subtraction. */
4904 do_pending_stack_adjust ();
4905 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
, GET_MODE (from
),
4908 /* If not, do the signed "fix" and branch around fixup code. */
4909 expand_fix (to
, from
, 0);
4910 emit_jump_insn (targetm
.gen_jump (lab2
));
4913 /* Otherwise, subtract 2**(N-1), convert to signed number,
4914 then add 2**(N-1). Do the addition using XOR since this
4915 will often generate better code. */
4917 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
4918 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
4919 expand_fix (to
, target
, 0);
4920 target
= expand_binop (GET_MODE (to
), xor_optab
, to
,
4922 (HOST_WIDE_INT_1
<< (bitsize
- 1),
4924 to
, 1, OPTAB_LIB_WIDEN
);
4927 emit_move_insn (to
, target
);
4931 if (optab_handler (mov_optab
, GET_MODE (to
)) != CODE_FOR_nothing
)
4933 /* Make a place for a REG_NOTE and add it. */
4934 insn
= emit_move_insn (to
, to
);
4935 set_dst_reg_note (insn
, REG_EQUAL
,
4936 gen_rtx_fmt_e (UNSIGNED_FIX
, GET_MODE (to
),
4944 /* We can't do it with an insn, so use a library call. But first ensure
4945 that the mode of TO is at least as wide as SImode, since those are the
4946 only library calls we know about. */
4948 if (GET_MODE_PRECISION (GET_MODE (to
)) < GET_MODE_PRECISION (SImode
))
4950 target
= gen_reg_rtx (SImode
);
4952 expand_fix (target
, from
, unsignedp
);
4960 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
4961 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
4962 gcc_assert (libfunc
);
4966 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4967 GET_MODE (to
), 1, from
,
4969 insns
= get_insns ();
4972 emit_libcall_block (insns
, target
, value
,
4973 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
4974 GET_MODE (to
), from
));
4979 if (GET_MODE (to
) == GET_MODE (target
))
4980 emit_move_insn (to
, target
);
4982 convert_move (to
, target
, 0);
4987 /* Promote integer arguments for a libcall if necessary.
4988 emit_library_call_value cannot do the promotion because it does not
4989 know if it should do a signed or unsigned promotion. This is because
4990 there are no tree types defined for libcalls. */
4993 prepare_libcall_arg (rtx arg
, int uintp
)
4995 machine_mode mode
= GET_MODE (arg
);
4996 machine_mode arg_mode
;
4997 if (SCALAR_INT_MODE_P (mode
))
4999 /* If we need to promote the integer function argument we need to do
5000 it here instead of inside emit_library_call_value because in
5001 emit_library_call_value we don't know if we should do a signed or
5002 unsigned promotion. */
5005 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
5006 &unsigned_p
, NULL_TREE
, 0);
5007 if (arg_mode
!= mode
)
5008 return convert_to_mode (arg_mode
, arg
, uintp
);
5013 /* Generate code to convert FROM or TO a fixed-point.
5014 If UINTP is true, either TO or FROM is an unsigned integer.
5015 If SATP is true, we need to saturate the result. */
5018 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
5020 machine_mode to_mode
= GET_MODE (to
);
5021 machine_mode from_mode
= GET_MODE (from
);
5023 enum rtx_code this_code
;
5024 enum insn_code code
;
5029 if (to_mode
== from_mode
)
5031 emit_move_insn (to
, from
);
5037 tab
= satp
? satfractuns_optab
: fractuns_optab
;
5038 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
5042 tab
= satp
? satfract_optab
: fract_optab
;
5043 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
5045 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
5046 if (code
!= CODE_FOR_nothing
)
5048 emit_unop_insn (code
, to
, from
, this_code
);
5052 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
5053 gcc_assert (libfunc
);
5055 from
= prepare_libcall_arg (from
, uintp
);
5056 from_mode
= GET_MODE (from
);
5059 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
5060 1, from
, from_mode
);
5061 insns
= get_insns ();
5064 emit_libcall_block (insns
, to
, value
,
5065 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
5068 /* Generate code to convert FROM to fixed point and store in TO. FROM
5069 must be floating point, TO must be signed. Use the conversion optab
5070 TAB to do the conversion. */
5073 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
5075 enum insn_code icode
;
5077 machine_mode fmode
, imode
;
5079 /* We first try to find a pair of modes, one real and one integer, at
5080 least as wide as FROM and TO, respectively, in which we can open-code
5081 this conversion. If the integer mode is wider than the mode of TO,
5082 we can do the conversion either signed or unsigned. */
5084 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5085 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5087 icode
= convert_optab_handler (tab
, imode
, fmode
);
5088 if (icode
!= CODE_FOR_nothing
)
5090 rtx_insn
*last
= get_last_insn ();
5091 if (fmode
!= GET_MODE (from
))
5092 from
= convert_to_mode (fmode
, from
, 0);
5094 if (imode
!= GET_MODE (to
))
5095 target
= gen_reg_rtx (imode
);
5097 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
5099 delete_insns_since (last
);
5103 convert_move (to
, target
, 0);
5111 /* Report whether we have an instruction to perform the operation
5112 specified by CODE on operands of mode MODE. */
5114 have_insn_for (enum rtx_code code
, machine_mode mode
)
5116 return (code_to_optab (code
)
5117 && (optab_handler (code_to_optab (code
), mode
)
5118 != CODE_FOR_nothing
));
5121 /* Print information about the current contents of the optabs on
5125 debug_optab_libfuncs (void)
5129 /* Dump the arithmetic optabs. */
5130 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
5131 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5133 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
5136 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5137 fprintf (stderr
, "%s\t%s:\t%s\n",
5138 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5144 /* Dump the conversion optabs. */
5145 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
5146 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5147 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
5149 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
5153 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5154 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
5155 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5163 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5164 CODE. Return 0 on failure. */
5167 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
5169 machine_mode mode
= GET_MODE (op1
);
5170 enum insn_code icode
;
5174 if (mode
== VOIDmode
)
5177 icode
= optab_handler (ctrap_optab
, mode
);
5178 if (icode
== CODE_FOR_nothing
)
5181 /* Some targets only accept a zero trap code. */
5182 if (!insn_operand_matches (icode
, 3, tcode
))
5185 do_pending_stack_adjust ();
5187 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
5192 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
5195 /* If that failed, then give up. */
5203 insn
= get_insns ();
5208 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5209 or unsigned operation code. */
5212 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
5224 code
= unsignedp
? LTU
: LT
;
5227 code
= unsignedp
? LEU
: LE
;
5230 code
= unsignedp
? GTU
: GT
;
5233 code
= unsignedp
? GEU
: GE
;
5236 case UNORDERED_EXPR
:
5275 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5276 select signed or unsigned operators. OPNO holds the index of the
5277 first comparison operand for insn ICODE. Do not generate the
5278 compare instruction itself. */
5281 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
5282 tree t_op0
, tree t_op1
, bool unsignedp
,
5283 enum insn_code icode
, unsigned int opno
)
5285 struct expand_operand ops
[2];
5286 rtx rtx_op0
, rtx_op1
;
5287 machine_mode m0
, m1
;
5288 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
5290 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
5292 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5293 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5294 cases, use the original mode. */
5295 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
5297 m0
= GET_MODE (rtx_op0
);
5299 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
5301 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
5303 m1
= GET_MODE (rtx_op1
);
5305 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
5307 create_input_operand (&ops
[0], rtx_op0
, m0
);
5308 create_input_operand (&ops
[1], rtx_op1
, m1
);
5309 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
5311 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
5314 /* Checks if vec_perm mask SEL is a constant equivalent to a shift of the first
5315 vec_perm operand, assuming the second operand is a constant vector of zeroes.
5316 Return the shift distance in bits if so, or NULL_RTX if the vec_perm is not a
5319 shift_amt_for_vec_perm_mask (rtx sel
)
5321 unsigned int i
, first
, nelt
= GET_MODE_NUNITS (GET_MODE (sel
));
5322 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (GET_MODE (sel
));
5324 if (GET_CODE (sel
) != CONST_VECTOR
)
5327 first
= INTVAL (CONST_VECTOR_ELT (sel
, 0));
5330 for (i
= 1; i
< nelt
; i
++)
5332 int idx
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
5333 unsigned int expected
= i
+ first
;
5334 /* Indices into the second vector are all equivalent. */
5335 if (idx
< 0 || (MIN (nelt
, (unsigned) idx
) != MIN (nelt
, expected
)))
5339 return GEN_INT (first
* bitsize
);
5342 /* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
5345 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
5346 rtx v0
, rtx v1
, rtx sel
)
5348 machine_mode tmode
= GET_MODE (target
);
5349 machine_mode smode
= GET_MODE (sel
);
5350 struct expand_operand ops
[4];
5352 create_output_operand (&ops
[0], target
, tmode
);
5353 create_input_operand (&ops
[3], sel
, smode
);
5355 /* Make an effort to preserve v0 == v1. The target expander is able to
5356 rely on this to determine if we're permuting a single input operand. */
5357 if (rtx_equal_p (v0
, v1
))
5359 if (!insn_operand_matches (icode
, 1, v0
))
5360 v0
= force_reg (tmode
, v0
);
5361 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
5362 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
5364 create_fixed_operand (&ops
[1], v0
);
5365 create_fixed_operand (&ops
[2], v0
);
5369 create_input_operand (&ops
[1], v0
, tmode
);
5370 create_input_operand (&ops
[2], v1
, tmode
);
5373 if (maybe_expand_insn (icode
, 4, ops
))
5374 return ops
[0].value
;
5378 /* Generate instructions for vec_perm optab given its mode
5379 and three operands. */
5382 expand_vec_perm (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
5384 enum insn_code icode
;
5385 machine_mode qimode
;
5386 unsigned int i
, w
, e
, u
;
5387 rtx tmp
, sel_qi
= NULL
;
5390 if (!target
|| GET_MODE (target
) != mode
)
5391 target
= gen_reg_rtx (mode
);
5393 w
= GET_MODE_SIZE (mode
);
5394 e
= GET_MODE_NUNITS (mode
);
5395 u
= GET_MODE_UNIT_SIZE (mode
);
5397 /* Set QIMODE to a different vector mode with byte elements.
5398 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5400 if (GET_MODE_INNER (mode
) != QImode
)
5402 qimode
= mode_for_vector (QImode
, w
);
5403 if (!VECTOR_MODE_P (qimode
))
5407 /* If the input is a constant, expand it specially. */
5408 gcc_assert (GET_MODE_CLASS (GET_MODE (sel
)) == MODE_VECTOR_INT
);
5409 if (GET_CODE (sel
) == CONST_VECTOR
)
5411 /* See if this can be handled with a vec_shr. We only do this if the
5412 second vector is all zeroes. */
5413 enum insn_code shift_code
= optab_handler (vec_shr_optab
, mode
);
5414 enum insn_code shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
5415 ? optab_handler (vec_shr_optab
, qimode
)
5416 : CODE_FOR_nothing
);
5417 rtx shift_amt
= NULL_RTX
;
5418 if (v1
== CONST0_RTX (GET_MODE (v1
))
5419 && (shift_code
!= CODE_FOR_nothing
5420 || shift_code_qi
!= CODE_FOR_nothing
))
5422 shift_amt
= shift_amt_for_vec_perm_mask (sel
);
5425 struct expand_operand ops
[3];
5426 if (shift_code
!= CODE_FOR_nothing
)
5428 create_output_operand (&ops
[0], target
, mode
);
5429 create_input_operand (&ops
[1], v0
, mode
);
5430 create_convert_operand_from_type (&ops
[2], shift_amt
,
5432 if (maybe_expand_insn (shift_code
, 3, ops
))
5433 return ops
[0].value
;
5435 if (shift_code_qi
!= CODE_FOR_nothing
)
5437 tmp
= gen_reg_rtx (qimode
);
5438 create_output_operand (&ops
[0], tmp
, qimode
);
5439 create_input_operand (&ops
[1], gen_lowpart (qimode
, v0
),
5441 create_convert_operand_from_type (&ops
[2], shift_amt
,
5443 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
5444 return gen_lowpart (mode
, ops
[0].value
);
5449 icode
= direct_optab_handler (vec_perm_const_optab
, mode
);
5450 if (icode
!= CODE_FOR_nothing
)
5452 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5457 /* Fall back to a constant byte-based permutation. */
5458 if (qimode
!= VOIDmode
)
5460 vec
= rtvec_alloc (w
);
5461 for (i
= 0; i
< e
; ++i
)
5463 unsigned int j
, this_e
;
5465 this_e
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
5466 this_e
&= 2 * e
- 1;
5469 for (j
= 0; j
< u
; ++j
)
5470 RTVEC_ELT (vec
, i
* u
+ j
) = GEN_INT (this_e
+ j
);
5472 sel_qi
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5474 icode
= direct_optab_handler (vec_perm_const_optab
, qimode
);
5475 if (icode
!= CODE_FOR_nothing
)
5477 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5478 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5479 gen_lowpart (qimode
, v1
), sel_qi
);
5481 return gen_lowpart (mode
, tmp
);
5486 /* Otherwise expand as a fully variable permuation. */
5487 icode
= direct_optab_handler (vec_perm_optab
, mode
);
5488 if (icode
!= CODE_FOR_nothing
)
5490 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5495 /* As a special case to aid several targets, lower the element-based
5496 permutation to a byte-based permutation and try again. */
5497 if (qimode
== VOIDmode
)
5499 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
5500 if (icode
== CODE_FOR_nothing
)
5505 /* Multiply each element by its byte size. */
5506 machine_mode selmode
= GET_MODE (sel
);
5508 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
5509 NULL
, 0, OPTAB_DIRECT
);
5511 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
5512 GEN_INT (exact_log2 (u
)),
5513 NULL
, 0, OPTAB_DIRECT
);
5514 gcc_assert (sel
!= NULL
);
5516 /* Broadcast the low byte each element into each of its bytes. */
5517 vec
= rtvec_alloc (w
);
5518 for (i
= 0; i
< w
; ++i
)
5520 int this_e
= i
/ u
* u
;
5521 if (BYTES_BIG_ENDIAN
)
5523 RTVEC_ELT (vec
, i
) = GEN_INT (this_e
);
5525 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5526 sel
= gen_lowpart (qimode
, sel
);
5527 sel
= expand_vec_perm (qimode
, sel
, sel
, tmp
, NULL
);
5528 gcc_assert (sel
!= NULL
);
5530 /* Add the byte offset to each byte element. */
5531 /* Note that the definition of the indicies here is memory ordering,
5532 so there should be no difference between big and little endian. */
5533 vec
= rtvec_alloc (w
);
5534 for (i
= 0; i
< w
; ++i
)
5535 RTVEC_ELT (vec
, i
) = GEN_INT (i
% u
);
5536 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5537 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
5538 sel
, 0, OPTAB_DIRECT
);
5539 gcc_assert (sel_qi
!= NULL
);
5542 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5543 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5544 gen_lowpart (qimode
, v1
), sel_qi
);
5546 tmp
= gen_lowpart (mode
, tmp
);
5550 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5554 expand_vec_cond_mask_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5557 struct expand_operand ops
[4];
5558 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5559 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
5560 enum insn_code icode
= get_vcond_mask_icode (mode
, mask_mode
);
5561 rtx mask
, rtx_op1
, rtx_op2
;
5563 if (icode
== CODE_FOR_nothing
)
5566 mask
= expand_normal (op0
);
5567 rtx_op1
= expand_normal (op1
);
5568 rtx_op2
= expand_normal (op2
);
5570 mask
= force_reg (mask_mode
, mask
);
5571 rtx_op1
= force_reg (GET_MODE (rtx_op1
), rtx_op1
);
5573 create_output_operand (&ops
[0], target
, mode
);
5574 create_input_operand (&ops
[1], rtx_op1
, mode
);
5575 create_input_operand (&ops
[2], rtx_op2
, mode
);
5576 create_input_operand (&ops
[3], mask
, mask_mode
);
5577 expand_insn (icode
, 4, ops
);
5579 return ops
[0].value
;
5582 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5586 expand_vec_cond_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5589 struct expand_operand ops
[6];
5590 enum insn_code icode
;
5591 rtx comparison
, rtx_op1
, rtx_op2
;
5592 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5593 machine_mode cmp_op_mode
;
5596 enum tree_code tcode
;
5598 if (COMPARISON_CLASS_P (op0
))
5600 op0a
= TREE_OPERAND (op0
, 0);
5601 op0b
= TREE_OPERAND (op0
, 1);
5602 tcode
= TREE_CODE (op0
);
5606 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0
)));
5607 if (get_vcond_mask_icode (mode
, TYPE_MODE (TREE_TYPE (op0
)))
5608 != CODE_FOR_nothing
)
5609 return expand_vec_cond_mask_expr (vec_cond_type
, op0
, op1
,
5614 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0
)))
5615 == MODE_VECTOR_INT
);
5617 op0b
= build_zero_cst (TREE_TYPE (op0
));
5621 cmp_op_mode
= TYPE_MODE (TREE_TYPE (op0a
));
5622 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5625 gcc_assert (GET_MODE_SIZE (mode
) == GET_MODE_SIZE (cmp_op_mode
)
5626 && GET_MODE_NUNITS (mode
) == GET_MODE_NUNITS (cmp_op_mode
));
5628 icode
= get_vcond_icode (mode
, cmp_op_mode
, unsignedp
);
5629 if (icode
== CODE_FOR_nothing
)
5631 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5632 icode
= get_vcond_eq_icode (mode
, cmp_op_mode
);
5633 if (icode
== CODE_FOR_nothing
)
5637 comparison
= vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
,
5639 rtx_op1
= expand_normal (op1
);
5640 rtx_op2
= expand_normal (op2
);
5642 create_output_operand (&ops
[0], target
, mode
);
5643 create_input_operand (&ops
[1], rtx_op1
, mode
);
5644 create_input_operand (&ops
[2], rtx_op2
, mode
);
5645 create_fixed_operand (&ops
[3], comparison
);
5646 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
5647 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
5648 expand_insn (icode
, 6, ops
);
5649 return ops
[0].value
;
5652 /* Generate insns for a vector comparison into a mask. */
5655 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
5657 struct expand_operand ops
[4];
5658 enum insn_code icode
;
5660 machine_mode mask_mode
= TYPE_MODE (type
);
5664 enum tree_code tcode
;
5666 op0a
= TREE_OPERAND (exp
, 0);
5667 op0b
= TREE_OPERAND (exp
, 1);
5668 tcode
= TREE_CODE (exp
);
5670 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5671 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
5673 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
5674 if (icode
== CODE_FOR_nothing
)
5676 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5677 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
5678 if (icode
== CODE_FOR_nothing
)
5682 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
5683 unsignedp
, icode
, 2);
5684 create_output_operand (&ops
[0], target
, mask_mode
);
5685 create_fixed_operand (&ops
[1], comparison
);
5686 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
5687 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
5688 expand_insn (icode
, 4, ops
);
5689 return ops
[0].value
;
5692 /* Expand a highpart multiply. */
5695 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
5696 rtx target
, bool uns_p
)
5698 struct expand_operand eops
[3];
5699 enum insn_code icode
;
5700 int method
, i
, nunits
;
5706 method
= can_mult_highpart_p (mode
, uns_p
);
5712 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
5713 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
5716 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
5717 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
5720 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
5721 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
5722 if (BYTES_BIG_ENDIAN
)
5723 std::swap (tab1
, tab2
);
5729 icode
= optab_handler (tab1
, mode
);
5730 nunits
= GET_MODE_NUNITS (mode
);
5731 wmode
= insn_data
[icode
].operand
[0].mode
;
5732 gcc_checking_assert (2 * GET_MODE_NUNITS (wmode
) == nunits
);
5733 gcc_checking_assert (GET_MODE_SIZE (wmode
) == GET_MODE_SIZE (mode
));
5735 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5736 create_input_operand (&eops
[1], op0
, mode
);
5737 create_input_operand (&eops
[2], op1
, mode
);
5738 expand_insn (icode
, 3, eops
);
5739 m1
= gen_lowpart (mode
, eops
[0].value
);
5741 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5742 create_input_operand (&eops
[1], op0
, mode
);
5743 create_input_operand (&eops
[2], op1
, mode
);
5744 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
5745 m2
= gen_lowpart (mode
, eops
[0].value
);
5747 v
= rtvec_alloc (nunits
);
5750 for (i
= 0; i
< nunits
; ++i
)
5751 RTVEC_ELT (v
, i
) = GEN_INT (!BYTES_BIG_ENDIAN
+ (i
& ~1)
5752 + ((i
& 1) ? nunits
: 0));
5756 for (i
= 0; i
< nunits
; ++i
)
5757 RTVEC_ELT (v
, i
) = GEN_INT (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
5759 perm
= gen_rtx_CONST_VECTOR (mode
, v
);
5761 return expand_vec_perm (mode
, m1
, m2
, perm
, target
);
5764 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
5768 find_cc_set (rtx x
, const_rtx pat
, void *data
)
5770 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
5771 && GET_CODE (pat
) == SET
)
5773 rtx
*p_cc_reg
= (rtx
*) data
;
5774 gcc_assert (!*p_cc_reg
);
5779 /* This is a helper function for the other atomic operations. This function
5780 emits a loop that contains SEQ that iterates until a compare-and-swap
5781 operation at the end succeeds. MEM is the memory to be modified. SEQ is
5782 a set of instructions that takes a value from OLD_REG as an input and
5783 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
5784 set to the current contents of MEM. After SEQ, a compare-and-swap will
5785 attempt to update MEM with NEW_REG. The function returns true when the
5786 loop was generated successfully. */
5789 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
5791 machine_mode mode
= GET_MODE (mem
);
5792 rtx_code_label
*label
;
5793 rtx cmp_reg
, success
, oldval
;
5795 /* The loop we want to generate looks like
5801 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
5805 Note that we only do the plain load from memory once. Subsequent
5806 iterations use the value loaded by the compare-and-swap pattern. */
5808 label
= gen_label_rtx ();
5809 cmp_reg
= gen_reg_rtx (mode
);
5811 emit_move_insn (cmp_reg
, mem
);
5813 emit_move_insn (old_reg
, cmp_reg
);
5819 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
5820 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
5824 if (oldval
!= cmp_reg
)
5825 emit_move_insn (cmp_reg
, oldval
);
5827 /* Mark this jump predicted not taken. */
5828 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
5829 GET_MODE (success
), 1, label
,
5830 profile_probability::guessed_never ());
5835 /* This function tries to emit an atomic_exchange intruction. VAL is written
5836 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
5837 using TARGET if possible. */
5840 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
5842 machine_mode mode
= GET_MODE (mem
);
5843 enum insn_code icode
;
5845 /* If the target supports the exchange directly, great. */
5846 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
5847 if (icode
!= CODE_FOR_nothing
)
5849 struct expand_operand ops
[4];
5851 create_output_operand (&ops
[0], target
, mode
);
5852 create_fixed_operand (&ops
[1], mem
);
5853 create_input_operand (&ops
[2], val
, mode
);
5854 create_integer_operand (&ops
[3], model
);
5855 if (maybe_expand_insn (icode
, 4, ops
))
5856 return ops
[0].value
;
5862 /* This function tries to implement an atomic exchange operation using
5863 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
5864 The previous contents of *MEM are returned, using TARGET if possible.
5865 Since this instructionn is an acquire barrier only, stronger memory
5866 models may require additional barriers to be emitted. */
5869 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
5870 enum memmodel model
)
5872 machine_mode mode
= GET_MODE (mem
);
5873 enum insn_code icode
;
5874 rtx_insn
*last_insn
= get_last_insn ();
5876 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
5878 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
5879 exists, and the memory model is stronger than acquire, add a release
5880 barrier before the instruction. */
5882 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
5883 expand_mem_thread_fence (model
);
5885 if (icode
!= CODE_FOR_nothing
)
5887 struct expand_operand ops
[3];
5888 create_output_operand (&ops
[0], target
, mode
);
5889 create_fixed_operand (&ops
[1], mem
);
5890 create_input_operand (&ops
[2], val
, mode
);
5891 if (maybe_expand_insn (icode
, 3, ops
))
5892 return ops
[0].value
;
5895 /* If an external test-and-set libcall is provided, use that instead of
5896 any external compare-and-swap that we might get from the compare-and-
5897 swap-loop expansion later. */
5898 if (!can_compare_and_swap_p (mode
, false))
5900 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
5901 if (libfunc
!= NULL
)
5905 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
5906 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
5907 mode
, 2, addr
, ptr_mode
,
5912 /* If the test_and_set can't be emitted, eliminate any barrier that might
5913 have been emitted. */
5914 delete_insns_since (last_insn
);
5918 /* This function tries to implement an atomic exchange operation using a
5919 compare_and_swap loop. VAL is written to *MEM. The previous contents of
5920 *MEM are returned, using TARGET if possible. No memory model is required
5921 since a compare_and_swap loop is seq-cst. */
5924 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
5926 machine_mode mode
= GET_MODE (mem
);
5928 if (can_compare_and_swap_p (mode
, true))
5930 if (!target
|| !register_operand (target
, mode
))
5931 target
= gen_reg_rtx (mode
);
5932 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
5939 /* This function tries to implement an atomic test-and-set operation
5940 using the atomic_test_and_set instruction pattern. A boolean value
5941 is returned from the operation, using TARGET if possible. */
5944 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
5946 machine_mode pat_bool_mode
;
5947 struct expand_operand ops
[3];
5949 if (!targetm
.have_atomic_test_and_set ())
5952 /* While we always get QImode from __atomic_test_and_set, we get
5953 other memory modes from __sync_lock_test_and_set. Note that we
5954 use no endian adjustment here. This matches the 4.6 behavior
5955 in the Sparc backend. */
5956 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
5957 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
5958 if (GET_MODE (mem
) != QImode
)
5959 mem
= adjust_address_nv (mem
, QImode
, 0);
5961 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
5962 create_output_operand (&ops
[0], target
, pat_bool_mode
);
5963 create_fixed_operand (&ops
[1], mem
);
5964 create_integer_operand (&ops
[2], model
);
5966 if (maybe_expand_insn (icode
, 3, ops
))
5967 return ops
[0].value
;
5971 /* This function expands the legacy _sync_lock test_and_set operation which is
5972 generally an atomic exchange. Some limited targets only allow the
5973 constant 1 to be stored. This is an ACQUIRE operation.
5975 TARGET is an optional place to stick the return value.
5976 MEM is where VAL is stored. */
5979 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
5983 /* Try an atomic_exchange first. */
5984 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
5988 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
5989 MEMMODEL_SYNC_ACQUIRE
);
5993 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
5997 /* If there are no other options, try atomic_test_and_set if the value
5998 being stored is 1. */
5999 if (val
== const1_rtx
)
6000 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
6005 /* This function expands the atomic test_and_set operation:
6006 atomically store a boolean TRUE into MEM and return the previous value.
6008 MEMMODEL is the memory model variant to use.
6009 TARGET is an optional place to stick the return value. */
6012 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6014 machine_mode mode
= GET_MODE (mem
);
6015 rtx ret
, trueval
, subtarget
;
6017 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
6021 /* Be binary compatible with non-default settings of trueval, and different
6022 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6023 another only has atomic-exchange. */
6024 if (targetm
.atomic_test_and_set_trueval
== 1)
6026 trueval
= const1_rtx
;
6027 subtarget
= target
? target
: gen_reg_rtx (mode
);
6031 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
6032 subtarget
= gen_reg_rtx (mode
);
6035 /* Try the atomic-exchange optab... */
6036 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
6038 /* ... then an atomic-compare-and-swap loop ... */
6040 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
6042 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6044 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
6046 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6047 things with the value 1. Thus we try again without trueval. */
6048 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
6049 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
6051 /* Failing all else, assume a single threaded environment and simply
6052 perform the operation. */
6055 /* If the result is ignored skip the move to target. */
6056 if (subtarget
!= const0_rtx
)
6057 emit_move_insn (subtarget
, mem
);
6059 emit_move_insn (mem
, trueval
);
6063 /* Recall that have to return a boolean value; rectify if trueval
6064 is not exactly one. */
6065 if (targetm
.atomic_test_and_set_trueval
!= 1)
6066 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
6071 /* This function expands the atomic exchange operation:
6072 atomically store VAL in MEM and return the previous value in MEM.
6074 MEMMODEL is the memory model variant to use.
6075 TARGET is an optional place to stick the return value. */
6078 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6080 machine_mode mode
= GET_MODE (mem
);
6083 /* If loads are not atomic for the required size and we are not called to
6084 provide a __sync builtin, do not do anything so that we stay consistent
6085 with atomic loads of the same size. */
6086 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6089 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6091 /* Next try a compare-and-swap loop for the exchange. */
6093 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6098 /* This function expands the atomic compare exchange operation:
6100 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6101 *PTARGET_OVAL is an optional place to store the old value from memory.
6102 Both target parameters may be NULL or const0_rtx to indicate that we do
6103 not care about that return value. Both target parameters are updated on
6104 success to the actual location of the corresponding result.
6106 MEMMODEL is the memory model variant to use.
6108 The return value of the function is true for success. */
6111 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
6112 rtx mem
, rtx expected
, rtx desired
,
6113 bool is_weak
, enum memmodel succ_model
,
6114 enum memmodel fail_model
)
6116 machine_mode mode
= GET_MODE (mem
);
6117 struct expand_operand ops
[8];
6118 enum insn_code icode
;
6119 rtx target_oval
, target_bool
= NULL_RTX
;
6122 /* If loads are not atomic for the required size and we are not called to
6123 provide a __sync builtin, do not do anything so that we stay consistent
6124 with atomic loads of the same size. */
6125 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
6128 /* Load expected into a register for the compare and swap. */
6129 if (MEM_P (expected
))
6130 expected
= copy_to_reg (expected
);
6132 /* Make sure we always have some place to put the return oldval.
6133 Further, make sure that place is distinct from the input expected,
6134 just in case we need that path down below. */
6135 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
6136 ptarget_oval
= NULL
;
6138 if (ptarget_oval
== NULL
6139 || (target_oval
= *ptarget_oval
) == NULL
6140 || reg_overlap_mentioned_p (expected
, target_oval
))
6141 target_oval
= gen_reg_rtx (mode
);
6143 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
6144 if (icode
!= CODE_FOR_nothing
)
6146 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
6148 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
6149 ptarget_bool
= NULL
;
6151 /* Make sure we always have a place for the bool operand. */
6152 if (ptarget_bool
== NULL
6153 || (target_bool
= *ptarget_bool
) == NULL
6154 || GET_MODE (target_bool
) != bool_mode
)
6155 target_bool
= gen_reg_rtx (bool_mode
);
6157 /* Emit the compare_and_swap. */
6158 create_output_operand (&ops
[0], target_bool
, bool_mode
);
6159 create_output_operand (&ops
[1], target_oval
, mode
);
6160 create_fixed_operand (&ops
[2], mem
);
6161 create_input_operand (&ops
[3], expected
, mode
);
6162 create_input_operand (&ops
[4], desired
, mode
);
6163 create_integer_operand (&ops
[5], is_weak
);
6164 create_integer_operand (&ops
[6], succ_model
);
6165 create_integer_operand (&ops
[7], fail_model
);
6166 if (maybe_expand_insn (icode
, 8, ops
))
6168 /* Return success/failure. */
6169 target_bool
= ops
[0].value
;
6170 target_oval
= ops
[1].value
;
6175 /* Otherwise fall back to the original __sync_val_compare_and_swap
6176 which is always seq-cst. */
6177 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
6178 if (icode
!= CODE_FOR_nothing
)
6182 create_output_operand (&ops
[0], target_oval
, mode
);
6183 create_fixed_operand (&ops
[1], mem
);
6184 create_input_operand (&ops
[2], expected
, mode
);
6185 create_input_operand (&ops
[3], desired
, mode
);
6186 if (!maybe_expand_insn (icode
, 4, ops
))
6189 target_oval
= ops
[0].value
;
6191 /* If the caller isn't interested in the boolean return value,
6192 skip the computation of it. */
6193 if (ptarget_bool
== NULL
)
6196 /* Otherwise, work out if the compare-and-swap succeeded. */
6198 if (have_insn_for (COMPARE
, CCmode
))
6199 note_stores (PATTERN (get_last_insn ()), find_cc_set
, &cc_reg
);
6202 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
6203 const0_rtx
, VOIDmode
, 0, 1);
6206 goto success_bool_from_val
;
6209 /* Also check for library support for __sync_val_compare_and_swap. */
6210 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
6211 if (libfunc
!= NULL
)
6213 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6214 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6215 mode
, 3, addr
, ptr_mode
,
6216 expected
, mode
, desired
, mode
);
6217 emit_move_insn (target_oval
, target
);
6219 /* Compute the boolean return value only if requested. */
6221 goto success_bool_from_val
;
6229 success_bool_from_val
:
6230 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
6231 expected
, VOIDmode
, 1, 1);
6233 /* Make sure that the oval output winds up where the caller asked. */
6235 *ptarget_oval
= target_oval
;
6237 *ptarget_bool
= target_bool
;
6241 /* Generate asm volatile("" : : : "memory") as the memory barrier. */
6244 expand_asm_memory_barrier (void)
6248 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
6249 rtvec_alloc (0), rtvec_alloc (0),
6250 rtvec_alloc (0), UNKNOWN_LOCATION
);
6251 MEM_VOLATILE_P (asm_op
) = 1;
6253 clob
= gen_rtx_SCRATCH (VOIDmode
);
6254 clob
= gen_rtx_MEM (BLKmode
, clob
);
6255 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
6257 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
6260 /* This routine will either emit the mem_thread_fence pattern or issue a
6261 sync_synchronize to generate a fence for memory model MEMMODEL. */
6264 expand_mem_thread_fence (enum memmodel model
)
6266 if (is_mm_relaxed (model
))
6268 if (targetm
.have_mem_thread_fence ())
6270 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
6271 expand_asm_memory_barrier ();
6273 else if (targetm
.have_memory_barrier ())
6274 emit_insn (targetm
.gen_memory_barrier ());
6275 else if (synchronize_libfunc
!= NULL_RTX
)
6276 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
, 0);
6278 expand_asm_memory_barrier ();
6281 /* This routine will either emit the mem_signal_fence pattern or issue a
6282 sync_synchronize to generate a fence for memory model MEMMODEL. */
6285 expand_mem_signal_fence (enum memmodel model
)
6287 if (targetm
.have_mem_signal_fence ())
6288 emit_insn (targetm
.gen_mem_signal_fence (GEN_INT (model
)));
6289 else if (!is_mm_relaxed (model
))
6291 /* By default targets are coherent between a thread and the signal
6292 handler running on the same thread. Thus this really becomes a
6293 compiler barrier, in that stores must not be sunk past
6294 (or raised above) a given point. */
6295 expand_asm_memory_barrier ();
6299 /* This function expands the atomic load operation:
6300 return the atomically loaded value in MEM.
6302 MEMMODEL is the memory model variant to use.
6303 TARGET is an option place to stick the return value. */
6306 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
6308 machine_mode mode
= GET_MODE (mem
);
6309 enum insn_code icode
;
6311 /* If the target supports the load directly, great. */
6312 icode
= direct_optab_handler (atomic_load_optab
, mode
);
6313 if (icode
!= CODE_FOR_nothing
)
6315 struct expand_operand ops
[3];
6317 create_output_operand (&ops
[0], target
, mode
);
6318 create_fixed_operand (&ops
[1], mem
);
6319 create_integer_operand (&ops
[2], model
);
6320 if (maybe_expand_insn (icode
, 3, ops
))
6321 return ops
[0].value
;
6324 /* If the size of the object is greater than word size on this target,
6325 then we assume that a load will not be atomic. We could try to
6326 emulate a load with a compare-and-swap operation, but the store that
6327 doing this could result in would be incorrect if this is a volatile
6328 atomic load or targetting read-only-mapped memory. */
6329 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
6330 /* If there is no atomic load, leave the library call. */
6333 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6334 if (!target
|| target
== const0_rtx
)
6335 target
= gen_reg_rtx (mode
);
6337 /* For SEQ_CST, emit a barrier before the load. */
6338 if (is_mm_seq_cst (model
))
6339 expand_mem_thread_fence (model
);
6341 emit_move_insn (target
, mem
);
6343 /* Emit the appropriate barrier after the load. */
6344 expand_mem_thread_fence (model
);
6349 /* This function expands the atomic store operation:
6350 Atomically store VAL in MEM.
6351 MEMMODEL is the memory model variant to use.
6352 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6353 function returns const0_rtx if a pattern was emitted. */
6356 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
6358 machine_mode mode
= GET_MODE (mem
);
6359 enum insn_code icode
;
6360 struct expand_operand ops
[3];
6362 /* If the target supports the store directly, great. */
6363 icode
= direct_optab_handler (atomic_store_optab
, mode
);
6364 if (icode
!= CODE_FOR_nothing
)
6366 create_fixed_operand (&ops
[0], mem
);
6367 create_input_operand (&ops
[1], val
, mode
);
6368 create_integer_operand (&ops
[2], model
);
6369 if (maybe_expand_insn (icode
, 3, ops
))
6373 /* If using __sync_lock_release is a viable alternative, try it.
6374 Note that this will not be set to true if we are expanding a generic
6375 __atomic_store_n. */
6378 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
6379 if (icode
!= CODE_FOR_nothing
)
6381 create_fixed_operand (&ops
[0], mem
);
6382 create_input_operand (&ops
[1], const0_rtx
, mode
);
6383 if (maybe_expand_insn (icode
, 2, ops
))
6385 /* lock_release is only a release barrier. */
6386 if (is_mm_seq_cst (model
))
6387 expand_mem_thread_fence (model
);
6393 /* If the size of the object is greater than word size on this target,
6394 a default store will not be atomic. */
6395 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
6397 /* If loads are atomic or we are called to provide a __sync builtin,
6398 we can try a atomic_exchange and throw away the result. Otherwise,
6399 don't do anything so that we do not create an inconsistency between
6400 loads and stores. */
6401 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
6403 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
6405 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
6413 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6414 expand_mem_thread_fence (model
);
6416 emit_move_insn (mem
, val
);
6418 /* For SEQ_CST, also emit a barrier after the store. */
6419 if (is_mm_seq_cst (model
))
6420 expand_mem_thread_fence (model
);
6426 /* Structure containing the pointers and values required to process the
6427 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6429 struct atomic_op_functions
6431 direct_optab mem_fetch_before
;
6432 direct_optab mem_fetch_after
;
6433 direct_optab mem_no_result
;
6436 direct_optab no_result
;
6437 enum rtx_code reverse_code
;
6441 /* Fill in structure pointed to by OP with the various optab entries for an
6442 operation of type CODE. */
6445 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
6447 gcc_assert (op
!= NULL
);
6449 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6450 in the source code during compilation, and the optab entries are not
6451 computable until runtime. Fill in the values at runtime. */
6455 op
->mem_fetch_before
= atomic_fetch_add_optab
;
6456 op
->mem_fetch_after
= atomic_add_fetch_optab
;
6457 op
->mem_no_result
= atomic_add_optab
;
6458 op
->fetch_before
= sync_old_add_optab
;
6459 op
->fetch_after
= sync_new_add_optab
;
6460 op
->no_result
= sync_add_optab
;
6461 op
->reverse_code
= MINUS
;
6464 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
6465 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
6466 op
->mem_no_result
= atomic_sub_optab
;
6467 op
->fetch_before
= sync_old_sub_optab
;
6468 op
->fetch_after
= sync_new_sub_optab
;
6469 op
->no_result
= sync_sub_optab
;
6470 op
->reverse_code
= PLUS
;
6473 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
6474 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
6475 op
->mem_no_result
= atomic_xor_optab
;
6476 op
->fetch_before
= sync_old_xor_optab
;
6477 op
->fetch_after
= sync_new_xor_optab
;
6478 op
->no_result
= sync_xor_optab
;
6479 op
->reverse_code
= XOR
;
6482 op
->mem_fetch_before
= atomic_fetch_and_optab
;
6483 op
->mem_fetch_after
= atomic_and_fetch_optab
;
6484 op
->mem_no_result
= atomic_and_optab
;
6485 op
->fetch_before
= sync_old_and_optab
;
6486 op
->fetch_after
= sync_new_and_optab
;
6487 op
->no_result
= sync_and_optab
;
6488 op
->reverse_code
= UNKNOWN
;
6491 op
->mem_fetch_before
= atomic_fetch_or_optab
;
6492 op
->mem_fetch_after
= atomic_or_fetch_optab
;
6493 op
->mem_no_result
= atomic_or_optab
;
6494 op
->fetch_before
= sync_old_ior_optab
;
6495 op
->fetch_after
= sync_new_ior_optab
;
6496 op
->no_result
= sync_ior_optab
;
6497 op
->reverse_code
= UNKNOWN
;
6500 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
6501 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
6502 op
->mem_no_result
= atomic_nand_optab
;
6503 op
->fetch_before
= sync_old_nand_optab
;
6504 op
->fetch_after
= sync_new_nand_optab
;
6505 op
->no_result
= sync_nand_optab
;
6506 op
->reverse_code
= UNKNOWN
;
6513 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6514 using memory order MODEL. If AFTER is true the operation needs to return
6515 the value of *MEM after the operation, otherwise the previous value.
6516 TARGET is an optional place to place the result. The result is unused if
6518 Return the result if there is a better sequence, otherwise NULL_RTX. */
6521 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6522 enum memmodel model
, bool after
)
6524 /* If the value is prefetched, or not used, it may be possible to replace
6525 the sequence with a native exchange operation. */
6526 if (!after
|| target
== const0_rtx
)
6528 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
6529 if (code
== AND
&& val
== const0_rtx
)
6531 if (target
== const0_rtx
)
6532 target
= gen_reg_rtx (GET_MODE (mem
));
6533 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6536 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
6537 if (code
== IOR
&& val
== constm1_rtx
)
6539 if (target
== const0_rtx
)
6540 target
= gen_reg_rtx (GET_MODE (mem
));
6541 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6548 /* Try to emit an instruction for a specific operation varaition.
6549 OPTAB contains the OP functions.
6550 TARGET is an optional place to return the result. const0_rtx means unused.
6551 MEM is the memory location to operate on.
6552 VAL is the value to use in the operation.
6553 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6554 MODEL is the memory model, if used.
6555 AFTER is true if the returned result is the value after the operation. */
6558 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
6559 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
6561 machine_mode mode
= GET_MODE (mem
);
6562 struct expand_operand ops
[4];
6563 enum insn_code icode
;
6567 /* Check to see if there is a result returned. */
6568 if (target
== const0_rtx
)
6572 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
6573 create_integer_operand (&ops
[2], model
);
6578 icode
= direct_optab_handler (optab
->no_result
, mode
);
6582 /* Otherwise, we need to generate a result. */
6587 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
6588 : optab
->mem_fetch_before
, mode
);
6589 create_integer_operand (&ops
[3], model
);
6594 icode
= optab_handler (after
? optab
->fetch_after
6595 : optab
->fetch_before
, mode
);
6598 create_output_operand (&ops
[op_counter
++], target
, mode
);
6600 if (icode
== CODE_FOR_nothing
)
6603 create_fixed_operand (&ops
[op_counter
++], mem
);
6604 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6605 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
6607 if (maybe_expand_insn (icode
, num_ops
, ops
))
6608 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
6614 /* This function expands an atomic fetch_OP or OP_fetch operation:
6615 TARGET is an option place to stick the return value. const0_rtx indicates
6616 the result is unused.
6617 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6618 CODE is the operation being performed (OP)
6619 MEMMODEL is the memory model variant to use.
6620 AFTER is true to return the result of the operation (OP_fetch).
6621 AFTER is false to return the value before the operation (fetch_OP).
6623 This function will *only* generate instructions if there is a direct
6624 optab. No compare and swap loops or libcalls will be generated. */
6627 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
6628 enum rtx_code code
, enum memmodel model
,
6631 machine_mode mode
= GET_MODE (mem
);
6632 struct atomic_op_functions optab
;
6634 bool unused_result
= (target
== const0_rtx
);
6636 get_atomic_op_for_code (&optab
, code
);
6638 /* Check to see if there are any better instructions. */
6639 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
6643 /* Check for the case where the result isn't used and try those patterns. */
6646 /* Try the memory model variant first. */
6647 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
6651 /* Next try the old style withuot a memory model. */
6652 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
6656 /* There is no no-result pattern, so try patterns with a result. */
6660 /* Try the __atomic version. */
6661 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
6665 /* Try the older __sync version. */
6666 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
6670 /* If the fetch value can be calculated from the other variation of fetch,
6671 try that operation. */
6672 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
6674 /* Try the __atomic version, then the older __sync version. */
6675 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
6677 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
6681 /* If the result isn't used, no need to do compensation code. */
6685 /* Issue compensation code. Fetch_after == fetch_before OP val.
6686 Fetch_before == after REVERSE_OP val. */
6688 code
= optab
.reverse_code
;
6691 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
6692 true, OPTAB_LIB_WIDEN
);
6693 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
6696 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6697 true, OPTAB_LIB_WIDEN
);
6702 /* No direct opcode can be generated. */
6708 /* This function expands an atomic fetch_OP or OP_fetch operation:
6709 TARGET is an option place to stick the return value. const0_rtx indicates
6710 the result is unused.
6711 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6712 CODE is the operation being performed (OP)
6713 MEMMODEL is the memory model variant to use.
6714 AFTER is true to return the result of the operation (OP_fetch).
6715 AFTER is false to return the value before the operation (fetch_OP). */
6717 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6718 enum memmodel model
, bool after
)
6720 machine_mode mode
= GET_MODE (mem
);
6722 bool unused_result
= (target
== const0_rtx
);
6724 /* If loads are not atomic for the required size and we are not called to
6725 provide a __sync builtin, do not do anything so that we stay consistent
6726 with atomic loads of the same size. */
6727 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6730 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
6736 /* Add/sub can be implemented by doing the reverse operation with -(val). */
6737 if (code
== PLUS
|| code
== MINUS
)
6740 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
6743 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
6744 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
6748 /* PLUS worked so emit the insns and return. */
6755 /* PLUS did not work, so throw away the negation code and continue. */
6759 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
6760 if (!can_compare_and_swap_p (mode
, false))
6764 enum rtx_code orig_code
= code
;
6765 struct atomic_op_functions optab
;
6767 get_atomic_op_for_code (&optab
, code
);
6768 libfunc
= optab_libfunc (after
? optab
.fetch_after
6769 : optab
.fetch_before
, mode
);
6771 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
6775 code
= optab
.reverse_code
;
6776 libfunc
= optab_libfunc (after
? optab
.fetch_before
6777 : optab
.fetch_after
, mode
);
6779 if (libfunc
!= NULL
)
6781 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6782 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
6783 2, addr
, ptr_mode
, val
, mode
);
6785 if (!unused_result
&& fixup
)
6786 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6787 true, OPTAB_LIB_WIDEN
);
6791 /* We need the original code for any further attempts. */
6795 /* If nothing else has succeeded, default to a compare and swap loop. */
6796 if (can_compare_and_swap_p (mode
, true))
6799 rtx t0
= gen_reg_rtx (mode
), t1
;
6803 /* If the result is used, get a register for it. */
6806 if (!target
|| !register_operand (target
, mode
))
6807 target
= gen_reg_rtx (mode
);
6808 /* If fetch_before, copy the value now. */
6810 emit_move_insn (target
, t0
);
6813 target
= const0_rtx
;
6818 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
6819 true, OPTAB_LIB_WIDEN
);
6820 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
6823 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
6826 /* For after, copy the value now. */
6827 if (!unused_result
&& after
)
6828 emit_move_insn (target
, t1
);
6829 insn
= get_insns ();
6832 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
6839 /* Return true if OPERAND is suitable for operand number OPNO of
6840 instruction ICODE. */
6843 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
6845 return (!insn_data
[(int) icode
].operand
[opno
].predicate
6846 || (insn_data
[(int) icode
].operand
[opno
].predicate
6847 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
6850 /* TARGET is a target of a multiword operation that we are going to
6851 implement as a series of word-mode operations. Return true if
6852 TARGET is suitable for this purpose. */
6855 valid_multiword_target_p (rtx target
)
6860 mode
= GET_MODE (target
);
6861 for (i
= 0; i
< GET_MODE_SIZE (mode
); i
+= UNITS_PER_WORD
)
6862 if (!validate_subreg (word_mode
, mode
, target
, i
))
6867 /* Like maybe_legitimize_operand, but do not change the code of the
6868 current rtx value. */
6871 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
6872 struct expand_operand
*op
)
6874 /* See if the operand matches in its current form. */
6875 if (insn_operand_matches (icode
, opno
, op
->value
))
6878 /* If the operand is a memory whose address has no side effects,
6879 try forcing the address into a non-virtual pseudo register.
6880 The check for side effects is important because copy_to_mode_reg
6881 cannot handle things like auto-modified addresses. */
6882 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
6887 addr
= XEXP (mem
, 0);
6888 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
6889 && !side_effects_p (addr
))
6894 last
= get_last_insn ();
6895 mode
= get_address_mode (mem
);
6896 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
6897 if (insn_operand_matches (icode
, opno
, mem
))
6902 delete_insns_since (last
);
6909 /* Try to make OP match operand OPNO of instruction ICODE. Return true
6910 on success, storing the new operand value back in OP. */
6913 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
6914 struct expand_operand
*op
)
6916 machine_mode mode
, imode
;
6917 bool old_volatile_ok
, result
;
6923 old_volatile_ok
= volatile_ok
;
6925 result
= maybe_legitimize_operand_same_code (icode
, opno
, op
);
6926 volatile_ok
= old_volatile_ok
;
6930 gcc_assert (mode
!= VOIDmode
);
6932 && op
->value
!= const0_rtx
6933 && GET_MODE (op
->value
) == mode
6934 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
6937 op
->value
= gen_reg_rtx (mode
);
6943 gcc_assert (mode
!= VOIDmode
);
6944 gcc_assert (GET_MODE (op
->value
) == VOIDmode
6945 || GET_MODE (op
->value
) == mode
);
6946 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
6949 op
->value
= copy_to_mode_reg (mode
, op
->value
);
6952 case EXPAND_CONVERT_TO
:
6953 gcc_assert (mode
!= VOIDmode
);
6954 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
6957 case EXPAND_CONVERT_FROM
:
6958 if (GET_MODE (op
->value
) != VOIDmode
)
6959 mode
= GET_MODE (op
->value
);
6961 /* The caller must tell us what mode this value has. */
6962 gcc_assert (mode
!= VOIDmode
);
6964 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
6965 if (imode
!= VOIDmode
&& imode
!= mode
)
6967 op
->value
= convert_modes (imode
, mode
, op
->value
, op
->unsigned_p
);
6972 case EXPAND_ADDRESS
:
6973 gcc_assert (mode
!= VOIDmode
);
6974 op
->value
= convert_memory_address (mode
, op
->value
);
6977 case EXPAND_INTEGER
:
6978 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
6979 if (mode
!= VOIDmode
&& const_int_operand (op
->value
, mode
))
6983 return insn_operand_matches (icode
, opno
, op
->value
);
6986 /* Make OP describe an input operand that should have the same value
6987 as VALUE, after any mode conversion that the target might request.
6988 TYPE is the type of VALUE. */
6991 create_convert_operand_from_type (struct expand_operand
*op
,
6992 rtx value
, tree type
)
6994 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
6995 TYPE_UNSIGNED (type
));
6998 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
6999 of instruction ICODE. Return true on success, leaving the new operand
7000 values in the OPS themselves. Emit no code on failure. */
7003 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
7004 unsigned int nops
, struct expand_operand
*ops
)
7009 last
= get_last_insn ();
7010 for (i
= 0; i
< nops
; i
++)
7011 if (!maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
7013 delete_insns_since (last
);
7019 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7020 as its operands. Return the instruction pattern on success,
7021 and emit any necessary set-up code. Return null and emit no
7025 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
7026 struct expand_operand
*ops
)
7028 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
7029 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
7035 return GEN_FCN (icode
) (ops
[0].value
);
7037 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
7039 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
7041 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7044 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7045 ops
[3].value
, ops
[4].value
);
7047 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7048 ops
[3].value
, ops
[4].value
, ops
[5].value
);
7050 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7051 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7054 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7055 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7056 ops
[6].value
, ops
[7].value
);
7058 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7059 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7060 ops
[6].value
, ops
[7].value
, ops
[8].value
);
7065 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7066 as its operands. Return true on success and emit no code on failure. */
7069 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
7070 struct expand_operand
*ops
)
7072 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7081 /* Like maybe_expand_insn, but for jumps. */
7084 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7085 struct expand_operand
*ops
)
7087 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7090 emit_jump_insn (pat
);
7096 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7100 expand_insn (enum insn_code icode
, unsigned int nops
,
7101 struct expand_operand
*ops
)
7103 if (!maybe_expand_insn (icode
, nops
, ops
))
7107 /* Like expand_insn, but for jumps. */
7110 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7111 struct expand_operand
*ops
)
7113 if (!maybe_expand_jump_insn (icode
, nops
, ops
))